From f99ee0fc517afda11bb3f7fab514f72dd6c64929 Mon Sep 17 00:00:00 2001 From: kirkscheper Date: Wed, 2 Nov 2016 19:31:24 +0100 Subject: [PATCH] update clipping of image intensity --- conf/modules/video_rtp_stream.xml | 6 ++-- sw/airborne/modules/computer_vision/cv.c | 1 + .../modules/computer_vision/cv_ae_awb.c | 30 +++++++++---------- .../computer_vision/lib/encoding/rtp.c | 3 +- .../modules/computer_vision/lib/isp/libisp.c | 6 ---- .../computer_vision/lib/isp/libisp_config.h | 2 +- sw/tools/rtp_viewer/rtp_viewer.py | 3 +- 7 files changed, 21 insertions(+), 30 deletions(-) diff --git a/conf/modules/video_rtp_stream.xml b/conf/modules/video_rtp_stream.xml index 43d8163ec1..f9523dd6fa 100644 --- a/conf/modules/video_rtp_stream.xml +++ b/conf/modules/video_rtp_stream.xml @@ -5,10 +5,7 @@ Video streaming for Linux based devices. Works e.g. in conjunction with Parrot Drones where the autopilot is the Paparazzi autopilot. - Features: - - Sends a RTP/UDP stream of the camera image, a.k.a. live video - - Possibility to save an image(shot) on internal or external storage space even in full size, best quality - Example to add to ARdrone2 airframe with extra USB stick plugger in: + Sends a RTP/UDP stream of the camera image, a.k.a. live video @@ -20,6 +17,7 @@ + diff --git a/sw/airborne/modules/computer_vision/cv.c b/sw/airborne/modules/computer_vision/cv.c index b9c14facab..5c347ee15d 100644 --- a/sw/airborne/modules/computer_vision/cv.c +++ b/sw/airborne/modules/computer_vision/cv.c @@ -115,6 +115,7 @@ int8_t cv_async_function(struct cv_async *async, struct image_t *img) } // Copy image +// TODO:this takes time causing some thread lag, should be replaced with gpu operation image_copy(img, &async->img_copy); // Inform thread of new image diff --git a/sw/airborne/modules/computer_vision/cv_ae_awb.c b/sw/airborne/modules/computer_vision/cv_ae_awb.c index 79040ff538..8ee5e11081 100644 --- a/sw/airborne/modules/computer_vision/cv_ae_awb.c +++ b/sw/airborne/modules/computer_vision/cv_ae_awb.c @@ -28,7 +28,7 @@ #include "boards/bebop/mt9f002.h" #include "lib/isp/libisp.h" -#define MAX_HIST_Y (256-10) +#define MAX_HIST_Y 255 #define sgn(x) (float)((x < 0) ? -1 : (x > 0)) @@ -59,30 +59,27 @@ void cv_ae_awb_periodic(void) } // Calculate bright and saturated pixels - uint32_t bright_pixels = cdf[MAX_HIST_Y - 1] - cdf[MAX_HIST_Y - 21]; // Top 20 bins + uint32_t bright_pixels = cdf[MAX_HIST_Y - 1] - cdf[MAX_HIST_Y - 26]; // Top 25 bins uint32_t saturated_pixels = cdf[MAX_HIST_Y - 1] - cdf[MAX_HIST_Y - 6]; // top 5 bins - uint32_t target_bright_pixels = yuv_stats.nb_valid_Y / 10; // 10% - uint32_t max_saturated_pixels = yuv_stats.nb_valid_Y / 400; // 0.25% + uint32_t target_bright_pixels = yuv_stats.nb_valid_Y / 20; // 5% + uint32_t max_saturated_pixels = yuv_stats.nb_valid_Y / 100; // 1% float adjustment = 1.0f; - printf("%d %d\n", bright_pixels, target_bright_pixels); - - // Fix saturated pixels - if (saturated_pixels > max_saturated_pixels) { - adjustment = 1.0f - ((float)(saturated_pixels - max_saturated_pixels)) / yuv_stats.nb_valid_Y; - } else if (bright_pixels + target_bright_pixels / 10 < - target_bright_pixels) { // Fix bright pixels if outside of 10% of target + if (saturated_pixels + max_saturated_pixels / 10 > max_saturated_pixels) { + // Fix saturated pixels + adjustment = 1.0f - (float)saturated_pixels / yuv_stats.nb_valid_Y; + adjustment *= adjustment * adjustment; // speed up + } else if (bright_pixels + target_bright_pixels / 10 < target_bright_pixels) { // increase brightness to try and hit the desired number of well exposed pixels - int l = MAX_HIST_Y - 11; + int l = MAX_HIST_Y - 1; while (bright_pixels < target_bright_pixels && l > 0) { bright_pixels += cdf[l]; bright_pixels -= cdf[l - 1]; l--; } - adjustment = (float)(MAX_HIST_Y - 11 + 1) / (l + 1); - } else if (bright_pixels - target_bright_pixels / 10 > - target_bright_pixels) { // Fix bright pixels if outside of 10% of target + adjustment = (float)MAX_HIST_Y / (l + 1); + } else if (bright_pixels - target_bright_pixels / 10 > target_bright_pixels) { // decrease brightness to try and hit the desired number of well exposed pixels int l = MAX_HIST_Y - 20; while (bright_pixels > target_bright_pixels && l < MAX_HIST_Y) { @@ -102,11 +99,12 @@ void cv_ae_awb_periodic(void) #endif #if CV_AUTO_WHITE_BALANCE + // It is very important that the auto exposure converges faster than the color correction // Calculate AWB and project from original scale [0,255] onto more typical scale[-0.5,0.5] float avgU = ((float) yuv_stats.awb_sum_U / (float) yuv_stats.awb_nb_grey_pixels) / 256. - 0.5; float avgV = ((float) yuv_stats.awb_sum_V / (float) yuv_stats.awb_nb_grey_pixels) / 256. - 0.5; float threshold = 0.002f; - float gain = 1.; + float gain = 0.5; bool changed = false; if (fabs(avgU) > threshold) { diff --git a/sw/airborne/modules/computer_vision/lib/encoding/rtp.c b/sw/airborne/modules/computer_vision/lib/encoding/rtp.c index b011bec61e..bd88aa219c 100644 --- a/sw/airborne/modules/computer_vision/lib/encoding/rtp.c +++ b/sw/airborne/modules/computer_vision/lib/encoding/rtp.c @@ -92,7 +92,8 @@ void rtp_frame_test(struct UdpSocket *udp) * @param[in] format_code 0 for YUV422 and 1 for YUV421 * @param[in] quality_code The JPEG encoding quality * @param[in] has_dri_header Whether we have an DRI header or not - * @param[in] delta_t Time between images in usec (if set to 0 or less it is calculated) + * @param[in] frame_time Time image was taken in usec (if set to 0 or less it is calculated) + * @param[in] packet_number The frame number of the rtp stream */ void rtp_frame_send(struct UdpSocket *udp, struct image_t *img, uint8_t format_code, uint8_t quality_code, uint8_t has_dri_header, uint32_t frame_time, uint32_t *packet_number) diff --git a/sw/airborne/modules/computer_vision/lib/isp/libisp.c b/sw/airborne/modules/computer_vision/lib/isp/libisp.c index 7997bf1f8e..9b90e6e20c 100644 --- a/sw/airborne/modules/computer_vision/lib/isp/libisp.c +++ b/sw/airborne/modules/computer_vision/lib/isp/libisp.c @@ -182,12 +182,6 @@ int configure_isp(struct v4l2_device *dev) avi_isp_lens_shading_correction_blue_coeff_mem_set_registers(&isp_ctx, &isp_config.lsc_blue_coeffs); avi_isp_bayer_set_registers(&isp_ctx, &isp_config.bayer); avi_isp_color_correction_set_registers(&isp_ctx, &isp_config.color_correction); - // printf("cc coeff_01_00: %d %d\r\n", isp_config.color_correction.coeff_01_00.coeff_00, isp_config.color_correction.coeff_01_00.coeff_01); - // printf("cc coeff_10_02: %d %d\r\n", isp_config.color_correction.coeff_10_02.coeff_02, isp_config.color_correction.coeff_10_02.coeff_10); - // printf("cc coeff_12_11: %d %d\r\n", isp_config.color_correction.coeff_12_11.coeff_11, isp_config.color_correction.coeff_12_11.coeff_12); - // printf("cc coeff_21_20: %d %d\r\n", isp_config.color_correction.coeff_21_20.coeff_20, isp_config.color_correction.coeff_21_20.coeff_21); - // printf("cc coeff_22: %d\r\n", isp_config.color_correction.coeff_22.coeff_22); - // printf("cc clip_ry: %8X\r\n", isp_config.color_correction.clip_ry._register); avi_isp_vlformat_40to32_set_registers(&isp_ctx, &isp_config.vlformat_40to32); avi_isp_gamma_corrector_set_registers(&isp_ctx, &isp_config.gamma_corrector); avi_isp_gamma_corrector_ry_lut_set_registers(&isp_ctx, &isp_config.gc_ry_lut); diff --git a/sw/airborne/modules/computer_vision/lib/isp/libisp_config.h b/sw/airborne/modules/computer_vision/lib/isp/libisp_config.h index 07d5e39109..036a46b290 100644 --- a/sw/airborne/modules/computer_vision/lib/isp/libisp_config.h +++ b/sw/airborne/modules/computer_vision/lib/isp/libisp_config.h @@ -422,7 +422,7 @@ struct libisp_config isp_config = { .coeff_21_20 = {{ 1260, 15392 }}, .coeff_22 = {{ 16179 }}, .offset_ry = {{ 0, 16 }}, - .clip_ry = {{ 16, 235 }}, + .clip_ry = {{ 16, 255 }}, .offset_gu = {{ 0, 128 }}, .clip_gu = {{ 16, 240 }}, .offset_bv = {{ 0, 128 }}, diff --git a/sw/tools/rtp_viewer/rtp_viewer.py b/sw/tools/rtp_viewer/rtp_viewer.py index 26e3bb3a7e..a6067c124e 100755 --- a/sw/tools/rtp_viewer/rtp_viewer.py +++ b/sw/tools/rtp_viewer/rtp_viewer.py @@ -3,7 +3,6 @@ import cv2 import sys import argparse -import re from os import path, getenv PPRZ_SRC = getenv("PAPARAZZI_SRC", path.normpath(path.join(path.dirname(path.abspath(__file__)), '../../../'))) @@ -78,7 +77,7 @@ class RtpViewer: self.mouse['start'] = None def on_mouse(self, event, x, y, flags, param): - if event == cv2.EVENT_LBUTTONDOWN and self.rotate == 0 and False: + if event == cv2.EVENT_LBUTTONDOWN and self.rotate == 0: self.mouse['start'] = (x, y) if event == cv2.EVENT_RBUTTONDOWN: