Long exposure time IMX4777

szymwit · November 2, 2021, 8:13am

At request of @dustinkerstein I’m posting my code implementing long exposure for IMX477 camera regarding to topic: Long exposure time because it has been closed. I’m posting it on new topic to make it accessible by other developers and reopen discussion about increasing of exposure time.

Below there are patches for kernel and gstnvarguscamera sources. Patches are for JetPack 4.5.1 with applied this patch first: NVIDIA-Jetson-IMX477-RPIV3/patches_nano at jetpack-4.5 · RidgeRun/NVIDIA-Jetson-IMX477-RPIV3 · GitHub.

Long exposure kernel patch:

diff --color -ruN kernel_org/hardware/nvidia/platform/t210/porg/kernel-dts/porg-platforms/tegra210-camera-rbpcv3-dual-imx477.dtsi kernel_new/hardware/nvidia/platform/t210/porg/kernel-dts/porg-platforms/tegra210-camera-rbpcv3-dual-imx477.dtsi
--- kernel_org/hardware/nvidia/platform/t210/porg/kernel-dts/porg-platforms/tegra210-camera-rbpcv3-dual-imx477.dtsi	2021-10-15 11:12:14.832714213 +0200
+++ kernel_new/hardware/nvidia/platform/t210/porg/kernel-dts/porg-platforms/tegra210-camera-rbpcv3-dual-imx477.dtsi	2021-10-15 10:53:54.968209251 +0200
@@ -221,12 +221,12 @@
 					default_gain = "16"; /* 1.00x */
 					min_hdr_ratio = "1";
 					max_hdr_ratio = "1";
-					min_framerate = "2000000"; /* 2.0 fps */
+					min_framerate = "20000"; /* 0.02 fps */
 					max_framerate = "30000000"; /* 30.0 fps */
 					step_framerate = "1";
 					default_framerate = "30000000"; /* 30.0 fps */
 					min_exp_time = "11"; /* us */
-					max_exp_time = "500000"; /* us */
+					max_exp_time = "50000000"; /* us */
 					step_exp_time = "1";
 					default_exp_time = "2495"; /* us */
 
@@ -261,12 +261,12 @@
 					default_gain = "16"; /* 1.00x */
 					min_hdr_ratio = "1";
 					max_hdr_ratio = "1";
-					min_framerate = "2000000"; /* 2.0 fps */
+					min_framerate = "20000"; /* 2.0 fps */
 					max_framerate = "60000000"; /* 60.0 fps */
 					step_framerate = "1";
 					default_framerate = "60000000"; /* 60.0 fps */
 					min_exp_time = "11"; /* us */
-					max_exp_time = "500000"; /* us */
+					max_exp_time = "50000000"; /* us */
 					step_exp_time = "1";
 					default_exp_time = "2495"; /* us */
 
diff --color -ruN kernel_org/hardware/nvidia/platform/t210/porg/kernel-dts/porg-platforms/tegra210-camera-rbpcv3-imx477.dtsi kernel_new/hardware/nvidia/platform/t210/porg/kernel-dts/porg-platforms/tegra210-camera-rbpcv3-imx477.dtsi
--- kernel_org/hardware/nvidia/platform/t210/porg/kernel-dts/porg-platforms/tegra210-camera-rbpcv3-imx477.dtsi	2021-10-15 11:12:14.832714213 +0200
+++ kernel_new/hardware/nvidia/platform/t210/porg/kernel-dts/porg-platforms/tegra210-camera-rbpcv3-imx477.dtsi	2021-10-15 10:53:54.968209251 +0200
@@ -190,12 +190,12 @@
 					default_gain = "16"; /* 1.00x */
 					min_hdr_ratio = "1";
 					max_hdr_ratio = "1";
-					min_framerate = "2000000"; /* 2.0 fps */
+					min_framerate = "20000"; /* 0.02 fps */
 					max_framerate = "30000000"; /* 30.0 fps */
 					step_framerate = "1";
 					default_framerate = "30000000"; /* 30.0 fps */
 					min_exp_time = "11"; /* us */
-					max_exp_time = "500000"; /* us */
+					max_exp_time = "50000000"; /* us */
 					step_exp_time = "1";
 					default_exp_time = "2495"; /* us */
 
@@ -230,12 +230,12 @@
 					default_gain = "16"; /* 1.00x */
 					min_hdr_ratio = "1";
 					max_hdr_ratio = "1";
-					min_framerate = "2000000"; /* 2.0 fps */
+					min_framerate = "20000"; /* 0.02 fps */
 					max_framerate = "60000000"; /* 60.0 fps */
 					step_framerate = "1";
 					default_framerate = "60000000"; /* 60.0 fps */
 					min_exp_time = "11"; /* us */
-					max_exp_time = "500000"; /* us */
+					max_exp_time = "50000000"; /* us */
 					step_exp_time = "1";
 					default_exp_time = "2495"; /* us */
 
diff --color -ruN kernel_org/kernel/nvidia/drivers/media/i2c/imx477.c kernel_new/kernel/nvidia/drivers/media/i2c/imx477.c
--- kernel_org/kernel/nvidia/drivers/media/i2c/imx477.c	2021-10-15 11:12:14.832714213 +0200
+++ kernel_new/kernel/nvidia/drivers/media/i2c/imx477.c	2021-10-15 10:52:21.150927499 +0200
@@ -55,6 +55,7 @@
 	u32 frame_length;
 	struct camera_common_data *s_data;
 	struct tegracam_device *tc_dev;
+	u8 long_exp_shift;
 };
 
 static const struct regmap_config sensor_regmap_config = {
@@ -221,6 +222,7 @@
 	imx477_reg fl_regs[2];
 	u32 frame_length;
 	int i;
+	u8 long_exp_shift = 0;
 
 	dev_dbg(dev, "%s: Setting framerate control to: %lld\n", __func__, val);
 
@@ -228,14 +230,21 @@
 			      (u64) mode->control_properties.framerate_factor /
 			      mode->image_properties.line_length / val);
 
+	while ( frame_length > IMX477_MAX_FRAME_LENGTH &&
+	        long_exp_shift < IMX477_MAX_LONG_EXP_SHIFT ) {
+
+		long_exp_shift++;
+		frame_length >>= 1;
+	}
+
 	if (frame_length < IMX477_MIN_FRAME_LENGTH)
 		frame_length = IMX477_MIN_FRAME_LENGTH;
 	else if (frame_length > IMX477_MAX_FRAME_LENGTH)
 		frame_length = IMX477_MAX_FRAME_LENGTH;
 
 	dev_dbg(dev,
-		"%s: val: %llde-6 [fps], frame_length: %u [lines]\n",
-		__func__, val, frame_length);
+		"%s: val: %llde-6 [fps], frame_length: %u [lines], long_exp_shift: %u\n",
+		__func__, val, frame_length, long_exp_shift);
 
 	imx477_get_frame_length_regs(fl_regs, frame_length);
 	for (i = 0; i < 2; i++) {
@@ -247,7 +256,15 @@
 		}
 	}
 
+	err = imx477_write_reg(s_data, IMX477_LONG_EXP_SHIFT_ADDR, long_exp_shift);
+	if (err) {
+		dev_err(dev,
+			"%s: frame_length control error: failed to write long exp shift\n", __func__);
+		return err;
+	}
+	
 	priv->frame_length = frame_length;
+	priv->long_exp_shift = long_exp_shift;
 
 	return err;
 }
@@ -268,8 +285,8 @@
 	u32 int_factor = val * mode->signal_properties.pixel_clock.val /
 	    mode->control_properties.exposure_factor;
 
-	coarse_time = int_factor / mode->image_properties.line_length;
-	fine_int_time = int_factor % mode->image_properties.line_length;
+	coarse_time = (int_factor / mode->image_properties.line_length) >> priv->long_exp_shift;
+	fine_int_time = (int_factor % mode->image_properties.line_length) >> priv->long_exp_shift;
 
 	dev_dbg(dev, "%s: Setting exposure control to: %lld\n", __func__, val);
 
diff --color -ruN kernel_org/kernel/nvidia/include/media/imx477.h kernel_new/kernel/nvidia/include/media/imx477.h
--- kernel_org/kernel/nvidia/include/media/imx477.h	2021-10-15 10:55:53.709904180 +0200
+++ kernel_new/kernel/nvidia/include/media/imx477.h	2021-10-15 10:52:20.074912715 +0200
@@ -32,6 +32,7 @@
 #define IMX477_MAX_COARSE_DIFF		        (10)
 #define IMX477_MASK_LSB_2_BITS			0x0003
 #define IMX477_MASK_LSB_8_BITS			0x00ff
+#define IMX477_MAX_LONG_EXP_SHIFT	    7
 
 /* imx477 sensor register address */
 #define IMX477_MODEL_ID_ADDR_MSB		0x0000
@@ -47,5 +48,6 @@
 #define IMX477_FINE_INTEG_TIME_ADDR_MSB		0x0200
 #define IMX477_FINE_INTEG_TIME_ADDR_LSB		0x0201
 #define IMX477_GROUP_HOLD_ADDR		        0x0104
+#define IMX477_LONG_EXP_SHIFT_ADDR	    0x3100
 
 #endif /* __IMX477_H__ */

gstnvarguscamera patch:

diff --color -ruN gst-nvarguscamera_org/gstnvarguscamerasrc.cpp gst-nvarguscamera_new/gstnvarguscamerasrc.cpp
--- gst-nvarguscamera_org/gstnvarguscamerasrc.cpp	2021-10-15 11:35:58.428253998 +0200
+++ gst-nvarguscamera_new/gstnvarguscamerasrc.cpp	2021-10-15 11:38:48.434674148 +0200
@@ -70,18 +70,18 @@
 #define MAX_BUFFERS 8
 
 #define MIN_GAIN 1
-#define MAX_GAIN 16
+#define MAX_GAIN 22
 
 #define MIN_EXPOSURE_TIME 34000
-#define MAX_EXPOSURE_TIME 358733000
+#define MAX_EXPOSURE_TIME 50000000000
 
 #define MIN_DIGITAL_GAIN 1
 #define MAX_DIGITAL_GAIN 256
 
 #define GST_NVARGUS_MEMORY_TYPE "nvarguscam"
 static const int DEFAULT_FPS        = 30;
-static const uint64_t TIMEOUT_FIVE_SECONDS  = 5000000000;
-static const uint64_t ACQUIRE_FRAME_TIMEOUT = 1000000000;
+static const uint64_t TIMEOUT_FIVE_SECONDS  = 50000000000;
+static const uint64_t ACQUIRE_FRAME_TIMEOUT = 10000000000;
 
 #ifdef __cplusplus
 extern "C"
@@ -725,7 +725,7 @@
   src->total_sensor_modes = modes.size();
 
  GST_ARGUS_PRINT("Available Sensor modes :\n");
-  frameRate = src->fps_n/ src->fps_d;
+  frameRate = src->fps_n/ static_cast<gfloat>(src->fps_d);
   duration = 1e9 * src->fps_d/ src->fps_n;
   ISensorMode *iSensorMode[modes.size()];
   Range<float> sensorModeAnalogGainRange;
Binarne pliki gst-nvarguscamera_org/gstnvarguscamerasrc.o i gst-nvarguscamera_new/gstnvarguscamerasrc.o różnią się
Binarne pliki gst-nvarguscamera_org/gstnvarguscamera_utils.o i gst-nvarguscamera_new/gstnvarguscamera_utils.o różnią się
Binarne pliki gst-nvarguscamera_org/libgstnvarguscamerasrc.so i gst-nvarguscamera_new/libgstnvarguscamerasrc.so różnią się
diff --color -ruN gst-nvarguscamera_org/Makefile gst-nvarguscamera_new/Makefile
--- gst-nvarguscamera_org/Makefile	2021-10-15 11:35:58.428253998 +0200
+++ gst-nvarguscamera_new/Makefile	2021-10-15 11:35:58.656257113 +0200
@@ -49,6 +49,8 @@
 
 OBJS := $(SRCS:.cpp=.o)
 
+CFLAGS += -DDEBUG -g
+
 CFLAGS += -fPIC
 
 CFLAGS += `pkg-config --cflags $(PKGS)`

I used below instruction to rebuild the kernel:

ShaneCCC · November 2, 2021, 6:50pm

Thanks for sharing this.

dustinkerstein · November 4, 2021, 12:16pm

Thanks for posting this @szymwit

@ShaneCCC, given the simplicity of the above code changes, do you think it’s possible to include these changes in upcoming L4T releases (for the IMX477 and maybe other similar sensors)? Even though this change only applies to video streaming modes (rather than an ideal software/hardware trigger-based still capture), having access to these longer exposure values is extremely helpful for certain use-cases. And I don’t see a particular downside to allowing access to them. Do you? Thanks!

ShaneCCC · November 4, 2021, 1:21pm

Suppose have difficult to merge it due may cause some test failed due to extremely low frame rate.

dustinkerstein · November 4, 2021, 1:23pm

Could the tests be updated to support the slower framerates?

system · December 1, 2021, 2:58am

This topic was automatically closed 14 days after the last reply. New replies are no longer allowed.