Hello,
I have a sensor (onSemi Ar0231) connected to a Jetson AGX Xavier over GMSL2. I want to get the embedded data out from the sensor through the NVIDIA pipeline. This seemed like an easy task in the beginning but I am finding it more complicated than I first thought.
I first enabled the appropriate registers to allow the sensor to output the embedded data. According to the datasheet, two lines of embedded data are prepended on the frame output (and the statistics data is appended to the frame buffer).
For now, I am focusing on only the embedded data. Although this post has some information about getting the statistics data out, I think I will focus on that after I get this first step done of getting the embedded data out.
After setting the appropriate registers in the sensor, I modified the device tree node of the sensor to have embedded_metadata_height=2
because the datasheet for the sensor mentions 2 rows of data get prepended.
That unfortunately didn’t work. After searching around on the forum more, I found this post (which pointed to this other post). So I modified the vi5_fops.c vi5_capture_dequeue
function. Here is the modified version:
static void vi5_capture_dequeue(struct tegra_channel *chan,
struct tegra_channel_buffer *buf)
{
int err = 0;
int vi_port = 0;
int gang_prev_frame_id = 0;
unsigned long flags;
struct tegra_mc_vi *vi = chan->vi;
struct vb2_v4l2_buffer *vb = &buf->buf;
struct timespec ts;
struct capture_descriptor *descr = NULL;
void* frm_buffer = vb2_plane_vaddr(&(vb->vb2_buf), 0);
pr_info("%s: a. embedded_buffer_size: %d\n", __func__, chan->vi->emb_buf_size);
for(vi_port = 0; vi_port < chan->valid_ports; vi_port++) {
descr = &chan->request[vi_port][buf->capture_descr_index[vi_port]];
if (buf->vb2_state != VB2_BUF_STATE_ACTIVE)
goto rel_buf;
/* Dequeue a frame and check its capture status */
pr_info("%s: b. embedded_buffer_size: %d, vi_port: %d\n", __func__, chan->vi->emb_buf_size, vi_port);
if(frm_buffer != NULL) {
pr_info("%s: copying the embedded data onto the frame buffer\n", __func__);
memcpy(frm_buffer,chan->vi->emb_buf_addr,chan->vi->emb_buf_size);
}
err = vi_capture_status(chan->tegra_vi_channel[vi_port], CAPTURE_TIMEOUT_MS);
if (err) {
if (err == -ETIMEDOUT) {
dev_err(vi->dev,
"uncorr_err: request timed out after %d ms\n",
CAPTURE_TIMEOUT_MS);
} else {
dev_err(vi->dev, "uncorr_err: request err %d\n", err);
}
goto uncorr_err;
} else if (descr->status.status != CAPTURE_STATUS_SUCCESS) {
if ((descr->status.flags
& CAPTURE_STATUS_FLAG_CHANNEL_IN_ERROR) != 0) {
chan->queue_error = true;
dev_err(vi->dev, "uncorr_err: flags %d, err_data %d\n",
descr->status.flags, descr->status.err_data);
} else {
dev_warn(vi->dev,
"corr_err: discarding frame %d, flags: %d, "
"err_data %d\n",
descr->status.frame_id, descr->status.flags,
descr->status.err_data);
buf->vb2_state = VB2_BUF_STATE_REQUEUEING;
goto done;
}
} else if (!vi_port) {
gang_prev_frame_id = descr->status.frame_id;
} else if (descr->status.frame_id != gang_prev_frame_id) {
dev_err(vi->dev, "frame_id out of sync: ch2 %d vs ch1 %d\n",
gang_prev_frame_id, descr->status.frame_id);
goto uncorr_err;
}
pr_info("%s: c. embedded_buffer_size: %d, vi_port: %d\n", __func__, chan->vi->emb_buf_size, vi_port);
spin_lock_irqsave(&chan->capture_state_lock, flags);
if (chan->capture_state != CAPTURE_ERROR) {
chan->capture_reqs_enqueued -= 1;
chan->capture_state = CAPTURE_GOOD;
}
spin_unlock_irqrestore(&chan->capture_state_lock, flags);
pr_info("%s: d. embedded_buffer_size: %d, vi_port: %d\n", __func__, chan->vi->emb_buf_size, vi_port);
}
pr_info("%s: e. embedded_buffer_size: %d, vi_port: %d\n", __func__, chan->vi->emb_buf_size, vi_port);
wake_up_interruptible(&chan->start_wait);
/* Read SOF from capture descriptor */
ts = ns_to_timespec((s64)descr->status.sof_timestamp);
trace_tegra_channel_capture_frame("sof", ts);
#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 9, 0)
/* update time stamp of the buffer */
vb->timestamp.tv_sec = ts.tv_sec;
vb->timestamp.tv_usec = ts.tv_nsec / NSEC_PER_USEC;
#else
vb->vb2_buf.timestamp = descr->status.sof_timestamp;
#endif
buf->vb2_state = VB2_BUF_STATE_DONE;
/* Read EOF from capture descriptor */
ts = ns_to_timespec((s64)descr->status.eof_timestamp);
trace_tegra_channel_capture_frame("eof", ts);
done:
goto rel_buf;
uncorr_err:
spin_lock_irqsave(&chan->capture_state_lock, flags);
chan->capture_state = CAPTURE_ERROR;
spin_unlock_irqrestore(&chan->capture_state_lock, flags);
buf->vb2_state = VB2_BUF_STATE_ERROR;
rel_buf:
vi5_release_buffer(chan, buf);
}
That also didn’t work.
Then I experimented some more with the embedded_metadata_height
parameter in the device tree. Here are some of my observations:
Dmesg output Snippets
- embedded_metadata_height = 0
[ 225.118625] ar0231 2-0009: frame size index 0x0001 is not handled
[ 225.118763] ar0231 2-0009: frame size index 0x0001 is not handled
[ 225.123216] vi5_capture_dequeue: a. embedded_buffer_size: 0
[ 225.123343] vi5_capture_dequeue: b. embedded_buffer_size: 0, vi_port: 0
[ 225.123453] vi5_capture_dequeue: copying the embedded data onto the frame buffer
[ 225.142269] [RCE] Configuring VI GoS.
[ 225.142354] [RCE] VM GOS[#0] addr=0xc2100000
[ 225.142435] [RCE] VM GOS[#1] addr=0xc2101000
[ 225.142520] [RCE] VM GOS[#2] addr=0xc2102000
[ 225.142602] [RCE] VM GOS[#3] addr=0xc2103000
[ 225.142678] [RCE] VM GOS[#4] addr=0xc2104000
[ 225.142758] [RCE] VM GOS[#5] addr=0xc2105000
[ 225.142840] [RCE] vi5_hwinit: firmware CL2018101701 protocol version 2.2
[ 225.142955] [RCE] VI GOS[#0] set to VM GOS[4] base 0xc2104000
[ 225.210987] vi5_capture_dequeue: c. embedded_buffer_size: 0, vi_port: 0
[ 225.211127] vi5_capture_dequeue: d. embedded_buffer_size: 0, vi_port: 0
[ 225.211239] vi5_capture_dequeue: e. embedded_buffer_size: 0, vi_port: 1
[ 225.211376] vi5_capture_dequeue: a. embedded_buffer_size: 0
[ 225.211470] vi5_capture_dequeue: b. embedded_buffer_size: 0, vi_port: 0
[ 225.211579] vi5_capture_dequeue: copying the embedded data onto the frame buffer
[ 225.244240] vi5_capture_dequeue: c. embedded_buffer_size: 0, vi_port: 0
[ 225.244395] vi5_capture_dequeue: d. embedded_buffer_size: 0, vi_port: 0
[ 225.244506] vi5_capture_dequeue: e. embedded_buffer_size: 0, vi_port: 1
[ 225.358176] vi5_capture_dequeue: a. embedded_buffer_size: 0
[ 225.358362] vi5_capture_dequeue: b. embedded_buffer_size: 0, vi_port: 0
[ 225.358471] vi5_capture_dequeue: copying the embedded data onto the frame buffer
[ 225.410884] vi5_capture_dequeue: c. embedded_buffer_size: 0, vi_port: 0
[ 225.411020] vi5_capture_dequeue: d. embedded_buffer_size: 0, vi_port: 0
[ 225.411132] vi5_capture_dequeue: e. embedded_buffer_size: 0, vi_port: 1
- embedded_metadata_height = 1
[ 623.975716] ar0231 2-0009: frame size index 0x0001 is not handled
[ 623.975877] ar0231 2-0009: frame size index 0x0001 is not handled
[ 623.980052] vi5_capture_dequeue: a. embedded_buffer_size: 4096
[ 623.980169] vi5_capture_dequeue: b. embedded_buffer_size: 4096, vi_port: 0
[ 623.980278] vi5_capture_dequeue: copying the embedded data onto the frame buffer
[ 623.988974] [RCE] Configuring VI GoS.
[ 623.989055] [RCE] VM GOS[#0] addr=0xc2100000
[ 623.989148] [RCE] VM GOS[#1] addr=0xc2101000
[ 623.989240] [RCE] VM GOS[#2] addr=0xc2102000
[ 623.989340] [RCE] VM GOS[#3] addr=0xc2103000
[ 623.989431] [RCE] VM GOS[#4] addr=0xc2104000
[ 623.989519] [RCE] VM GOS[#5] addr=0xc2105000
[ 623.989614] [RCE] vi5_hwinit: firmware CL2018101701 protocol version 2.2
[ 623.989737] [RCE] VI GOS[#0] set to VM GOS[4] base 0xc2104000
[ 626.560956] tegra194-vi5 15c10000.vi: no reply from camera processor
[ 626.561118] tegra194-vi5 15c10000.vi: uncorr_err: request timed out after 2500 ms
[ 626.561252] tegra194-vi5 15c10000.vi: err_rec: attempting to reset the capture channel
[ 626.564146] vi5_capture_dequeue: a. embedded_buffer_size: 4096
[ 626.564571] tegra-capture-ivc ivc-bc00000.rtcpu:ivccontrol@3: No callback found for msg id: 0x39
[ 626.564719] tegra-capture-ivc ivc-bc00000.rtcpu:ivccontrol@3: No callback found for msg id: 0x41
[ 626.564874] tegra-capture-ivc ivc-bc00000.rtcpu:ivccontrol@3: No callback found for msg id: 0x37
[ 626.565022] [RCE] Configuring VI GoS.
[ 626.565026] [RCE] VM GOS[#0] addr=0xc2100000
[ 626.565028] [RCE] VM GOS[#1] addr=0xc2101000
[ 626.565030] [RCE] VM GOS[#2] addr=0xc2102000
[ 626.565032] [RCE] VM GOS[#3] addr=0xc2103000
[ 626.565034] [RCE] VM GOS[#4] addr=0xc2104000
[ 626.565047] [RCE] VM GOS[#5] addr=0xc2105000
[ 626.565563] tegra194-vi5 15c10000.vi: err_rec: successfully reset the capture channel
[ 626.705634] vi5_capture_dequeue: a. embedded_buffer_size: 4096
[ 626.705758] vi5_capture_dequeue: b. embedded_buffer_size: 4096, vi_port: 0
[ 626.705867] vi5_capture_dequeue: copying the embedded data onto the frame buffer
[ 629.376961] tegra194-vi5 15c10000.vi: no reply from camera processor
[ 629.377094] tegra194-vi5 15c10000.vi: uncorr_err: request timed out after 2500 ms
[ 629.377239] tegra194-vi5 15c10000.vi: err_rec: attempting to reset the capture channel
[ 629.381495] tegra194-vi5 15c10000.vi: err_rec: successfully reset the capture channel
[ 629.421013] [RCE] WARNING: t194/vi5.c:459 [vi5_channel_disable] "ch 35 ABORT trial# 2/5 unsuccessful"
[ 629.421200] [RCE] Configuring VI GoS.
[ 629.421263] [RCE] VM GOS[#0] addr=0xc2100000
[ 629.421346] [RCE] VM GOS[#1] addr=0xc2101000
[ 629.421427] [RCE] VM GOS[#2] addr=0xc2102000
[ 629.421498] [RCE] VM GOS[#3] addr=0xc2103000
[ 629.421594] [RCE] VM GOS[#4] addr=0xc2104000
[ 629.421672] [RCE] VM GOS[#5] addr=0xc2105000
- embedded_metadata_height = 2
[ 45.681975] ar0231 2-0009: frame size index 0x0001 is not handled
[ 45.682115] ar0231 2-0009: frame size index 0x0001 is not handled
[ 45.689542] vi5_capture_dequeue: a. embedded_buffer_size: 8192
[ 45.689687] vi5_capture_dequeue: b. embedded_buffer_size: 8192, vi_port: 0
[ 45.689823] vi5_capture_dequeue: copying the embedded data onto the frame buffer
[ 45.741125] [RCE] Configuring VI GoS.
[ 45.741249] [RCE] VM GOS[#0] addr=0xc2100000
[ 45.741336] [RCE] VM GOS[#1] addr=0xc2101000
[ 45.741407] [RCE] VM GOS[#2] addr=0xc2102000
[ 45.741478] [RCE] VM GOS[#3] addr=0xc2103000
[ 45.741564] [RCE] VM GOS[#4] addr=0xc2104000
[ 45.741635] [RCE] VM GOS[#5] addr=0xc2105000
[ 45.741708] [RCE] vi5_hwinit: firmware CL2018101701 protocol version 2.2
[ 45.741837] [RCE] VI GOS[#0] set to VM GOS[4] base 0xc2104000
[ 48.245134] tegra194-vi5 15c10000.vi: no reply from camera processor
[ 48.245271] tegra194-vi5 15c10000.vi: uncorr_err: request timed out after 2500 ms
[ 48.245439] tegra194-vi5 15c10000.vi: err_rec: attempting to reset the capture channel
[ 48.248285] vi5_capture_dequeue: a. embedded_buffer_size: 8192
[ 48.248688] tegra-capture-ivc ivc-bc00000.rtcpu:ivccontrol@3: No callback found for msg id: 0x39
[ 48.248837] tegra-capture-ivc ivc-bc00000.rtcpu:ivccontrol@3: No callback found for msg id: 0x41
[ 48.248976] tegra-capture-ivc ivc-bc00000.rtcpu:ivccontrol@3: No callback found for msg id: 0x37
[ 48.249168] tegra194-vi5 15c10000.vi: err_rec: successfully reset the capture channel
[ 48.261142] [RCE] Configuring VI GoS.
[ 48.261235] [RCE] VM GOS[#0] addr=0xc2100000
[ 48.261310] [RCE] VM GOS[#1] addr=0xc2101000
[ 48.261380] [RCE] VM GOS[#2] addr=0xc2102000
[ 48.261474] [RCE] VM GOS[#3] addr=0xc2103000
[ 48.261553] [RCE] VM GOS[#4] addr=0xc2104000
[ 48.261623] [RCE] VM GOS[#5] addr=0xc2105000
[ 48.408416] vi5_capture_dequeue: a. embedded_buffer_size: 8192
[ 48.408540] vi5_capture_dequeue: b. embedded_buffer_size: 8192, vi_port: 0
[ 48.408660] vi5_capture_dequeue: copying the embedded data onto the frame buffer
[ 51.061082] tegra194-vi5 15c10000.vi: no reply from camera processor
[ 51.061214] tegra194-vi5 15c10000.vi: uncorr_err: request timed out after 2500 ms
[ 51.061335] tegra194-vi5 15c10000.vi: err_rec: attempting to reset the capture channel
[ 51.075065] tegra194-vi5 15c10000.vi: err_rec: successfully reset the capture channel
[ 51.118294] [RCE] Configuring VI GoS.
[ 51.118390] [RCE] VM GOS[#0] addr=0xc2100000
[ 51.118593] [RCE] VM GOS[#1] addr=0xc2101000
[ 51.118678] [RCE] VM GOS[#2] addr=0xc2102000
[ 51.118749] [RCE] VM GOS[#3] addr=0xc2103000
[ 51.118830] [RCE] VM GOS[#4] addr=0xc2104000
[ 51.118926] [RCE] VM GOS[#5] addr=0xc2105000
Trace Outputs
-
embedded_metadata_height = 0
trace_embedded_metadata_height=0.txt|attachment (13.7 KB) -
embedded_metadata_height = 1
trace_embedded_metadata_height=1.txt|attachment (109.3 KB) -
embedded_metadata_height = 2
trace_embedded_metadata_height=2.txt|attachment (966.1 KB)
Here are the dtc files for your reference also.
live_dtc_embedded_metadata_height=0.txt|attachment (318.2 KB)
live_dtc_embedded_metadata_height=1.txt|attachment (318.2 KB)
live_dtc_embedded_metadata_height=2.txt|attachment (318.2 KB)
I have also read some documentation on the embedded data in the TRM. It talks about EMBED_Y.LINES, EMBED_Y.EXPECT and EMBED_Y.ENABLE. I tried finding those in the software but couldn’t.
Any help will be very much appreciated. Thanks!