PCIe to i210 not working with JetPack 5.1.2

Autonomous Machines Jetson & Embedded Systems Jetson AGX Xavier
1

Hello, Nvidia community,

we developed our own carrier board for the NVIDIA Jetson AGX module which we still used with JetPack 4.6.2 but with the latest version 5.1.2 we have some trouble with the PCIe link to the i210 controller on our board. The i210 is connected the UPHY8 / UPHY9.

We modified the DT according to what we did for the JetPack 4.6.2 but it doesn’t work and we have no idea what’s wrong.

Here is what we did for the JetPack 4.6.2:

pcie@14160000 {
compatible = “nvidia,tegra194-pcie”, “snps,dw-pcie”;
power-domains = <0x3 0x12>;
reg = <0x0 0x14160000 0x0 0x20000 0x0 0x36000000 0x0 0x40000 0x0 0x36040000 0x0 0x40000>;
reg-names = “appl”, “config”, “atu_dma”;
status = “okay”;
#address-cells = <0x3>;
#size-cells = <0x2>;
device_type = “pci”;
num-lanes = <0x4>;
linux,pci-domain = <0x4>;
clocks = <0x4 0xe0 0x4 0x143>;
clock-names = “core_clk”, “core_clk_m”;
resets = <0x5 0x7d 0x5 0x78>;
reset-names = “core_apb_rst”, “core_rst”;
interrupts = <0x0 0x33 0x4 0x0 0x34 0x4>;
interrupt-names = “intr”, “msi”;
iommus = <0x2 0x5a>;
dma-coherent;
#interrupt-cells = <0x1>;
interrupt-map-mask = <0x0 0x0 0x0 0x0>;
interrupt-map = <0x0 0x0 0x0 0x0 0x1 0x0 0x33 0x4>;
nvidia,dvfs-tbl = <0xc28cb00 0xc28cb00 0xc28cb00 0x18519600 0xc28cb00 0xc28cb00 0x18519600 0x2faf0800 0xc28cb00 0x18519600 0x2faf0800 0x5f5e1000 0x0 0x0 0x0 0x0>;
nvidia,max-speed = <0x4>;
nvidia,disable-aspm-states = <0xf>;
nvidia,controller-id = <0x3 0x4>;
nvidia,disable-l1-cpm;
nvidia,aux-clk-freq = <0x13>;
nvidia,preset-init = <0x5>;
nvidia,aspm-cmrt = <0x3c>;
nvidia,aspm-pwr-on-t = <0x14>;
nvidia,aspm-l0s-entrance-latency = <0x3>;
bus-range = <0x0 0xff>;
ranges = <0x81000000 0x0 0x36100000 0x0 0x36100000 0x0 0x100000 0x82000000 0x0 0x40000000 0x17 0x40000000 0x0 0xc0000000 0xc3000000 0x14 0x0 0x14 0x0 0x3 0x40000000>;
nvidia,cfg-link-cap-l1sub = <0x1b0>;
nvidia,cap-pl16g-status = <0x174>;
nvidia,cap-pl16g-cap-off = <0x188>;
nvidia,event-cntr-ctrl = <0x1c4>;
nvidia,event-cntr-data = <0x1c8>;
nvidia,margin-port-cap = <0x190>;
nvidia,margin-lane-cntrl = <0x194>;
nvidia,dl-feature-cap = <0x2f8>;
nvidia,enable-power-down;
nvidia,disable-clock-request;
nvidia,plat-gpios;
phys = <0x160 0x161>;
phy-names = “pcie-p2u-0”, “pcie-p2u-1”;
vddio-pex-ctl-supply = <0x13>;
};

Here is what we did for the latest version 5.1.2:

pcie@14160000 {
compatible = “nvidia,tegra194-pcie\0snps,dw-pcie”;
power-domains = <0x04 0x12>;
reg = <0x00 0x14160000 0x00 0x20000 0x00 0x36000000 0x00 0x40000 0x00 0x36040000 0x00 0x40000 0x00 0x36080000 0x00 0x40000>;
reg-names = “appl\0config\0atu_dma\0dbi”;
status = “okay”;
#address-cells = <0x03>;
#size-cells = <0x02>;
device_type = “pci”;
num-lanes = <0x04>;
num-viewport = <0x08>;
linux,pci-domain = <0x04>;
clocks = <0x04 0xe0 0x04 0x143>;
clock-names = “core\0core_m”;
resets = <0x04 0x7d 0x04 0x78>;
reset-names = “apb\0core”;
interrupts = <0x00 0x33 0x04 0x00 0x34 0x04>;
interrupt-names = “intr\0msi”;
iommus = <0x02 0x5a>;
dma-coherent;
iommu-map = <0x00 0x02 0x5a 0x1000>;
iommu-map-mask = <0x00>;
interconnects = <0x03 0xe0 0x03 0xe1>;
interconnect-names = “dma-mem\0dma-mem”;
#interrupt-cells = <0x01>;
interrupt-map-mask = <0x00 0x00 0x00 0x00>;
interrupt-map = <0x00 0x00 0x00 0x00 0x01 0x00 0x33 0x04>;
nvidia,dvfs-tbl = <0xc28cb00 0xc28cb00 0xc28cb00 0x18519600 0xc28cb00 0xc28cb00 0x18519600 0x2faf0800 0xc28cb00 0x18519600 0x2faf0800 0x5f5e1000 0x00 0x00 0x00 0x00>;
nvidia,max-speed = <0x04>;
nvidia,disable-aspm-states = <0x0f>;
nvidia,disable-l1-cpm;
nvidia,aux-clk-freq = <0x13>;
nvidia,preset-init = <0x05>;
nvidia,bpmp = <0x04 0x04>;
nvidia,aspm-cmrt-us = <0x3c>;
nvidia,aspm-pwr-on-t-us = <0x14>;
nvidia,aspm-l0s-entrance-latency-us = <0x03>;
nvidia,controller-id = <0x04 0x04>;
nvidia,aspm-cmrt = <0x3c>;
nvidia,aspm-pwr-on-t = <0x14>;
nvidia,aspm-l0s-entrance-latency = <0x03>;
bus-range = <0x00 0xff>;
ranges = <0x81000000 0x00 0x36100000 0x00 0x36100000 0x00 0x100000 0xc3000000 0x14 0x00 0x14 0x00 0x03 0x40000000 0x82000000 0x00 0x40000000 0x17 0x40000000 0x00 0xc0000000>;
nvidia,cfg-link-cap-l1sub = <0x1b0>;
nvidia,cap-pl16g-status = <0x174>;
nvidia,cap-pl16g-cap-off = <0x188>;
nvidia,event-cntr-ctrl = <0x1c4>;
nvidia,event-cntr-data = <0x1c8>;
nvidia,dl-feature-cap = <0x2f8>;
nvidia,enable-power-down;
nvidia,disable-clock-request;
phys = <0x2fa 0x2fb>;
phy-names = “p2u-0”, “p2u-1”;
vddio-pex-ctl-supply = <0x15>;
};

Do you have idea / hint what is wrong in our changes or what we missed to change.

Thank you,

Christian

3

You should share what is 0x160 0x161 and what is 0x2fa 0x2fb.

They are just hex values to me if you don’t provide the mapping of this phandle.

4

Hi WayneWWW,

thanks for your fast feedback to my post.

The <0x160 0x161> for the JetPack 4.6.2 part is from the hsio_p2u description for UPHY8 and UPHY9.

p2u_8 = “/hsio_p2u/p2u@03e90000”;
p2u_9 = “/hsio_p2u/p2u@03ea0000”;

p2u@03e90000 {
compatible = “nvidia,phy-p2u”;
reg = <0x0 0x3e90000 0x0 0x10000>;
reg-names = “base”;
interrupts = <0x0 0x158 0x4>;
interrupt-names = “intr”;
nvidia,uphy-id = <0x8>;
#phy-cells = <0x0>;
linux,phandle = <0x160>;
phandle = <0x160>;
};

p2u@03ea0000 {
compatible = “nvidia,phy-p2u”;
reg = <0x0 0x3ea0000 0x0 0x10000>;
reg-names = “base”;
interrupts = <0x0 0x159 0x4>;
interrupt-names = “intr”;
nvidia,uphy-id = <0x9>;
#phy-cells = <0x0>;
linux,phandle = <0x161>;
phandle = <0x161>;
};

The <0x2fa 0x2fb> for JetPack 5.1.2 part is the also for the cbb description, this was renamed in JP 5.1.2 if we understood correct.

p2u_hsio_8 = “/cbb/p2u@3e90000”;
p2u_hsio_9 = “/cbb/p2u@3ea0000”;

p2u@3e90000 {
compatible = “nvidia,tegra194-p2u”;
reg = <0x3e90000 0x10000>;
reg-names = “ctl”;
#phy-cells = <0x00>;
phandle = <0x2fa>;
};

p2u@3ea0000 {
compatible = “nvidia,tegra194-p2u”;
reg = <0x3ea0000 0x10000>;
reg-names = “ctl”;
#phy-cells = <0x00>;
phandle = <0x2fb>;
};

Best regards,
Christian

5

Please check if any specific GPIO is needed to be enabled to make such card work.

Or try other NIC. If any of PCIe device is working, then configuring the PCIe controller is not needed.

6

Hi WayneWWW,

the hardware which we are using is working for the JetPack 4.6.2 but not working with the same changes with JetPack 5.1.2. We didn’t used any special GPIO for that PCIe device.

Maybe our changes in the code are no more enough with the JetPack 5.1.2, but here we are lost and need your support. Or is that PCIe interface only usable if another is disabled? Maybe new with JetPack 5.1.2.

I didn’t tried with another version between the 4.6.2 and 5.1.2 maybe also with another version e.g. 5.0.1 the link is still working. I can try this in the next days, but if this is also not working how should I search for the right changes in the DT?

Best regards,
Christian

7

Hi,

There is no certain answer for your question. A PCIe card could have many reasons to not enable.

My point here is simple, you don’t need to limit yourself to test only I210.

If other NIC card can work, then this PCIe is already configured correctly. For that case, we need to check something else.

If none of the PCIe can work, then we keep checking the device tree.

We didn’t used any special GPIO for that PCIe device.

It is not you to answer the question, it is the vendor to answer the question.

Or is that PCIe interface only usable if another is disabled? Maybe new with JetPack 5.1.2.

No, PCIe controllers are independent to each other.

I didn’t tried with another version between the 4.6.2 and 5.1.2 maybe also with another version e.g. 5.0.1 the link is still working. I can try this in the next days

Please also check with jp5.0.2 and jp5.1.1. The device tree for PCIe controller in this part should be all the same.

8

Hi WayneWWW,

we can not change the NIC because the i210 controller is directly soldered / installed on our carrier board.

We can not replace them without changing the design.

Okay, we will try also with the older JP versions and see if this is working or not.

I will come back to you when I have the results on that.

Best regards,
Christian

9

Hi WayneWWW,
sorry for the late response, I didn’t had the time to test it.
I retested now with older version of the JetPack 5.x.x and it looks like that only the JP 5.0.2 is working. I tried also with JP 5.1.1 and it’s the same behavior as with JP 5.1.2.

I couldn’t see that the link is going up also in the dmesg output not.
Was there anything specially changed for the PCIe port 8+9?

I think we didn’t changed anything in other files than in the dtb file, but maybe there is something changed in the pinmux file or so?

In generally there is something changed between 5.0.2 to the higher versions.

It looks like that when the board ist starting and the first NVIDIA logo is showing, the LEDs of the i210 are on and it looks like the PCIe link is also working. But when it start over into the OS then the functionality of the i210 is stopped and the LEDs of the RJ45 connector are off.

Best regards,
Christian

10

Sorry, what is port 8+9?

11

I mean with port 8 + 9 the two pcie ports

p2u_8 = “/hsio_p2u/p2u@03e90000”;
p2u_9 = “/hsio_p2u/p2u@03ea0000”;

12

Sorry that it has been a while. Is this on your carrier board or devkit?

13

on our own carrier board

14

Will the card work if you unbind and bind the driver after boot up?

15

If you mean to unbind and bind with that two commands:

cd /sys/bus/platform/drivers/tegra194-pcie
echo 14160000.pcie > unbind
echo 14160000.pcie > bind

That is not possible because after booting up I can only see the pcie devices
14100000.pcie
14140000.pcie
14180000.pcie
141a0000.pcie

14160000.pcie which would be the right one for that link is not available in the list.

I reinstalled 5.0.2 which I can use the i210 controller, I can see that the kernel driver which is used is igb. Could this be a problem? Is the igb driver no more used under 5.1.2?

16

Hi,

No, it has nothing to do with igb. The logic of a pcie device detection is the pcie driver can detect your card first and then call the igb driver.

If there is no igb driver, then the lspci shall still see the device if it is detected.

Please try to put “nvidia,disable-power-down” to the device tree and you shall see 14160000.pcie.

17

I have nvidia,enable-power-down; in my device tree.
Are you sure with nvidia,disable-power-down;??

19

Or you can directly refer to this patch first…

If you didn’t disable power down, then it will be power down after link is not detected.

That is why you said “That is not possible because after booting up I can only see the pcie devices”.

20

I tried no also with nvidia,disable-power-down; but still with the same result, I can not see the link also not under /sys/bus/platform/drivers/tegra194-pcie the link is not visible.

Maybe it helps, below the whole configuration of the pcie link:

pcie@14160000 {
compatible = “nvidia,tegra194-pcie\0snps,dw-pcie”;
power-domains = <0x04 0x12>;
reg = <0x00 0x14160000 0x00 0x20000 0x00 0x36000000 0x00 0x40000 0x00 0x36040000 0x00 0x40000 0x00 0x36080000 0x00 0x40000>;
reg-names = “appl\0config\0atu_dma\0dbi”;
status = “okay”;
#address-cells = <0x03>;
#size-cells = <0x02>;
device_type = “pci”;
num-lanes = <0x04>;
num-viewport = <0x08>;
linux,pci-domain = <0x04>;
clocks = <0x04 0xe0 0x04 0x143>;
clock-names = “core\0core_m”;
resets = <0x04 0x7d 0x04 0x78>;
reset-names = “apb\0core”;
interrupts = <0x00 0x33 0x04 0x00 0x34 0x04>;
interrupt-names = “intr\0msi”;
iommus = <0x02 0x5a>;
dma-coherent;
iommu-map = <0x00 0x02 0x5a 0x1000>;
iommu-map-mask = <0x00>;
interconnects = <0x03 0xe0 0x03 0xe1>;
interconnect-names = “dma-mem\0dma-mem”;
#interrupt-cells = <0x01>;
interrupt-map-mask = <0x00 0x00 0x00 0x00>;
interrupt-map = <0x00 0x00 0x00 0x00 0x01 0x00 0x33 0x04>;
nvidia,dvfs-tbl = <0xc28cb00 0xc28cb00 0xc28cb00 0x18519600 0xc28cb00 0xc28cb00 0x18519600 0x2faf0800 0xc28cb00 0x18519600 0x2faf0800 0x5f5e1000 0x00 0x00 0x00 0x00>;
nvidia,max-speed = <0x04>;
nvidia,disable-aspm-states = <0x0f>;
nvidia,disable-l1-cpm;
nvidia,aux-clk-freq = <0x13>;
nvidia,preset-init = <0x05>;
nvidia,bpmp = <0x04 0x04>;
nvidia,aspm-cmrt-us = <0x3c>;
nvidia,aspm-pwr-on-t-us = <0x14>;
nvidia,aspm-l0s-entrance-latency-us = <0x03>;
nvidia,controller-id = <0x04 0x04>;
nvidia,aspm-cmrt = <0x3c>;
nvidia,aspm-pwr-on-t = <0x14>;
nvidia,aspm-l0s-entrance-latency = <0x03>;
bus-range = <0x00 0xff>;
ranges = <0x81000000 0x00 0x36100000 0x00 0x36100000 0x00 0x100000 0xc3000000 0x14 0x00 0x14 0x00 0x03 0x40000000 0x82000000 0x00 0x40000000 0x17 0x40000000 0x00 0xc0000000>;
nvidia,cfg-link-cap-l1sub = <0x1b0>;
nvidia,cap-pl16g-status = <0x174>;
nvidia,cap-pl16g-cap-off = <0x188>;
nvidia,event-cntr-ctrl = <0x1c4>;
nvidia,event-cntr-data = <0x1c8>;
nvidia,dl-feature-cap = <0x2f8>;
nvidia,disable-power-down;
nvidia,disable-clock-request;
phys = <0x2fa 0x2fb>;
phy-names = “p2u-0”, “p2u-1”;
vddio-pex-ctl-supply = <0x15>;
};

Maybe you see something which is not correct configured.

21

Please share the dmesg and lspci result

22

dmesg_output.txt (73.8 KB)
lspci_output.txt (8.3 KB)

Find attached the output of dmesg and lspci