I do something try:
1. Seems there are 4 PCIe on my board: network controller, Ethernet controller, memory controller and the one my target to set EP mode. Evidence is:
when I run lspci -nn:
0001:00:00.0 PCI bridge [0604]: NVIDIA Corporation Device [10de:229e] (rev a1)
0001:01:00.0 Network controller [0280]: Realtek Semiconductor Co., Ltd. RTL8822CE 802.11ac PCIe Wireless Network Adapter [10ec:c822]
0004:00:00.0 PCI bridge [0604]: NVIDIA Corporation Device [10de:229c] (rev a1)
0004:01:00.0 Non-Volatile memory controller [0108]: Realtek Semiconductor Co., Ltd. Device [10ec:5765] (rev 01)
0008:00:00.0 PCI bridge [0604]: NVIDIA Corporation Device [10de:229c] (rev a1)
0008:01:00.0 Ethernet controller [0200]: Realtek Semiconductor Co., Ltd. RTL8111/8168/8411 PCI Express Gigabit Ethernet Controller [10ec:8168] (rev 15)
and when I run sudo dmesg | grep -i pcie:
[ 4.924630] tegra194-pcie 14100000.pcie: Adding to iommu group 3
[ 4.927485] tegra194-pcie 14100000.pcie: host bridge /bus@0/pcie@14100000 ranges:
[ 4.927509] tegra194-pcie 14100000.pcie: MEM 0x2080000000..0x20a7ffffff → 0x2080000000
[ 4.927515] tegra194-pcie 14100000.pcie: MEM 0x20a8000000..0x20afffffff → 0x0040000000
[ 4.927519] tegra194-pcie 14100000.pcie: IO 0x0030100000..0x00301fffff → 0x0030100000
[ 4.927831] tegra194-pcie 14100000.pcie: iATU unroll: enabled
[ 4.927833] tegra194-pcie 14100000.pcie: Detected iATU regions: 8 outbound, 2 inbound
[ 5.032548] tegra194-pcie 14100000.pcie: Link up
[ 5.034008] tegra194-pcie 14100000.pcie: Link up
[ 5.034072] tegra194-pcie 14100000.pcie: PCI host bridge to bus 0001:00
[ 5.051051] pcieport 0001:00:00.0: Adding to iommu group 3
[ 5.051143] pcieport 0001:00:00.0: PME: Signaling with IRQ 190
[ 5.052504] pcieport 0001:00:00.0: AER: enabled with IRQ 190
[ 5.053376] tegra194-pcie 14160000.pcie: Adding to iommu group 5
[ 5.057411] tegra194-pcie 14160000.pcie: host bridge /bus@0/pcie@14160000 ranges:
[ 5.057429] tegra194-pcie 14160000.pcie: MEM 0x2140000000..0x2427ffffff → 0x2140000000
[ 5.057436] tegra194-pcie 14160000.pcie: MEM 0x2428000000..0x242fffffff → 0x0040000000
[ 5.057439] tegra194-pcie 14160000.pcie: IO 0x0036100000..0x00361fffff → 0x0036100000
[ 5.057934] tegra194-pcie 14160000.pcie: iATU unroll: enabled
[ 5.057937] tegra194-pcie 14160000.pcie: Detected iATU regions: 8 outbound, 2 inbound
[ 5.164299] tegra194-pcie 14160000.pcie: Link up
[ 5.165662] tegra194-pcie 14160000.pcie: Link up
[ 5.165713] tegra194-pcie 14160000.pcie: PCI host bridge to bus 0004:00
[ 5.182945] pcieport 0004:00:00.0: Adding to iommu group 5
[ 5.183026] pcieport 0004:00:00.0: PME: Signaling with IRQ 192
[ 5.184296] pcieport 0004:00:00.0: AER: enabled with IRQ 192
[ 5.185760] tegra194-pcie 141e0000.pcie: Adding to iommu group 6
[ 5.188008] tegra194-pcie 141e0000.pcie: host bridge /bus@0/pcie@141e0000 ranges:
[ 5.188024] tegra194-pcie 141e0000.pcie: MEM 0x3000000000..0x3227ffffff → 0x3000000000
[ 5.188031] tegra194-pcie 141e0000.pcie: MEM 0x3228000000..0x322fffffff → 0x0040000000
[ 5.188035] tegra194-pcie 141e0000.pcie: IO 0x003e100000..0x003e1fffff → 0x003e100000
[ 5.188532] tegra194-pcie 141e0000.pcie: iATU unroll: enabled
[ 5.188535] tegra194-pcie 141e0000.pcie: Detected iATU regions: 8 outbound, 2 inbound
[ 6.300303] tegra194-pcie 141e0000.pcie: Phy link never came up
[ 7.300336] tegra194-pcie 141e0000.pcie: Phy link never came up
[ 7.300498] tegra194-pcie 141e0000.pcie: PCI host bridge to bus 0007:00
[ 7.312697] pcieport 0007:00:00.0: Adding to iommu group 6
[ 7.325601] pcieport 0007:00:00.0: PME: Signaling with IRQ 194
[ 7.334600] pcieport 0007:00:00.0: AER: enabled with IRQ 194
[ 7.378497] tegra194-pcie 140a0000.pcie: Adding to iommu group 51
[ 7.520839] tegra194-pcie 140a0000.pcie: host bridge /bus@0/pcie@140a0000 ranges:
[ 7.520870] tegra194-pcie 140a0000.pcie: MEM 0x3240000000..0x3527ffffff → 0x3240000000
[ 7.520876] tegra194-pcie 140a0000.pcie: MEM 0x3528000000..0x352fffffff → 0x0040000000
[ 7.520879] tegra194-pcie 140a0000.pcie: IO 0x002a100000..0x002a1fffff → 0x002a100000
[ 7.529240] tegra194-pcie 140a0000.pcie: iATU unroll: enabled
[ 7.529248] tegra194-pcie 140a0000.pcie: Detected iATU regions: 8 outbound, 2 inbound
[ 7.638242] tegra194-pcie 140a0000.pcie: Link up
[ 7.641045] tegra194-pcie 140a0000.pcie: Link up
[ 7.647848] tegra194-pcie 140a0000.pcie: PCI host bridge to bus 0008:00
[ 7.704031] pcieport 0008:00:00.0: Adding to iommu group 51
[ 7.704189] pcieport 0008:00:00.0: PME: Signaling with IRQ 188
[ 7.706069] pcieport 0008:00:00.0: AER: enabled with IRQ 188
So I think 141e0000 is my target PCIe.
2. Then I try to set EP mode only for 141e0000 (not set ODMDATA as tutorial), by modifying /boot/dtb/kernel_tegra234-p3768-0000+p3767-0000-nv-super.dtb to change the status about its pcie & pcie-ep:
pcie@141e0000 {
compatible = “nvidia,tegra234-pcie”;
power-domains = <0x03 0x10>;
reg = <0x00 0x141e0000 0x00 0x20000 0x00 0x3e000000 0x00 0x40000 0x00 0x3e040000 0x00 0x40000 0x00 0x3e080000 0x00 0x40000 0x32 0x30000000 0x00 0x10000000>;
reg-names = “appl\0config\0atu_dma\0dbi\0ecam”;
#address-cells = <0x03>;
#size-cells = <0x02>;
device_type = “pci”;
num-lanes = <0x08>;
num-viewport = <0x08>;
linux,pci-domain = <0x07>;
clocks = <0x03 0xab>;
clock-names = “core”;
resets = <0x03 0x0f 0x03 0x0e>;
reset-names = “apb\0core”;
interrupts = <0x00 0x162 0x04 0x00 0x163 0x04>;
interrupt-names = “intr\0msi”;
#interrupt-cells = <0x01>;
interrupt-map-mask = <0x00 0x00 0x00 0x00>;
interrupt-map = <0x00 0x00 0x00 0x00 0x01 0x00 0x162 0x04>;
nvidia,bpmp = <0x03 0x07>;
nvidia,aspm-cmrt-us = <0x3c>;
nvidia,aspm-pwr-on-t-us = <0x14>;
nvidia,aspm-l0s-entrance-latency-us = <0x03>;
bus-range = <0x00 0xff>;
ranges = <0x43000000 0x30 0x00 0x30 0x00 0x02 0x28000000 0x2000000 0x00 0x40000000 0x32 0x28000000 0x00 0x8000000 0x1000000 0x00 0x3e100000 0x00 0x3e100000 0x00 0x100000>;
interconnects = <0x57 0x2a 0x58 0x57 0x30 0x58>;
interconnect-names = “dma-mem\0write”;
iommu-map = <0x00 0xf0 0x08 0x1000>;
iommu-map-mask = <0x00>;
dma-coherent;
status = “disabled”;
vddio-pex-ctl-supply = <0xf5>;
phys = <0x118 0x119>;
phy-names = “p2u-0\0p2u-1”;
iommus = <0xf0 0x08>;
};
pcie-ep@141e0000 {
compatible = “nvidia,tegra234-pcie-ep”;
power-domains = <0x03 0x10>;
reg = <0x00 0x141e0000 0x00 0x20000 0x00 0x3e040000 0x00 0x40000 0x00 0x3e080000 0x00 0x40000 0x2e 0x40000000 0x04 0x00>;
reg-names = “appl\0atu_dma\0dbi\0addr_space”;
num-lanes = <0x08>;
clocks = <0x03 0xab>;
clock-names = “core”;
resets = <0x03 0x0f 0x03 0x0e>;
reset-names = “apb\0core”;
interrupts = <0x00 0x162 0x04>;
interrupt-names = “intr”;
nvidia,bpmp = <0x03 0x07>;
nvidia,enable-ext-refclk;
nvidia,aspm-cmrt-us = <0x3c>;
nvidia,aspm-pwr-on-t-us = <0x14>;
nvidia,aspm-l0s-entrance-latency-us = <0x03>;
interconnects = <0x57 0x2a 0x58 0x57 0x30 0x58>;
interconnect-names = “dma-mem\0write”;
iommu-map = <0x00 0xf0 0x08 0x1000>;
iommu-map-mask = <0x00>;
dma-coherent;
status = “okay”;
phys = <0x118 0x119>;
phy-names = “p2u-0\0p2u-1”;
iommus = <0xf0 0x08>;
pinctrl-names = “default”;
pinctrl-0 = <0x11a>;
nvidia,host1x = <0x110>;
num-ib-windows = <0x02>;
num-ob-windows = <0x08>;
};
Replace this file with host-PC > L4T/kernel/dtb/tegra234-p3768-0000+p3767-0000-nv-super.dtb, and flash again by running sudo ./tools/kernel_flash/l4t_initrd_flash.sh --external-device nvme0n1p1 -c tools/kernel_flash/flash_l4t_t234_nvme.xml -p "-c bootloader/generic/cfg/flash_t234_qspi.xml" --showlogs --network usb0 jetson-orin-nano-devkit-super internal
It worked, the /boot/dtb/kernel_tegra234-p3768-0000+p3767-0000-nv-super.dtband the /proc/device-tree have been modified. To compare, the 141e0000 is pcie-mode while the 14160000 (the NVME SSD) is pcie.
But the controllers folder is also empty, and the output of sudo dmesg | grep -i "141e0000\|pcie.*ep\|tegra.*pcie" is:
[ 6.505319] tegra194-pcie 141e0000.pcie-ep: Adding to iommu group 3
[ 6.506001] tegra194-pcie 141e0000.pcie-ep: Failed to find PHY entries: -22
[ 6.506004] tegra194-pcie 141e0000.pcie-ep: Failed to parse device tree: -22
[ 6.506023] tegra194-pcie: probe of 141e0000.pcie-ep failed with error -22
- So miss mapping to the physical addr, right? I try to add PHY in the dtb file:
pcie-ep@141e0000 {
compatible = “nvidia,tegra234-pcie-ep”;
power-domains = <0x03 0x10>;
reg = <0x00 0x141e0000 0x00 0x20000 0x00 0x3e040000 0x00 0x40000 0x00 0x3e080000 0x00 0x40000 0x2e 0x40000000 0x04 0x00>;
reg-names = “appl\0atu_dma\0dbi\0addr_space”;
num-lanes = <0x08>;
clocks = <0x03 0xab>;
clock-names = “core”;
resets = <0x03 0x0f 0x03 0x0e>;
reset-names = “apb\0core”;
interrupts = <0x00 0x162 0x04>;
interrupt-names = “intr”;
nvidia,bpmp = <0x03 0x07>;
nvidia,enable-ext-refclk;
nvidia,aspm-cmrt-us = <0x3c>;
nvidia,aspm-pwr-on-t-us = <0x14>;
nvidia,aspm-l0s-entrance-latency-us = <0x03>;
interconnects = <0x57 0x2a 0x58 0x57 0x30 0x58>;
interconnect-names = “dma-mem\0write”;
iommu-map = <0x00 0xf0 0x08 0x1000>;
iommu-map-mask = <0x00>;
dma-coherent;
status = “okay”;
phys = <0x118 0x119>;
phy-names = “p2u-0\0p2u-1”;
iommus = <0xf0 0x08>;
pinctrl-names = “default”;
pinctrl-0 = <0x11a>;
nvidia,host1x = <0x110>;
num-ib-windows = <0x02>;
num-ob-windows = <0x08>;
};
and flash again. I find this file has been modified, but the /proc/device-tree no change, and the msg is also:
[ 6.505319] tegra194-pcie 141e0000.pcie-ep: Adding to iommu group 3
[ 6.506001] tegra194-pcie 141e0000.pcie-ep: Failed to find PHY entries: -22
[ 6.506004] tegra194-pcie 141e0000.pcie-ep: Failed to parse device tree: -22
[ 6.506023] tegra194-pcie: probe of 141e0000.pcie-ep failed with error -22
Can you have a look about my question:
- Can I set EP mode only for one PCIe interface?
- Was my action effective?
- Why the modify about PHY is not work?
Thank you so much~