When start DMA ,system restart, jetson nx

Hello, when I use DMA on jetson nx, the system is crashed, Did someone know why?
新建文本文档.txt (16.9 KB)

It’s better provide detail information how do you use DMA and reproduce step.

  1. First allocated memory in kernel space
    u32 status = 0;
    //uint8_t i,k;
    u64 *phy_addr;
    unsigned long phyvirt_addr;
    if(pdx->m_bBufferAllocate == TRUE)
    {
    DmaMemFreePool(pdx);
    }

     pdx->m_pbyAudioBuffer =  kmalloc(MAX_AUDIO_CAP_SIZE, GFP_KERNEL);	
     if(!pdx->m_pbyAudioBuffer)
     {
     	pdx->m_bBufferAllocate = TRUE;
     	DmaMemFreePool(pdx);
     	pdx->m_bBufferAllocate = FALSE;			
     	status = -1;
    
     	return status;
     }		
     pdx->m_pbyAudioBuffer_area = (char *)(((unsigned long)pdx->m_pbyAudioBuffer + PAGE_SIZE -1) & PAGE_MASK);
     for (phyvirt_addr=(unsigned long)pdx->m_pbyAudioBuffer_area; phyvirt_addr < (unsigned long)pdx->m_pbyAudioBuffer_area + MAX_AUDIO_CAP_SIZE;
     	phyvirt_addr+=PAGE_SIZE) 
     {
     	// reserve all pages to make them remapable
     	SetPageReserved(virt_to_page(phyvirt_addr));
     } 
     	
     memset(pdx->m_pbyAudioBuffer,0x0,MAX_AUDIO_CAP_SIZE );
    
     //vmalloc_to_pfn()
     phy_addr= (u64*)virt_to_phys(pdx->m_pbyAudioBuffer);
     pdx->pci_dma_handle = phy_addr;
    
     printk("[MV]kmalloc_area at 0x%p (phys 0x%p)\n", pdx->m_pbyAudioBuffer,phy_addr);
     
     pdx->m_dwAudioBuffer = ((u64)phy_addr)&0xFFFFFFFF;
     pdx->m_dwAudioBufferHigh = ((u64)phy_addr>>32)&0xFFFFFFFF;;
    
     printk("[MV]Mem Video::m_dwAudioBuffer = %x\n", pdx->m_dwAudioBuffer);
     printk("[MV]Mem Video::m_dwAudioBufferHigh = %x\n",pdx->m_dwAudioBufferHigh);
     //KdPrint(("Mem allocate::m_dwAudioBuffer[%d] = %x\n", i, pdx->m_dwAudioBuffer));
     //--------------
     pdx->m_pAudioData =   kmalloc(MAX_AUDIO_CAP_SIZE, GFP_KERNEL);	
    
     if(!pdx->m_pAudioData)
     {
     	pdx->m_bBufferAllocate = TRUE;
     	DmaMemFreePool(pdx);
     	pdx->m_bBufferAllocate = FALSE;
         status = -1;
     	return status;
     }
     
     pdx->m_pAudioData_area = (char *)(((unsigned long)pdx->m_pAudioData + PAGE_SIZE -1) & PAGE_MASK);
     for (phyvirt_addr=(unsigned long)pdx->m_pAudioData_area; phyvirt_addr < (unsigned long)pdx->m_pAudioData_area + MAX_AUDIO_CAP_SIZE;
     	phyvirt_addr+=PAGE_SIZE) 
     {
     	// reserve all pages to make them remapable
     	SetPageReserved(virt_to_page(phyvirt_addr));
     } 
     
    
     //------------------------------------------------------------
     //KdPrint(("Mem allocate::m_pAudioData = %x\n",  pdx->m_pAudioData));
     pdx->m_bBufferAllocate = TRUE;
     //printk("DmaMemAllocPool  ed\n");
     return 0;
    

2.set dma address
u32 Addrmsk;
u32 AddrLowmsk;
u32 AddrPageSize;
u32 Addr2PageSize;
u32 PhyAddr_A_Low;
u32 PhyAddr_A_High;
u32 PCI_Addr;
//u32 AVALON_Addr;
u32 cnt;
u64 m_tmp64cnt = 0;
//u32 RDAvalon = 0;
//u32 m_AddreeSpace = 0;
int i = 0;
u32 m_ReadTmp;
u32 m_ReadTmp2;
int m_ReadTmp3;
PhyAddr_A_High = 0;
//------------------------------------------ // re write dma register
cnt = 0x208; // Table address

Addrmsk = PCI_E_BAR_ADD_MASK;
AddrLowmsk = PCI_E_BAR_ADD_LOWMASK;
//------------------------------------------ // re write dma register 


	


	PhyAddr_A_Low = pdx->m_dwAudioBuffer;
	PhyAddr_A_High = pdx->m_dwAudioBufferHigh;
	
	PCI_Addr = (PhyAddr_A_Low&AddrLowmsk);
	PhyAddr_A_Low = (PhyAddr_A_Low&Addrmsk);

	m_ReadTmp3 = (int)(PCI_E_BAR_PAGE_SIZE-(pdx->m_MaxHWVideoBufferSize)*2) -PCI_Addr; 

	WRITE_REGISTER_ULONG((PCI_ADDR_TABLE_BASE + cnt),PhyAddr_A_High);
	WRITE_REGISTER_ULONG((PCI_ADDR_TABLE_BASE + cnt+PCIE_BARADDROFSIZE),PhyAddr_A_Low);  //Entry 0
	
	m_ReadTmp = READ_REGISTER_ULONG(PCI_ADDR_TABLE_BASE + cnt);
	m_ReadTmp2 = READ_REGISTER_ULONG(PCI_ADDR_TABLE_BASE + cnt+PCIE_BARADDROFSIZE);
	printk("[MV]1-DispatchCreate[X64] PhyAddr_A_Low  %X=%X  PhyAddr_A_High %X=%X\n",PhyAddr_A_Low ,m_ReadTmp2, PhyAddr_A_High, m_ReadTmp);
	
	
	WRITE_REGISTER_ULONG(CBVS_IN_BUF_BASE + (8*PCIE_BARADDROFSIZE), (PCIEBAR_AXI_BASE+PCI_Addr)); //Buffer 1 address
    m_ReadTmp = READ_REGISTER_ULONG(CBVS_IN_BUF_BASE + (8*PCIE_BARADDROFSIZE));
    
	printk("[MV]1-DispatchCreate[X64]AUDIO BUF  %X=%X \n",(PCIEBAR_AXI_BASE+PCI_Addr), m_ReadTmp);

	WRITE_REGISTER_ULONG(INT_EN_REG_BASE, 0x3ffff); //enable PCI Interruput
	WRITE_REGISTER_ULONG(PCIEBR_EN_REG_BASE, 0xFFFFFFFF);

	m_ReadTmp2 = READ_REGISTER_ULONG(0x144);

3.start the dma, then error is happened,

	case  DVRS_HW_IOCSTART_DMA:
	{
			printk(" IOCTL_DVRS_MAP_START_DMA index[%d]\n",m_MemParam.dwIndex);
			memset(&m_MemParam, 0, sizeof(struct TMemParam));
			copy_from_user(&m_MemParam, (struct TMemParam *)arg, sizeof(struct TMemParam)); 
			StartSys();
		break;
	}

error log:
15:36:04.673]start sys [ 57.524201] mc-err: vpr base=0:c6000000, size=20, ctrl=3, override:(a01a8340, fcee10c1, 1, 0)
[15:36:04.683][ 57.524377] mc-err: (255) csw_pcie4w: MC request violates VPR requirements
[15:36:04.709][ 57.524508] mc-err: status = 0x0ff740e1; addr = 0xffffffff00; hi_adr_reg=008
[15:36:04.712][ 57.524636] mc-err: secure: yes, access-type: write
[15:36:04.725][ 57.524741] mc-err: mcerr: unknown intr source intstatus = 0x00000000, intstatus_1 = 0x00000000
[15:36:04.726][ 57.524760] t19x-arm-smmu 12000000.iommu: Unhandled context fault: smmu1, iova=0x211400000, fsynr=0x20011, cb=0, sid=90(0x5a - PCIE4), pgd=0, pud=0, pmd=0, pte=0
[15:36:04.735][ 57.524881] t19x-arm-smmu 12000000.iommu: Unhandled context fault: smmu1, iova=0x211402000, fsynr=0x20011, cb=0, sid=90(0x5a - PCIE4), pgd=0, pud=0, pmd=0, pte=0
[15:36:04.754][ 57.525000] t19x-arm-smmu 12000000.iommu: Unhandled context fault: smmu1, iova=0x211402800, fsynr=0x20011, cb=0, sid=90(0x5a - PCIE4), pgd=0, pud=0, pmd=0, pte=0
[15:36:04.755][ 57.526815] mc-err: vpr base=0:c6000000, size=20, ctrl=3, override:(a01a8340, fcee10c1, 1, 0)
[15:36:04.767][ 57.532857] mc-err: (255) csw_pcie4w: MC request violates VPR requirements
[15:36:04.782][ 57.539932] mc-err: status = 0x0ff740e1; addr = 0xffffffff00; hi_adr_reg=008
[15:36:04.785][ 57.547037] mc-err: secure: yes, access-type: write
[15:36:04.804][ 57.552381] t19x-arm-smmu 12000000.iommu: Unhandled context fault: smmu1, iova=0x211402c00, fsynr=0x20011, cb=0, sid=90(0x5a - PCIE4), pgd=0, pud=0, pmd=0, pte=0
[15:36:04.805][ 57.552398] mc-err: vpr base=0:c6000000, size=20, ctrl=3, override:(a01a8340, fcee10c1, 1, 0)
[15:36:04.816][ 57.552405] mc-err: (255) csw_pcie4w: MC request violates VPR requirements
[15:36:04.830][ 57.552417] mc-err: status = 0x0ff740e1; addr = 0xffffffff00; hi_adr_reg=008
[15:36:04.831][ 57.552420] mc-err: secure: yes, access-type: write
[15:36:04.839][ 57.594729] t19x-arm-smmu 12000000.iommu: Unhandled context fault: smmu1, iova=0x21140b800, fsynr=0x20011, cb=0, sid=90(0x5a - PCIE4), pgd=0, pud=0, pmd=0, pte=0
[15:36:04.856][ 57.609161] t19x-arm-smmu 12000000.iommu: Unhandled context fault: smmu1, iova=0x211437000, fsynr=0x20011, cb=0, sid=90(0x5a - PCIE4), pgd=0, pud=0, pmd=0, pte=0
[15:36:04.858][ 57.623923] mc-err: Too many MC errors; throttling prints
[15:36:04.868][ 57.629320] t19x-arm-smmu 12000000.iommu: Unhandled context fault: smmu1, iova=0x211463000, fsynr=0x20011, cb=0, sid=90(0x5a - PCIE4), pgd=0, pud=0, pmd=0, pte=0
[15:36:04.883][ 57.643812] t19x-arm-smmu 12000000.iommu: Unhandled context fault: smmu1, iova=0x21143ac00, fsynr=0x20011, cb=0, sid=90(0x5a - PCIE4), pgd=0, pud=0, pmd=0, pte=0
[15:36:04.896][ 57.658247] t19x-arm-smmu 12000000.iommu: Unhandled context fault: smmu1, iova=0x211402400, fsynr=0x20011, cb=0, sid=90(0x5a - PCIE4), pgd=0, pud=0, pmd=0, pte=0
[15:36:04.913][ 57.672617] t19x-arm-smmu 12000000.iommu: Unhandled context fault: smmu1, iova=0x21142d000, fsynr=0x20011, cb=0, sid=90(0x5a - PCIE4), pgd=0, pud=0, pmd=0, pte=0
[15:36:09.672][ 62.536318] __arm_smmu_context_fault: 40527 callbacks suppressed
[15:36:09.683][ 62.536340] t19x-arm-smmu 12000000.iommu: Unhandled context fault: smmu1, iova=0x211424800, fsynr=0x20011, cb=0, sid=90(0x5a - PCIE4), pgd=0, pud=0, pmd=0, pte=0
[15:36:09.705][ 62.536843] t19x-arm-smmu 12000000.iommu: Unhandled context fault: smmu1, iova=0x211426400, fsynr=0x20011, cb=0, sid=90(0x5a - PCIE4), pgd=0, pud=0, pmd=0, pte=0
[15:36:09.718][ 62.537280] t19x-arm-smmu 12000000.iommu: Unhandled context fault: smmu1, iova=0x211427400, fsynr=0x20011, cb=0, sid=90(0x5a - PCIE4), pgd=0, pud=0, pmd=0, pte=0
[15:36:09.730][ 62.537658] t19x-arm-smmu 12000000.iommu: Unhandled context fault: smmu1, iova=0x211428800, fsynr=0x20011, cb=0, sid=90(0x5a - PCIE4), pgd=0, pud=0, pmd=0, pte=0
[15:36:09.750][ 62.538026] t19x-arm-smmu 12000000.iommu: Unhandled context fault: smmu1, iova=0x211429c00, fsynr=0x20011, cb=0, sid=90(0x5a - PCIE4), pgd=0, pud=0, pmd=0, pte=0
[15:36:09.762][ 62.538446] t19x-arm-smmu 12000000.iommu: Unhandled context fault: smmu1, iova=0x21142af40, fsynr=0x20011, cb=0, sid=90(0x5a - PCIE4), pgd=0, pud=0, pmd=0, pte=0
[15:36:09.773][ 62.539519] t19x-arm-smmu 12000000.iommu: Unhandled context fault: smmu1, iova=0x21142e400, fsynr=0x20011, cb=0, sid=90(0x5a - PCIE4), pgd=0, pud=0, pmd=0, pte=0
[15:36:09.784][ 62.552672] t19x-arm-smmu 12000000.iommu: Unhandled context fault: smmu1, iova=0x211455c00, fsynr=0x20011, cb=0, sid=90(0x5a - PCIE4), pgd=0, pud=0, pmd=0, pte=0
[15:36:09.814][ 62.567513] t19x-arm-smmu 12000000.iommu: Unhandled context fault: smmu1, iova=0x21141e000, fsynr=0x20011, cb=0, sid=90(0x5a - PCIE4), pgd=0, pud=0, pmd=0, pte=0
[15:36:09.823][ 62.581984] t19x-arm-smmu 12000000.iommu: Unhandled context fault: smmu1, iova=0x211449800, fsynr=0x20011, cb=0, sid=90(0x5a - PCIE4), pgd=0, pud=0, pmd=0, pte=0
[15:36:11.863][ 64.726644] nvgpu: 17000000.gv11b gk20a_fifo_handle_pbdma_intr_0:2722 [ERR] semaphore acquire timeout!
[15:36:11.878][ 64.726891] nvgpu: 17000000.gv11b nvgpu_set_error_notifier_locked:137 [ERR] error notifier set to 24 for ch 509
[15:36:16.121][ 67.545492] __arm_smmu_context_fault: 41429 callbacks suppressed
[15:36:16.122][ 67.545514] t19x-arm-smmu 12000000.iommu: Unhandled context fault: smmu1, iova=0x211440400, fsynr=0x20011, cb=0, sid=90(0x5a - PCIE4), pgd=0, pud=0, pmd=0, pte=0
[15:36:16.122][ 67.546020] t19x-arm-smmu 12000000.iommu: Unhandled context fault: smmu1, iova=0x211442000, fsynr=0x20011, cb=0, sid=90(0x5a - PCIE4), pgd=0, pud=0, pmd=0, pte=0
[15:36:16.123][ 67.546466] t19x-arm-smmu 12000000.iommu: Unhandled context fault: smmu1, iova=0x211443000, fsynr=0x20011, cb=0, sid=90(0x5a - PCIE4), pgd=0, pud=0, pmd=0, pte=0
[15:36:16.124][ 67.546837] t19x-arm-smmu 12000000.iommu: Unhandled context fault: smmu1, iova=0x211444400, fsynr=0x20011, cb=0, sid=90(0x5a - PCIE4), pgd=0, pud=0, pmd=0, pte=0
[15:36:16.126][ 67.547205] t19x-arm-smmu 12000000.iommu: Unhandled context fault: smmu1, iova=0x211445800, fsynr=0x20011, cb=0, sid=90(0x5a - PCIE4), pgd=0, pud=0, pmd=0, pte=0
[15:36:16.127][ 67.547632] t19x-arm-smmu 12000000.iommu: Unhandled context fault: smmu1, iova=0x211446c00, fsynr=0x20011, cb=0, sid=90(0x5a - PCIE4), pgd=0, pud=0, pmd=0, pte=0

NX’s PCIe controllers have SMMU/IOMMU enabled. So, using APIs like virt_to_phys() doesn’t really work.
Please use the DMA APIs for any allocations/mapping memory before letting your endpoint access that memory.
Refer:
https://www.kernel.org/doc/html/latest/PCI/pci.html
https://www.kernel.org/doc/html/latest/core-api/dma-api-howto.html