[ 736.909932] nvgpu: 17000000.ga10b nvgpu_channel_recover_from_wdt:107 [ERR] Job on channel 509 timed out [ 736.920885] __ga10b__ NV_PGRAPH_STATUS: 0x12000a1 [ 736.909932] nvgpu: 17000000.ga10b nvgpu_channel_recover_from_wdt:107 [ERR] Job [ 736.92channel 54] 09 tim__ga10b__ NV RAPH_STATUS1: 0[ x1 .920885] __ga10b__ NV_PGRAPH_STATUS: 0x12000a1 [ 736.926064] __ga10b__ NV_PGRAPH_STATUS1: 0x1 [ 736.930554] __ga10b__ NV_PGRAPH_ENGINE_STATUS: 0x1 [ 736.935503] [ __g.930554] NV_PGa10b__ NV_PGRAPRFIH_ENGITUSNE_ : 0x2020: 0001x1 [ 736.935503] __ga10b__ NV_PGRAPH_GRFIFO_STATUS : 0x2020001 [ 736.941151] __ga10b__ NV_PGRAPH_GRFIFO_CONTROL : 0x0 0x036.946275] __ga10[ NV736_PGRAP115H_PRI_b__ NVS_H_PGOSTH_GT_STATUS : 0x0 [ 736.946275] __ga10b__ NV_PGRAPH_PRI_FECS_HOST_INT_STATUS : 0x0 [ 736.952334] __ga10b__ NV_PGRAPH_EXCEPTION : 0x0 [ 736.957112] __g[ 736.95b__4] NV_PGRAP NVH_F_PGECS_INTR : H_E0x0XCEPTION : 0x0 [ 736.957112] __ga10b__ NV_PGRAPH_FECS_INTR : 0x0 [ 736.961908] __ga10b__ NV_PFIFO_ENGINE_STATUS(GR) : 0x8002e000 [ 736.96781[ 7363] .96__ga10b__a10 NV NV_PFIFO_ENVITE_S 0x0 US(GR) : 0x8002e000 [ 736.967813] __ga10b__ NV_PGRAPH_ACTIVITY0: 0x0 [ 736.972404] __ga10b__ NV_PGRAPH_ACTIVITY1: 0xa00 [ 736.97713[ 9] 736__g240__gb__b__ NV NV_PGRAPH_ACTIVITVITY1:Y4:a003 [ 736.977139] __ga10b__ NV_PGRAPH_ACTIVITY4: 0x3 [ 736.981753] __ga10b__ NV_PGRAPH_PRI_SKED_ACTIVITY: 0x0 [ 736.987030] __ga10b__ NV_PGRAPH_PRI_GPC0_GPCCS_GPC_ACTIVITY0: 0x0ES_GPC_ACTVITIVITY0: 0x0 [ 736.993342] __ga10b__ NV_PGRAPH_PRI_GPC0_GPCCS_GPC_ACTIVITY1: 0x0 ACTIVITY1: 0x0 __g[ a103342] __ga10RAPRI_ NV_PGRAPPCCH_PACTRI_IVITY2: 0x0 [ 736.999596] __ga10b__ NV_PGRAPH_PRI_GPC0_GPCCS_GPC_ACTIVITY2: 0x0 [ 737.005886] __ga10b__ NV_PGRAP[ 737.00GPC0_G6] S_Ga10PC_ACTIVITY3RAPH_PRI_GPC0_GPCCS_GPC_ACTIVITY3: 0x0 [ 737.012160] __ga10b__ NV_PGRAPH_PRI_GPC0_GPCCS_GPC_ACTIVITY4: 0x3000 [ [ 737.018692160] __ga10b___PG NV_PGRAPRI_GPC0_TGPC0_GPCCCCSS_GCTIACTIVITY4 0x: 0x3000 [ 737.018690] __ga10b__ NV_PGRAPH_PRI_GPC0_TPC0_TPCCS_TPC_ACTIVITY0: 0x0 [ 737.025426] __ga10b__ NV_PGRAPH_PRI_GPC0_TPC1_TPCCS_TPC_ACTIVITY0: 0x0 [ [ 7375426] 8] __ga10b__a10b___PG_PGRAPH_PH_PRI_GPC0_TPC1_TPPC2CCS_TPC_A_TPC_ACTIY0:Y0: 0x0 0 [ 737.032128] __ga10b__ NV_PGRAPH_PRI_GPC0_TPC2_TPCCS_TPC_ACTIVITY0: 0x0 [ 737.038890] __ga10b__ NV_PGRAPH_PRI_GPC0_TPC3_TPCCS_TPC_ACTIVITY0: 0x0 0 [ .038890] __ga10b__ NVa10_PGRAPH_PRI_GPC0_TPC3_TPCCSPCC_TPPC_CTIVITTY0: 0x0 [ 737.048586] __ga10b__ NV_PGRAPH_PRI_GPCS_GPCCS_GPC_ACTIVITY0: 0[ x05322] 737a10532b__ NV_PGa10RAP_PGH_PH_PGPCGPCPCCPCCS_GACTPC_TY1IVI: 0: 0 [ 737.068629] __ga10b__ NV_PGRAPH_PRI_GPCS_GPCCS_GPC_ACTIVITY2: 0x0 [ 737.07[ 489.068621] __ga10b__b__ NV NV_PGRAPRAPH_PRI_GPCPCCPCCS_GPC_ACTIVITY2: 0x0 737.074891] __ga10b__ NV_PGRAPH_PRI_GPCS_GPCCS_GPC_ACTIVITY3: 0x0 [ 737.083062] __ga10b__ NV_PGRAPH_PRI_GPCS_GPCCS_GPC_ACTIVITY4: 0x3000 [ 737.09[ 5746] __g306a10__ga10b___PG NV_PGH_PH_PGPCS_TPCSS_GPCCS_G_TPACTC_AIVITY4: 0x3000 [ 737.095746] __ga10b__ NV_PGRAPH_PRI_GPCS_TPCS_TPCCS_TPC_ACTIVITY0: 0x737.10362[ 0] 362__ga10__gb__ NV NV_PGRAPH_PRI_DS_DS_MPIMPIPE_STATUS: 0x0 [ 737.115878] __ga10b__ NV_PGRAPH_PRI_FE_GO_IDLE_TIMEOUT : 0x800 [ 737.122353] __g5878] b____ga10RAPb__ NV_PGGO_H_PRI_NFO : IDLE_TIMEOUT : 0x800 [ 737.122353] __ga10b__ NV_PGRAPH_PRI_FE_GO_IDLE_INFO : [ .128447] __g a10b__[ NV_PG7] H_PRI_b__ NV0_TPC0_TEH_PRI_GPCUBUPC0NITS_STATX_M_TEX_SUBUNITS_STATUS: 0x0 [ 737.143344] __ga10b__ NV_PGRAPH_PRI_CWD_FS: 0x401 [ 737.14[ 8183] __ga10b__a10 NV_PGRAP_PGRI_FE_RI_CWD_FS(0)x401 [ 737.148183] __ga10b__ NV_PGRAPH_PRI_FE_TPC_FS(0): 0xf [ 737.153856] __ga10b__ NV_PGRAPH_PRI_CWD_GPC_TPC_ID: 0x10203 [ 737[ 737.153856] __ga10 NV_PGRAP_PGRAPH_PCWD_SM_GP(0): 0ID:203 0x [ 737.165227] __ga10b__ NV_PGRAPH_PRI_CWD_SM_ID(0): 0x10203737.17[ 126.178] 1268] __g_PGb__ NVRAPRI_RAPS_CRI_TXSW_STXSUS_W_S0: US_0x22000x21 1 [ 737.184826] __ga10b__ NV_PGRAPH_PRI_FECS_CTXSW_STATUS_1: 0x190 [ [ 737.19082__g3] a10b__ NV_PGRAPH_PRI_GPC0_GTXSW_STATUS_1: 0x190TATUS_737_0:.19 0x0823] __ga10b__ NV_PGRAPH_PRI_GPC0_GPCCS_CTXSW_STATUS_GPC_0: 0x0 [ 737.198996] __ga10b__ NV_PGRAPH_PRI_GPC0_GPCCS_CTXSW_STATUS_1: 0x390 [ [ .198996] __ga10b__ NVa10b__ NV_PGRI_0_GH_PRI_FECTXSS_CTATTXSW_IDLE90STA 0] __ga10b__ NV_PGRAPH_PRI_FECS_CTXSW_IDLESTATE : 0xe [ 737.218863] __ga10b__ NV_PGRAPH_PRI_GPC0_GPCCS_CTXSW_IDLESTATE : 0xf [ [ .23.21886__ga10__ga10_PG NVRAPH_PRI_FECRI_0_GURRENT_CTX : 0xDLEb012e62c [ 737.230236] __ga10b__ NV_PGRAPH_PRI_FECS_CURRENT_CTX : 0xb012e62c [ 737.236884] __ga10b__ NV_PGRAPH_PRI_FECS_NEW_CTX : 0x3012e62c [ 737.248529] __ga10b__ NVa10b__ NV_PGFECS_HH_POST_INS_NEW_LE : 0x7f0003 e62c [ 737.248529] __ga10b__ NV_PGRAPH_PRI_FECS_HOST_INT_ENABLE : 0x7f000[ .253569__ga10 737 NV_PG3] RAPRI___gS_HOST_INT_STAT NV_PG: 0RAPRI_FECS_HOST_INT_STATUS : 0x0 [ 737.268387] __ga10b__ NV_PGRAPH_PRI_GPCS_ROP0_CROP_STATUS1 : 0x5f00000 [ 737.27[ 7393] __g8387] __ga10_PGRAPH_PRI_GPCS_ROPS_CROP_RI_STAGPCS_ROP0 0x5f0STA0 TUS 0x5f00000 [ 737.277393] __ga10b__ NV_PGRAPH_PRI_GPCS_ROPS_CROP_STATUS1 :[ 0x5185] __g0b__ _PGRAPH_P518GPCS_ROP0_ZROP_b__STATUS : NV0x0 RAPH_PRI_GPCS_ROP0_ZROP_STATUS : 0x0 [ 737.297444] __ga10b__ NV_PGRAPH_PRI_GPCS_ROP0_ZROP_STATUS2 : 0x0 [ 737737.296807444] __ga10b__b__ NV_PGRAPRAPRI_H_PRI_OP1_ZROP_OP0STATUSTUSx0 2 : 0x0 [ 737.306803] __ga10b__ NV_PGRAPH_PRI_GPCS_ROP1_ZROP_STATUS: 0.31362 __ga10[ .31362_PGRAPa10b__ NVGPCS_RH_P_ZROP_STAOP1_ZROP_0x0 TUS2: 0x0 [ 737.326400] __ga10b__ NV_PGRAPH_PRI_GPCS_ROPS_ZROP_STATUS : 0x0 [ 737.33[ 737__g6400] b__ NV_PGRAP_PGH_PH_PGPCS_RS_R_ZROP_STAOP_STA2 : 0x : 0x0 [ 737.333310] __ga10b__ NV_PGRAPH_PRI_GPCS_ROPS_ZROP_STATUS2 :[ 737.34013[ .34013a10__g NVb__ NVRAPRI_RAPH_PRI_PCCS_GPC_0_GPCCS_GPC_EXCEPTEPTION: 0x0 [ 737.354486] __ga10b__ NV_PGRAPH_PRI_GPC0_GPCCS_GPC_EXCEPTION_EN: 0xfc0f6004 [ [ 7374482268] __ga10b___PG_PGRAPH_PRI_GPC0_TPC0_TP_GP_TPC_EXCEPTION:EN: 0xfc0f6004 [ 737.362268] __ga10b__ NV_PGRAPH_PRI_GPC0_TPC0_TPCCS_TPC_EXCEPTION:[ 737.36 897.368977] b____g_PGRAP NVH_PRAPH_PRI_GPC0_TPC0_TP_TPCCS_TP_TPXCEXCEPTIPTION_ON_ 0x1616 [ 737.385106] __ga10b__ NV_PGRAPH_PRI_GPCS_TPCS_SMS_HWW_GLOBAL_ESR_REPORT_MASK: 0x0 [ 737.39[ .385] __ga10b__ NV_PG_PGH_PRI_GPCS_TS_TPCS_SM_SMS_HS_HWW_GLOBALSR__ESR_REPO 0xMASK: 0x0 [ 737.393325] __ga10b__ NV_PGRAPH_PRI_GPCS_TPCS_SMS_HWW_WARP_ESR_REPORT_MA[ SK: 0x0111__g 737b__.40 NV_PG__ga10RAPb___PGRI_GPCH_PRI__SMS_TS_HWW_S_HGLOWW_GLOR: 0x0 R: 0x0 [ 737.416002] __ga10b__ NV_PGRAPH_PRI_GPCS_TPCS_SMS_DBGR_CONTROL0: 0x1c00 [ [ .416003524] __g__ga10b___PG NV_PGRAPGPCH_PRI_GPCS_T_CONTRS_DBGR_ST00ATU 0x [ 737.423524] __ga10b__ NV_PGRAPH_PRI_GPCS_TPCS_SMS_DBGR_STATUS0: 0x0 [ 737.430431] __ga10b__ NV_PGRAPH_PRI_GPCS_TPCS_SMS_DBGR_BPT_PAUSE_MASK_0: 0x0 [ [ 7374782] 1] __gb__ NVb__ NVH_PRI_GPCS_TPCS_SMGPCS_DBGR_BPT_PAUSE_MASK_1: 0xASK 0x0 [ 737.444782] __ga10b__ NV_PGRAPH_PRI_GPCS_TPCS_SMS_DBGR_BPT_PAUSE_MASK_1: 0x0[ .452391] __g737a102391] _PGRAPH_Pa10RI_b__GPC0_TPC0RAP_SM0_HGPCWW_WARPC0_SM 0xWW_WARP_ESR: 0x0 [ 737.466229] __ga10b__ NV_PGRAPH_PRI_GPC0_TPC0_SM0_HWW_WARP_ESR_REPORT_MASK: 0x0 [ 737.47[ 601.469] __ga10b__ NV_PGRAPRAPH_PRI_0_TPC00_TPC0WW__SMP_E0_HSR_REPORT_MA0x0 0 [ 737.476013] __ga10b__ NV_PGRAPH_PRI_GPC0_TPC0_SM0_HWW_GLOBAL_ESR: 0x0[ .482999] __ga109] NV__g_PGa10b__ NV_PGRAP0_TH_PPC0RI__SM0_HWW_0_HGLOWW_GLOEPORT__ESEPORT_MASK: 0x0 [ 737.499472] __ga10b__ NV_PGRAPH_PRI_GPC0_TPC0_SM0_DBGR_CONTROL0: 0x1c00 [ 737.50[ .49626947a10b__ NV_PGRAPH_PRI__PGRAPH_PRI_GPC0_D0_T_ST_SMATU0_DS0: 0xOL00 : 0x1c00 [ 737.506261] __ga10b__ NV_PGRAPH_PRI_GPC0_TPC0_SM0_DBGR_STATU[ 7370273 1] .51273b__1] NVRAP__gH_P NVGPC0_TPC0_SMRI_0_TWW__SMP_ESR: 0xWAR0 P_ESR: 0x0 [ 737.526078] __ga10b__ NV_PGRAPH_PRI_GPC0_TPC0_SM1_HWW_WARP_ESR_REPORT_MASK: 0x0 [ [ 7375868] 8] __ga10b__ NV_PGRAPH_PRI_GPC0_TPC0PC01_H1_HWW_WARP_ESR__ESREPORT_MASK: 0x0 [ 737.535868] __ga10b__ NV_PGRAPH_PRI_GPC0_TPC0_SM1_HWW_GLOBAL_ESR: [ 737.54249 737.54a108] NV__g_PGH_Pb__RI_GPCRAPH_P_SMGPC0_TGLOBAL1_HWW_R_RGLO_ESRT_R_RMAS0x0 RT_MASK: 0x0 [ 737.559145] __ga10b__ NV_PGRAPH_PRI_GPC0_TPC0_SM1_DBGR_CONTROL0: 0x1c00 [ 737.56[ 5936] __ga10a10 NV_PGRAPH_PRI_GPC0_TGPCPC00_T_SM1_DBGR_STATUNTRS0: 0x0 00 [ 737.565936] __ga10b__ NV_PGRAPH_PRI_GPC0_TPC0_SM1_DBGR_STATUS0:[ 0x.572976] [ 737a10.57b__ NV_PG__gb__ NVRI_GPC0_TPC1RI_GPC0_HWW_PC1_SMP_ESR: 0x0 P_ESR: 0x0 [ 737.586624] __ga10b__ NV_PGRAPH_PRI_GPC0_TPC1_SM0_HWW_WARP_ESR_REPORT_MASK: 0x0 [ [ 737615662__ga10b__a10 NVb__ NVH_P_PGRAPPC1RI_GPC0_HPC1_SM0_HGLOBALP_ER: 0x0 _MASK: 0x0 [ 737.596156] __ga10b__ NV_PGRAPH_PRI_GPC0_TPC1_SM0_HWW_GLOBAL_ESR: 2792] __g[ 737b__ NV_PG2] RAPb__H_P NVRI_H_P0_TRI_GPC0_T0_HGLOBAL0_HWW_R_RBAL_ESMASR_REPORT_MASK: 0x0 [ 737.618989] __ga10b__ NV_PGRAPH_PRI_GPC0_TPC1_SM0_DBGR_CONTROL0: 0x1c00 [ 737.62[ 737__g.61a109] b__ NV_PGRAPH_PRI_GPCH_PRI_GPC_SM0_TBGRPC1ATUS0:BGR 0x_CONTROL0: 0x1c00 [ 737.625784] __ga10b__ NV_PGRAPH_PRI_GPC0_TPC1_SM[ 0_D.63_ST5] S0:__ga10b__ 737.63_PG2245] H_PRI_GPCa10b___SM_PG1_HWARH_PSR:RI_ 0xPC1_SM1_HWW_WARP_ESR: 0x0 [ 737.646294] __ga10b__ NV_PGRAPH_PRI_GPC0_TPC1_SM1_HWW_WARP_ESR_REPORT_MASK: 0x0 [ [ 7376876294] a10b____g NV_PG_PGRAPH_PRI_GPC0_TPC1_SM1_HWW_WW_BALP_ESR_REPORT_MASK: 0x0 [ 737.656879] __ga10b__ NV_PGRAPH_PRI_GPC0_TPC1_SM1_HWW_GLOBAL_ESR: [ 737.66350.66__ga10b__a10b__ NVH_PRI_RAPH_PGPCRI_GPC1_HPC1_SMGLOBAL_ESWW_GLOBALEPOR_REPOK: 0x0 MASK: 0x0 [ 737.679891] __ga10b__ NV_PGRAPH_PRI_GPC0_TPC1_SM1_DBGR_CONTROL0: 0x1c00 [ 737.68[ 6686] 989__g__gb__ NVb__ NV_PGRAPRAPH_PRI_GPC0_T_SMPC11_DBGR_STATUS0: 0xOL0: 0x1c00 [ 737.686686] __ga10b__ NV_PGRAPH_PRI_GPC0_TPC1_SM1_DBGR_STATU737 0x.693984] [ 737.69 NV398RAPH_P__gRI_a10PC2b___SM0_HRAPWARRI_GPCP_ESR:_SM0_HWW_WARP_ESR: 0x0 [ 737.707811] __ga10b__ NV_PGRAPH_PRI_GPC0_TPC2_SM0_HWW_WARP_ESR_REPORT_MASK: 0x0 [ [ 737.707604] __ga10b__ NV_PGRAPH_PRI_RI_GPC0_TPC2_SM0_HWW_GLOBALSR_R: REPORT_MASK: 0x0 [ 737.717604] __ga10b__ NV_PGRAPH_PRI_GPC0_TPC2_SM0_HWW_GLOBAL_ES[ R: 7370x05] __g[ b__.72_PG5] RAPa10b__GPC NVRAPH_PRI__SM0_TPC2_SM0_HBALGLO_ESBALR_RR_RRT_MASMASK: 0x0 [ 737.741227] __ga10b__ NV_PGRAPH_PRI_GPC0_TPC2_SM0_DBGR_CONTROL0: 0x1c00 [ 737.74801[ 9] .74122a10__gb___PGRAPb__H_P NV_PGRAPH_P_SMRI__ST0_TPC2 0x0_DBGR_CONTROL0: 0x1c00 [ 737.748019] __ga10b__ NV_PGRAPH_PRI_GPC0_TPC2_SM0_DBGR_STATU.754475] __ga10737 NV_PGRAP5] __ga10b__0_T NV1_HRAPH_PP_ERI_0_TPC2SR: 0x0 WARP_ESR: 0x0 [ 737.767746] __ga10b__ NV_PGRAPH_PRI_GPC0_TPC2_SM1_HWW_WARP_ESR_REPORT_MASK: 0x0 [ [ 737.76.776] 1] __ga10b__ NV_PGRAPH_PRI_GPC0_TPC2_SMPC2WW_GLOBAL_ESR: P_ESR_REPORT_MASK: 0x0 [ 737.777541] __ga10b__ NV_PGRAPH_PRI_GPC0_TPC2_SM1_HWW_GLO[ 737BAL_ES416R: 0x08] a10b__ NV_PGRAP416H_P8] __ga10b__PC2_SM1_HWW_RAPGLORI_BALGPCR_REPO_SM1_HK: 0x0BAL_ESR_REPORT_MASK: 0x0 [ 737.800923] __ga10b__ NV_PGRAPH_PRI_GPC0_TPC2_SM1_DBGR_CONTROL0: 0x1c00 [ 737[ 8965] .80092a103] b__ NVb___PGRAPRAPH_PGPC0_TPC2_SMPC21_D1_DBGRBGR_ST: 0x1c 0x0 737.808965] __ga10b__ NV_PGRAPH_PRI_GPC0_TPC2_SM1_DBGR_STATUS0: 0x0 [ 737.816575] __ga10b__ NV_PGRAPH_PRI_GPC0_TPC3_SM0_HWW_WARP_ESR: 0x0 [ 737.82[ 9088] __ga10a10 NV_PG NV_PGRAPH_PRI_GPC0_TPC3_SM0_HWW_WARWARP_ESR_0REPORT[ _MASK: 0x9088] __ga10b__ NV_PGRAPH_PRI_GPC0_TPC3_SM0_HWW_WARP_ESR_REPORT_MASK: 0x0 [ 737.839140] __ga10b__ NV_PGRAPH_PRI_GPC0_TPC3_SM0_HWW_GLOBAL_ESR: 0x0 [ 737.85[ 1135] __ga10b__ NV_PGRAPH_PRI_H_PRI_0_TPC3_SM0_H_SMGLOBAL_ESGLOR_RBAL_ESR: K: 0x0 737.851135] __ga10b__ NV_PGRAPH_PRI_GPC0_TPC3_SM0_HWW_GLOBAL_ESR_REPORT_MASK: 0x0 [ 737.860663] __ga10b__ NV_PGRAPH_PRI_GPC0_TPC3_SM0_DBGR_CONTROL0: 0x1c00 [ 737.877373960668] 3] a10b__a10b__ NV_PGRAPH_PGPCRI__SM0_TPC3_ST_SMBGRATU_CONTROL0: 0x1c00 [ 737.873968] __ga10b__ NV_PGRAPH_PRI_GPC0_TPC3_SM0_DBGR_STATU.88S0: 0x03] __g[ 737.88042b__3] RAPH_Pa10RI_GPCRAPH_P0_TRI__SMGPC0_TWW_PC3SR: 0xWW_WARP_ESR: 0x0 [ 737.894801] __ga10b__ NV_PGRAPH_PRI_GPC0_TPC3_SM1_HWW_WARP_ESR_REPORT_MASK: 0x0 [ [ 7374804591] __ga10b__ NVRAPRAPH_P_PRRI__TP0_TSM1PC3_HWWW_GLO_ESR_R_ESEPORT_MASK: 0x0 [ 737.904591] __ga10b__ NV_PGRAPH_PRI_GPC0_TPC3_SM1_HWW_GLOBAL_ESR: 0x0[ 737.91[ 737.911234] __ga10__ga10b__ NV NVRAP_PGRAPH_PRI_GPC0_T0_TPC3_SM1_HWW_GLOBALBAL_ESR_REPORT_MASMASK: 0x0 [ 737.929090] __ga10b__ NV_PGRAPH_PRI_GPC0_TPC3_SM1_DBGR_CONTROL0: 0x1c00 [ 737.93[ 7] 737__ga10__g NV_PGRAP NV_PGRAPGPCH_PRI_GPCBGR0_TPC3ATU_SM1_DBGR_CONTROL0: 0x1c00 [ 737.935887] __ga10b__ NV_PGRAPH_PRI_GPC0_TPC3_SM1_DBGR_STATUS0: 0x0[ 737[ 737.94nvg2] pu: 17000000.ga.ga10b10b u_set_errvgpu_set_ier_lotifier_lo49 [ERR] ed:1 [ERR]oti errofieet r nto r sor et t9 [ 737.966919] nvgpu: 17000000.ga10b nvgpu_channel_recover_from_wdt:107 [ERR] Job on channel 511 timed out [ 737.977725] __ga10b__ NV_PGRAPH_STATUS: 0x12000a1 [ 737.966919] nvgpu: 17000000.ga[ 737 .98gpu2] __ga10recb__ NV_PGrom_wdTAT07 US1: 0RR] Job on channel 511 timed out [ 737.977725] __ga10b__ NV_PGRAPH_STATUS: 0x12000a1 [ 737.98737.982647102] 3] __g__ga10b__ NV_PGRAPRAPH_STATSTA: 0: 0x1 [ 737.987103] __ga10b__ NV_PGRAPH_ENGINE_STATUS: 0x1 [ 737.997363] __ga10b__ NV_PGRAPH_GRFIFO_STATUS : 0x2020001 [ 738.00[ 737.99a10736 NV3] _PGRAPH_GRFIRAPCONTROFO_ 0x0 : 0x2020001 [ 738.008202] __ga10b__ NV_PGRAPH_GRFIFO_CONTRO[ 738 0x33108] a10b__7383318] __ga10H_Pb__S_H_PGOSTH_PRI_FECS_HUS _INT_STAT : 0x0 [ 738.026573] __ga10b__ NV_PGRAP[ H_EXCE3] ON __ga10 b___PGRAPH_EXCEPTION : 0x0 [ 738.031336] __ga10b__ NV_PG[ H_F738.03133_INTR __g0x0a10 NV _PGRAPH_FECS_INTR : 0x0 [ 738.039186] __ga10b__ NV[ IFO_EN.039186] E_S__ga10 : 0x8_PFIFO_ENe00GINE_STATUS(GR) : 0x8002e000 [ 738.049240] __ga10b__ NV_PGRAPH_ACTIVITY0: 0x0 [ 738.055284] __ga10b__ NV_PGRAPH_ACTIVITY1:[ .049240] __ga10b__ NV_PGRAPH_ACTIVITY0: 0x0 [ 738.055284] [ a10b__.06027RAP__gCTIVITb__ NV_PGRAPH_ACTIVIT.06 0x0273 1] __ga10b__ NV_PGRAPH_ACTIVITY4: 0x3 [ 738.069728] __ga10b__ NV_PGRAPH_PRI_SKED_ACTIVITY: 0x0 [ 738.079519] __ga10b__ NV_PGRAPH_PRI_GPC0_G[ 738PC_ACTIVI9728] x0 b__ NV_PGRAPH_PRI_SKED_ACTIVITY: 0x0 [ 738.079519] __ga10b__ NV_PGRAPH_PRI_GPC0_GPCC__ga10b__ NVIVITY0: 0x0RI_ 738GPCPCC.08PC_1] __gIVIb__: 0_PG RAPH_PRI_GPC0_GPCCS_GPC_ACTIVITY1: 0x0 [ 738[ 7389641] __g964a10b__ NV NV_PGRAPRAPH_PH_P0_GPCCS_G0_GS_GACTIVITY2TY2x0: 0x0 GPC738.111097] __ga10b__ NV_PGRAPH_PRI_GPC[ .11109S_GPC_ACTa10 NV_PGRAPH_Px0 PCCS_GPC_ACTIVITY3: 0x0 : 0x30001[ 7383] .118273] __ga10H_Pb__RI_RAPH_PRI_GPCPC_0_GPCCIVITY4PC_ACTx3000 [ 738.128245] __ga10b__ NV_PGRAPH_PRI_GPC[ .128245] CCSa10b__CTI NV_PGY0: 0xH_PRI_GPC0_TPC0_TPCCS_TPC_ACTIVITY0: 0x0 [ 738.137386385] 6385] a10__ga10b__ NVRAPH_PRAPH_PRI_PC1_TPPC1_TP_TP_TPC_ACTIVITY0:Y0: 0x 0x0 [ 738.147931] __ga10b__ NV_PGRAPH_PRI_GPC[ 0_T7931] __g_TPb__ NVVITY0: 0x0 GPC0_TPC2_TPCCS_TPC_ACTIVITY0: 0x0 [ 738[ .154640] __ga10b__ NV_PGRAPH_PRI_GPC0_TPC3PC3_TPCCS_TPCTIVITCTIY0:Y0: 0x 0x0 [ 738.164996] __ga10b__ NV_PGRAPH_P[ 738GPCS_GPCCS_G6] __ga10b__ NVTY0_PGRAPH_PRI_GPCS_GPCCS_GPC_ACTIVITY0: 0x0 [ 738.171911] __ga10b__ NV_PGRAPH_PRI_GPCS_GPCCS_GPC_ACTIVITY1: 0x0 b__738.181359] __ga10b__ NV_PGRAPH_PRI_GPCS_GPCCS_GPC_ACTIVI1] TY2: 0x0 RAPH_PRI_GPCS_GPCCS_GPC_ACTIVITY1: 0x0 [ 738.181359] __ga10b__ NV_PG[ 738H_P.18RI___ga10PCCb___PGRAPPC_ACTGPCTY2: 0x0S_G PC_ACT.18IVITY3: 0a10b__ NV_PGRAPH_PRI_GPCS_GPCCS_GPC_ACTIVITY3: 0x0 [ 738.201226] __ga10b__ NV_PGRAPH_PRI_GPCS_GPCCS_GPC_ACTIVITY4: 0x3000 [ 738.212072] __ga10b__ NV_PGRAPH_PRI_GPC[ 738_TP.206] __gCCSa10 NVC_ACTIVITRI_GPCS_GPCCS_GPC_ACTIVITY4: 0x3000 [ 738.212072] __ga10b__ NV_PGRAPH_PRI_GPCS_TPCS[ CCS738985_TPCTIVITY0:a10b__ NV_PG[ H_P738RI_DS_4] MPIPE_b__ NVTUS: 0RAPH_Px0 DS_MPIPE_STATUS: 0x0 [ 738[ 2468] .23__ga10__gb__ NV_PGRAPH_PRAPRI_H_PGO_IDLGO_IDLIMEOUT : IME0x8OUT : 0x800 [ 738.242087] __ga10b__ NV_PGRAPH_PRI_FE_[ 738IDL.242087] __ga10710 _PGRAPH_PRI_FE_GO_IDLE_INFO : 0x1000710 [ 738[ 9877386] __ga10b__ NV NV_PG_PGRAPH_PGPC0_T0_TPC0_TEX_M_TE_TEX_SUBUNITS_STATNITS_STAT0 US:0 [ 738.262815] __ga10b__ NV_PGRAPH_PRI_CWD[ _FS: 0281x401 a10b__ NV_PGRAPH_PRI_CWD_FS: 0x401 [ 738.26[ 7662] 7662] __ga10 NV NV_PGH_PH_PRI_TPC_FSFE_(0): 0: 0 [ 738.275869] __ga10b__ NV_PGRAPH_PRI_CWD_GP.275869] __ga1010203 RAPH_PRI_CWD_GPC_TPC_ID: 0x10203 [ 738[ 2515] __ga102515] __ga10b__ NV_PGRAPCWDRI_CWD_SM_ID_IDx10: 0203 [ 738.291964] __ga10b__ NV_PGRAPH_PRI_FECS_CTXS738W_STAT196__ga100: 0x22001 NV_PGRAPH_PRI_FECS_CTXSW_STATUS_FE_0: 0x22001 [ 738.29[ 738.299747] __ga10a10b__b___PG_PGRAPH_PRI_FECS_CTXSW_SW_SUS_1: TAT901: 0x190 [ 738.311561] __ga10b__ NV_PGRAPH_PRI_GPC[ 0_GPCCS_C156W_STAT__ga10_0: 0x NV_PGRAPH_PRI_GPC0_GPCCS_CTXSW_STATUS_GPC_0: 0x0 [ 738.31[ 8995] __g__ga10a10b__b__ NV_PGRAPH_PGPCRI_0_GPCCS_CTXSS_CTXSUS_W_S0x3US_1: 0x390 RAPH_PRI_FECS_CTXSW_IDLESTATE : 0xePRI_FEC[ .33TXSW_IDLESTATE : 0xe [ 738.33[ 7771] 777__ga10b__ NV NV_PGRAPH_PRI_GPC0_GPCCS_CS_CW_IDLEW_ITE DLExfTE : 0xf [ 738.348832] __ga10b__ NV_PGRAPH_PRI_[ 738URR.342] __ga10 0x NVb012e6RAP2c FECS_CURRENT_CTX : 0xb012e62c [ 738738.355837] __ga10b__b___PGRAP_PGRAPH_PRI_FECEW_EW_CTXCTX : 0x30x3012e62c [ 738.364588] __ga10b__ NV_PGRAPH_PRI_FEC[ S_HOST4588] NABLE b__ NVx7f_PG000H_P3 FECS_HOST_INT_ENABLE : 0x7f0003 T_STATUS : 0x08.37__g3072] NV_PGRAPH_PRI__PGS_HOSTH_P_INT_STATS_HOST_INx0 [ 738.382348] __ga10b__ NV_PGRAPH_PRI_[ GPC.388] OP0a10_CRb__ NV_PGTUSRAPH_P 0xGPCS_ROP00 _CROP_STATUS1 : 0x5f00000 [ 738.39[ 738.39206__g1] __ga10b__RAPRI_GPCS_RRAPOPSRI__CRSTAGPCS_R1 :_CROP_STATUS1 : 0x0 5f00 [ 738.403431] __ga10b__ NV_PGRAPH_PRI_GPC[ S_R.40343OP0OP_STATUS : b__0x0 H_PRI_GPCS_ROP0_ZROP_STATUS : 0x0 [ 738.40[ 738.409824] __g NVb__ NV_PGRAPH_PGPCGPCS_ROP0OP0_ZROP_OP_STATUS2 :2 :00 RAPH_PRI_GPCS_ROP1_ZROP_STATUS: 0x0PRI_[ GPCS_R040OP1_ZROP_STAb__ NV_PGx0 [ 738.42[ 6423] 3] a10b__ NVb___PGRAPH_PH_PRI_RI_GPCOP1S_R_ZROP1_ZRSTAOP_TUS2: 2: 0x00x0 [ 738.437737] __ga10b__ NV_PGRAPH_PRI_[ GPCS_R773OPS7] __ga10TUSb__ : 0x0RAPH_PRI_GPCS_ROPS_ZROP_STATUS : 0x0 [ 738738.444648] __ga10a10b___PGRAPH_PRI_RAPH_PS_RRI_OPSS_R_ZROPSSTA_ZRSTATUS2 : 0x0 [ 738.455676] __ga10b__ NV_PGRAPH_P[ RI_GPC5676] __gS_GPC_ NVEPT_PGION: 0x0 PCCS_GPC_EXCEPTION: 0x0 [ 738[ 7389] 2939] b__ NV_PGRAPH_PRI_GPCRAPH_PS_GPC_0_GEPTIONS_G_EN: 0EPT0f6004ION: 0xfc0f6004 [ 738.474390] __ga10b__ NV_PGRAPH_P[ RI_GPC0_T__ga10b__ NV_PG_TPC_EXCEPTION: 0x0 _TPCCS_TPC_EXCEPTION: 0x0 [ 738.48[ 738.481483] __g NVb__ NV_PGRAPH_P0_TPC0GPC0_TPC0_TP_TPC_EC_EXCEON_EN:PTI 0xEN:16 [ 738.492680] __ga10b__ NV_PGRAPH_PRI_GPCS_T[ 738.49S_H268WW_0] __g_ESR_R NVEPO_PGMASRAPH_PRI_GPCS_TPCS_SMS_HWW_GLOBAL_ESR_REPORT_MASK: 0x0 [ 738.50738.504] __ga10__gb__b__ NV NV_PGRAPH_PRI_GPCGPCS_T_SMPCSS_HS_HWW_WARWW_P_ESR_REPORT_MAREPSK: 0x0 0x 0 [ 738.515600] __ga10b__ NV_PGRAPH_PRI_GPC[ .51560_SM__ga10b__ NV_PGRAPH_P0x0 GPCPCS_SMS_HWW_GLOBAL_ESR: 0x0 [ 738.52[ 2229] __ga10__g NV_PGRAPH_PRI_RAPH_PRI__SMS_DBGR_CONTRS_DBGROL0: 0NTR: 0x1c 00 [ 738.532500] __ga10b__ NV_PGRAPH_PRI_[ 738PCS_SM.53S_DBGR0] _ST__ga10b__ NV_PGRAPH_PRI_GPCS_TPCS_SMS_DBGR_STATUS0: 0x0 [ 738[ 7383] __ga10976b__a10b__ NV_PG_PGRAPRAPRI_GPCRI_S_TPCS_SMS_DS_D_BP_BPT_PAUSAUS_0: 0x_0: 0x0 [ 738.552965] __ga10b__ NV_PGRAPH_PRI_GPCS_T[ PCS296S_D__ga10b__T_PAUSRAPH_PRI_GPC0 PCS_SMS_DBGR_BPT_PAUSE_MASK_1: 0x0 [ 738[ .560928] __ga10b__b__ NVRAP_PGH_PRI_GPC0_TPC0_SM0_H_SM0_HWARWW_P_ESR:SR:00 [ 738.571172] __ga10b__ NV_PGRAPH_PRI_[ GPC1172] _SM0_HWW_WARP_ESR_REPORT_MASK: 0x0_TPC0_SM0_HWW_WARP_ESR_REPORT_MASK: 0x0 [ 738.580616] __ga10b__ NV_PGRAPH_PRI_GPC0_TPC0_SM0_HWW_GLOBAL_ESR: 0x0 [ 738.588932] __ga10b__ NV_PGRAPH_PRI_GPC[ 0_T.58_SM6] 0_H__ga10BAL_ES_PGRAPH_PRI_MAS0_T0x0_SM0_HWW_GLOBAL_ESR: 0x0 [ 738.588932] __ga10b__ NV_PGRAPH_PRI_GPC0_TPC0_SM0_HWW_GLOBAL_ES[ 738RT_MAS837K: 0x06] a10b__[ NV_PG.59RAP837H_PRI___gGPCa10PC0 NV0_D_PGRAPH_PRI_GPC: 00_T00 _SM0_DBGR_CONTROL0: 0x1c00 [ 738.613872] __ga10b__ NV_PGRAPH_PRI_GPC0_TPC0_SM0_DBGR_STATUS0: 0x0 [ 738.622270] __ga10b__ NV_PGRAPH_PRI_GPC0_T738.613872] _SM1_HWW_WARP_ESR: 0xH_PRI_GPC0_TPC0_SM0_DBGR_STATUS0: 0x0 [ 738.622270] __ga10b__ NV_PGRAPH_PRI_GPC6] PC0__g1_Hb__WAR_PGP_E 0xH_PRI_GPC0_TPC09186] __g1_Ha10WW_WARP_ESR_REPORT_MA0_TPC0_SM1_HWW_WARP_ESR_REPORT_MASK: 0x0 [ 738[ .644752] __ga10b__ NV_PGRAPRAPH_PGPCRI_PC00_TPC0_SM1_H1_HWW_WW_GLOR: BAL_ES0x00x0 [ 738.654908] __ga10b__ NV_PGRAPH_PRI_GPC0_T[ PC0.654901_HWW___gGLOBAL_ES NV_PGEPORT_MASRI_K: 0x0 PC0_SM1_HWW_GLOBAL_ESR_REPORT_MASK: 0x0 [ 738.664874] __ga10b__ NV[ _PG.66RAPH_PRI_GPCa10PC0_SM1_DBGR_PG_CORAPNTROL0GPC: 000 _SM1_DBGR_CONTROL0: 0x1c00 [ 738.67[ 738.67__g555a10__g NV_PGRAPH_PRI_H_PGPC0_TPC0PC0_SM1_DBGR1_DBGR_STATUS0:S0: 0x0 0 [ 738.685448] __ga10b__ NV_PGRAPH_PRI_GPC0_T[ PC1.685448] __ga10b__ NV 0x_PGRAPH_PRI_GPC0_TPC1_SM0_HWW_WARP_ESR: 0x0 [ 738.692796] __ga10b__ NV[ RAP.69RI_6] __gPC1a10b__0_H NVP_E_PGSR_RAPH_PRI_ORT0_TPC1_SM0_H0 WARP_ESR_REPORT_MASK: 0x0 [ 738.70[ 738.706537] __gb__a10b__ NVRAPRAPRI_RI_GPC0_TPC1PC1_SM0_HWW_WW_BAL_ESR: 0x0R: 0x0 [ 738.716245] __ga10b__ NV_PGRAPH_PRI_GPC0_T[ .71_SM5] __ga10GLOBAL_PGRAPH_PRT_MASK: 0x00_T PC1_SM0_HWW_GLOBAL_ESR_REPORT_MASK: 0x0 NTR738.727385428] 8] __ga10b__b__ NV_PGH_PRI_GPC0_TPC1_SMGPCBGR0_T_COOL0_SM0_D: 0BGR_CO00 x1c00 [ 738.736807] __ga10b__ NV_PGRAPH_P[ RI_GPC0_T6807] __ga10b__ATU_PGRAP 0x0 RI_GPC0_TPC1_SM0_DBGR_STATUS0: 0x0 [ 738.747383812] 2] a10__ga10b__ NV_PGRI_H_PRI_0_T_SM1_HWW_WARP_E1_HSR: 0x0 0x0 [ 738.753873] __ga10b__ NV_PGRAPH_PRI_GPC0_T738.75_SM1_HWW_WARP_Eb__ NVORT_PG_MARI_GPC0_T0 PC1_SM1_HWW_WARP_ESR_REPORT_MASK: 0x0 [ 738.76738.765] __ga10b__ NVb__ NVH_PRI_GPC0_TPC1_SMGPC0_TWW__SMGLO1_HBALGLO_ES_ESR: 0x0 [ 738.774170] __ga10b__ NV_PGRAPH_PRI_GPC[ 0_T738.771_HWW_GLOBALa10_ESR_R_PGRAPRT_H_PRI_GPCK: PC1_SM WW_GLOBAL_ESR_REPORT_MASK: 0x0 [ 738.78738.784577] 7] __ga10b__ NVRAPH_PRI_RAPH_P0_TPC1GPC_SM_SM1_DBGR_CO_COOL0NTR: 0x1cx1c00 [ 738.795170] __ga10b__ NV_PGRAPH_PRI_GPC0_TPC1738_SM1_DBGR_STATUS0: 0x NV0 H_PRI_GPC0_TPC1_SM1_DBGR_STATUS0: 0x0 [ 738.80[ 161.805] __ga10b__ NV_PGRAPH_PH_PRI_GPC0_T0_TPC2_SMWW_WARWW_WARP_ESR: 0x0 [ 738.812684] __ga10b__ NV_PGRAPH_PRI_GPC0_T[ PC2_SM2684] WAR__ga10P_ESR_REPRAPH_PRI_GPC0_TPC2_SM0_HWW_WARP_ESR_REPORT_MASK: 0x0 [ 738738.821769] 9] a10b__ NV_PGRAPH_PRI_GPCRI_PC2_SM0_H0_TPC2_SMBALR: 0x0GLOBAL_ESR: 0x0 [ 738.832535] __ga10b__ NV_PGRAPH_PRI_GPC0_T[ 738_SM2535] __ga10_ES NVEPORT_MASK: GPC0_TPC2_SM0_HWW_GLOBAL_ESR_REPORT_MASK: 0x0 [ 738.84738.841] __ga10b__ NVb__ NV_PGRAPH_PRI_GPC0_T0_TPC2_SMBGR_CO_CONTROL0: 0x1c00 [ 738.852306] __ga10b__ NV_PGRAPH_PRI_GPC[ 0_TPC2230_SM0_D__ga10BGR_ST NV_PGS0:RAPH_P0 RI_GPC0_TPC2_SM0_DBGR_STATUS0: 0x0 [ 738[ 738.859838] __ga10b__b__ NV NV_PG_PGRAPH_PGPCGPCPC2_SMPC2WW_WARWW_WARSR:P_ESR:0 0x [ 738.871035] __ga10b__ NV_PGRAPH_P[ RI_GPC0_T__g_SM1_HWW_WAR_PGP_ESR_REPORT_MASK:_SM0 WW_WARP_ESR_REPORT_MASK: 0x0 [ 738.88738.884] __ga10b__b__ NVRAPH_PRI_GPC0_TPC2_SMPC2WW_GLOWW_GLOR: _ES0x00x0 [ 738.889320] __ga10b__ NV_PGRAPH_PRI_[ GPC0_TPC20] a10b__ NV_PGGLORAPH_PRI_GPC0_TMASPC20x0 WW_GLOBAL_ESR_REPORT_MASK: 0x0 [ 738[ .898851] __ga10b__ NV_PGRAPRAPH_PRI_GPCRI_0_TGPCPC20_TPC2_SM1_DBGRBGR_CONTROL0OL0x1c: 000 [ 738.909969] __ga10b__ NV_PGRAPH_PRI_GPC[ 738_SM.901_D9969] ATU__gS0:a10b__ NV_PGRAPH_PRI_GPC0_TPC2_SM1_DBGR_STATUS0: 0x0 [ 738.916416] __ga10b__ NV_PGRAPH_PRI_GPC0_TPC3_SM0_HWW_WARP_ESR: 0x0 [ 738.926946] __ga10b__ NV_PGRAPH_PRI_GPC[ 0_T.91PC3641_SMWW_WARa10b__ NV_PG_MARAP 0xRI_GPC0_TPC3_SM0_HWW_WARP_ESR: 0x0 [ 738.926946] __ga10b__ NV_PGRAPH_PRI_GPC0_TPC3_SM738WW_WAR5951] __ga10b___MA NV_PG 0x0H_P RI_GPC5951] _SM__gb__ NVGLO_PGH_PRI_R: 0_TPC3_SM0_HWW_GLOBAL_ESR: 0x0 [ 738.950134] __ga10b__ NV_PGRAPH_PRI_GPC0_TPC3_SM0_HWW_GLOBAL_ESR_REPORT_MASK: 0x0 [ 738.961152] __ga10b__ NV_PGRAPH_PRI_GPC0_T[ .95_SM0_DBGRa10NTROL0: 0x1c00 RI_GPC0_TPC3_SM0_HWW_GLOBAL_ESR_REPORT_MASK: 0x0 [ 738.961152] __ga10b__ NV_PGRAP738.96795GPCPC3__g0_DBGR_CONTR NVRAPOL0x1c00RI_GPC[ PC3_SM.967956] BGRATUa10 0xb__ NV_PGRAPH_PRI_GPC0_TPC3_SM0_DBGR_STATUS0: 0x0 [ 738738.981721] 1] __ga10b__b__ NV_PGRAPH_PGPCRI_0_TPC3_SM1_HWW_WARPC3P_ESR:1_HWAR 0x0SR: 0x0 [ 738.994409] __ga10b__ NV_PGRAPH_PRI_[ 738.99PC34409] __ga10b__SR_REPORTRAPSK:RI_GPC0_TPC3_SM1_HWW_WARP_ESR_REPORT_MASK: 0x0 [ 739.007393065] __ga10__ga10b__ NV_PGRAPGPCH_PGPC0_TPC31_H1_HWW_GLOBALBAL_ESR: R: 0x0 [ 739.013659] __ga10b__ NV_PGRAPH_PRI_GPC0_T[ 739PC33651_H__ga10GLOBAL_ESR_REPORT_H_PMASRI_GPC PC3_SM1_HWW_GLOBAL_ESR_REPORT_MASK: 0x0 [ 739[ .023363] __ga10b__ NV_PGRAPH_PRI_GPC0_TPC3_SM1_DBGR_CONTROL0: 0x1c_CONTR OL0: 0x1c00 [ 739.034568] __ga10b__ NV_PGRAPH_PRI_GPC[ 0_TPC34561_D8] BGR_STb__S0: 0x0 RAPH_PRI_GPC0_TPC3_SM1_DBGR_STATUS0: 0x0 [ 739.041704] nvgpu: 17000000.ga10b nvgpu_set_err_notifier_locked:149 [ERR] error notifier set to 8 for ch 511 [ 739.041704] nvgpu: 17000000.ga10b nvgpu_set_err_notifier_locked:149 [ERR] error notifier set to 8 for ch 511 [ 739.060929] __ga10b__ Channel Status - chip ga10b [ 739.060931] __ga10b__ --------------739.06---------- b__ Channel Status - chip ga10b [ 739.060931] __ga10739.068714] ---__g----------------9-ga10b, TSG739.06, pid 1717, a10refs: 9, determ9-ginia10c: no,TSG domaiid ame171: (defrefs: 9, deterministic: no, domain name: (default) [ 739.073547] __ga10b__ channel status: not in use on_pbdma, on_eng, pending, pbdma_busy, eng_busy busy [ 739.094404] __ga10b__ RAMFC: TOP: 800000200400dc08 PUT: 0020040[ 0dc.0735408 __g: 00200400dc08 el FETtusCH:in 000000_pb0000 HdmaER: 21400n_eng,UNTndi000 pbdma_buPHORE:g_bdr busy004000 paylo739.094400 0200400dc08 PUT: 00200400dc08 GET: 00200400dc08 FETCH: 000000000000 HEADER: 2140006c COUNT: 00000000 SEMAPHORE: addr 002004010000 payload 0000000000000000 execute 00100001 [ [ .107617612] __ga10 [ 739.138347] __ga10b__ 510-ga10b, TSG: 1, pid 1477, refs: 2, deterministic: no, domain name: (default) [ 739.138347] __ga10b__ 510-ga10b, TSG: 1, pid 1477, refs: 2, deterministic: no, domain name: (default) [ [ .14086.15__g9] b__ channb__ RAMFC: in 80 idle not busy331 PUT[ 0207390403316089] GET__ga10040b__331 RAMFCFETCH:OP:000000000000331EADb8 ER: 21: 0020400040331b8 0000000 SEMAPHOb8 FETCH: ad 000020043200000 HEADad 21000000 COUNT0000000 e000ute 00PHORE:001 addr 001 [ 739[ 739.161811] __ga10a10b__ [ 739.203269] __ga10b__ 511-ga10b, TSG: 0, pid 1477, refs: 116, deterministic: no, domain name: (default) [ 739.205800] __ga10b__ channel status: not in use739_eng, 9] __ga10b__eng1-gsy busy : 0, pid 1477, refs: 116, deterministic: no, domain name: (default) [ 739.205800] __ga10b__ channel status: not in use on_eng, pending, eng_busy [ 739.21 739.21544544a10b__a10MFC RAMFC: T000 800200000404f0040PUT: 002078 040PUT: 078 0404f0: 0020: 00200404f04f0FET78 CH:000000000000000000ER:40006c CO 2140000000006c0 SPHO COUNT: 0 addr 0020 SPHO004020000 addr1 [ [ 739.222957] __ga10b__ [ 739.266097] __ga10b__ PBDMA Status - chip ga10b [ 739.268622] __ga10b__ -----------------------[ .266097] __ga10b__ PBDMA Status - chip ga10b [ 739.268622] __ga10b__ ------------------------- [ 739[ .273283281] __ga10b__ pb 0: 0: [ 739.277942] __ga10b__ id: 2 - [tsg] next_id: - -1 [channel] | status: valid - [tsg] next_id: - -1 [channel] | status: valid38 PBD2] __gGETa10b__ 20da0 [ 739.281080] __ga10b__ PBDMA_PUT 0000001ffa5213[ 739MA_8769] 000__gfa520da0GP_ 00[ 0009c19] P_Ga10b__ 00 9b0PUTETCH 0009c1 00000ET HE 00ADE000 FR 20ffETCH 000009bd HEADER 20000ff8 [ 739.296114] __ga10b__ HDR 200203fd SHADOW0 fa520ce8 SHADOW1 0006501f [ 739.315696] __ga10b__ pbdma 1: [ 739.296114] __ga10b__ HDR 200203fd SHADOW0 fa520ce8 SHADOW1 0006501f [ [ 739.32.316] __ga106] __ga10b__id: - [channel] next_id:739.32-1 6] ann__g | a10 tus -1 - [chlidel] next_id: - -1 [channel] | status: invalid [ 739.327328] __ga10b__ PBDMA_PUT 000000b64b5adf98 PBDMA_GET 000000d9b13d36f8 [ 739.343881] __ga10b__ GP_PUT 00000000[ 739 G.32ET 7fd61a103ceb__ETC PBD000PUT 00ADE000R a1d298 PBD4 MA_GET 000000d9b13d36f8 [ 739.343881] __ga10b__ GP_PUT 00000000 GP_GET 7fd613ce FETCH 00[ 000446000 HEADEa10b__ HDR [ 9fbc74469] __gOW0a10 b42dc 9f Sbc7HADbfcOW1 29 b42dc4e3 SHADOW1 29152802 [ 739[ .365916] __g__ga10b__ pbdmadma 2: [ 739.382555] __ga10b__ id: -1 - [channel] next_id: - -1 [channel] | status: invalid annel] next_id: - -1 [channel] | status: invaliddf946540 255__ga10GET 00000039a993e478 [ 739.385712] __ga10b__ PBDMA_PUT[ 000739067.390] __g40 b__MA_GET 00PUT000a993e4000000 P_G[ 739 246383c6 Fa10b__ 00000 000PUT HEADE R 8 00000000317P_Gc ET 24a913c6 FETCH 00000000 HEADER 8017317c [ 739.403731] __ga10b__ HDR c094b5c1 SHADOW0 c8658411 SHADOW1 f7ee03bc [ 739.423320] __ga10b__ pbdma 3: [ 739.403731] __ga10b__ HDR c094b5c1 SHADOW0 c8658411 SHADOW1 f7ee03bc [ 739.433323910] 0] __ga10b__ dma -1 - [chann739.433910] __ga10-1 [chann el] - [channtusel] next_ - -1 [channel] | status: invalid [ 739.437040] __ga10b__ PBDMA_PUT 00000[ 0d50b3d67.43704PBDMA_a10 00 PBDMA_0b3PUT 000000d50b3d6760 PBDMA_GET 000000d50b3d6760 [ 739.457394034039] __ga10b__b__ GP_PUTPUT000000 00 GP_G GET eaef7cET 3aETCf7cH 00H 000000ADER e00024bADE8R e 1d524b8 [ 739.464279] __ga10b__ HDR dfe77[ 739HAD.46 d19] 614a10b__ HDR 986 dfe77b54 SHADOW0 d142e614 SHADOW1 144a4986 [ 739.481242] __ga10b__ pb739dma 4:2] a10b__ pbdma 4: [ 739.488655] __ga10b__ id: -1 - [channel] next_id: - -1 [channel] | status: invalid [ 739.491817] __ga10b__ PBDMA_PUT 000000796bee53739bc 8655] __ga100000ddid: -1 - [ch annel] next_id: - -1 [channel] | status: invalid [ 739.491817] __ga10b__ PBDMA_739.500000796bea10e53b__PBDGETGP_ 00 000 00c1a000 GP_G739.500a04438] ETCH __g 00000000 PUTADER 81120000002eac P_GET 220a03c9 FETCH 00000000 HEADER 81122eac [ 739.511780] __ga10b__ HDR 222023c5 SHADOW0 2845d862 SHADOW1 00100e8f [ 739.530680] __ga10b__ pbdma 5: [ 739.511780] __ga10b__ HDR 222023c5 SHADOW0 2845d862 SHADOW1 00100e8f [ [ .53.540] 127a10b__ pbb__ 5: id: -1739.54[ch127ann7] __g nea10b__ - -1 [channel] | - tus[chnvalid next_id: - -1 [channel] | status: invalid [ 739.544415] __ga10b__ PBDMA_PUT 000000ce1aa97b50 PBDMA_GET 000000ce1aa97b50 [ 739.561753] __ga10b__ GP_PUT 00000[ 739000 G441ET __ga104ce 71c FETCH PUT 00000000 HEADER 897b1870d7c GET 000000ce1aa97b50 [ 739.561753] __ga10b__ GP_PUT 00000000 GP_GET 254ce71c FETCH 00[ 739 HE.57ADE7] __ga10b__ HDR[ e22076e5__g69fa10b__ 0c9d1HDR6d5 SHAD6e5 4034aHADOW0 0c9d16d5 SHADOW1 4034a404 [ 739.583615] __ga10b__ [ 739.599290] __ga10b__ ga10b eng 0: [ 739.601799] __ga10b__ id: 0 (tsg), next_id[ : 2.58361sg)5] status: switch [ .599290] __ga10b__ ga10b eng 0: [ 739.601799] __ga10b__ id: 0 (tsg), next_id[ : 25383] sg)__gtx sta bu: switch [ 739.605383] __ga10b__ bu[ 739.61486[ 739.61486 a10b__ [ 739.620761] __ga10[ b__0761] 10ba10b__g 1 ga10b g 1: invalid 2[ .622] __g__ga10b__ id: -: -1 (channennel), next_xt_id:id: (c -1 (cnelhanctx statu), s: stinvalid [ 739.632744] __ga10b__ [ 739.632744] __ga10b__ [ 739[ 739.647761] __g gab__ ga10b eng 2: [ 739.650276] __ga10b__ id: -1 ([ 739l),.650276] __ga10han id), 1 (ctxchanneatus: xt_id:ali -1 hannel), ctx status: invalid [ 739[ .653880] __ga10b__ b__ [ 739.666576] __ga10b__ ga10b eng 3: [ 739.669097] __ga10b__ id: -1 (channel), ne[ xt_id: -16] __gnelb__ctx statu ens: invali739 .669097] __ga10b__ id: -1 (channel), next_id: -1 (channel), ctx status: invalid [ [ 7392682680] __ga10b__ .684598] __ga10b__ ga10b eng 4: n739: st739.68962[ __ga10.68 id5] : -1 (a10b__ id ne: -xt_nne -1 (c next_), ctx -1 sthannelinv), ctxd s: invalid [ 739[ .694338] __ga10b__ [ 739.710324] __ga10b__ ga10b eng 5: : 739.712844] __ga10b__ id: -1 (channel), next_id:[ -1.71032nel__ga10b__ ga10b eng 5alid [ .712844] __ga10b__ id: -1 (channel), next_id: -1 (channel), ctx status: invalid [ 739.71642[ 739__g6426] b____ga10 b__ [ 739.726770] __ga10b__ [ 752.773732] rcu: INFO: rcu_preempt self-detected stall on CPU [ 752.782191] rcu: 0-....: (5232 ticks this GP) idle=71e/1/0x4000000000000004 softirq=11994/11995 fqs=2255 [ 752.792509] rcu: ====For debug only: Start Printing Blocked Tasks==== [ 752.800702] task:kworker/4:0 state:D stack: 0 pid: 35 ppid: 2 flags:0x00000028 [ 752.809312] Workqueue: pm pm_runtime_work [ 752.813445] Call trace: [ 752.815967] __switch_to+0xc8/0x120 [ 752.819570] __schedule+0x3d0/0x910 [ 752.823168] schedule+0x74/0x100 [ 752.826503] synchronize_irq+0x84/0xc0 [ 752.830371] disable_irq+0x38/0x50 [ 752.834106] nvgpu_disable_irqs+0x40/0x80 [nvgpu] [ 752.839139] gk20a_pm_prepare_poweroff+0x98/0x120 [nvgpu] [ 752.844868] gk20a_pm_runtime_suspend+0x78/0xf0 [nvgpu] [ 752.850253] pm_generic_runtime_suspend+0x3c/0x60 [ 752.855099] genpd_runtime_suspend+0xac/0x260 [ 752.859589] __rpm_callback+0xfc/0x1a0 [ 752.863457] rpm_callback+0x34/0xa0 [ 752.867056] rpm_suspend+0xe0/0x620 [ 752.870656] pm_runtime_work+0xbc/0xc0 [ 752.874529] process_one_work+0x1c0/0x4a0 [ 752.878663] worker_thread+0x50/0x420 [ 752.882435] kthread+0x148/0x170 [ 752.885766] ret_from_fork+0x10/0x18 [ 752.889478] task:khugepaged state:D stack: 0 pid: 63 ppid: 2 flags:0x00000028 [ 752.898079] Call trace: [ 752.900601] __switch_to+0xc8/0x120 [ 752.904207] __schedule+0x3d0/0x910 [ 752.907805] schedule+0x74/0x100 [ 752.911133] schedule_timeout+0x2d4/0x330 [ 752.915269] wait_for_completion+0x88/0x110 [ 752.919577] __flush_work.isra.0+0x104/0x220 [ 752.923977] flush_work+0x20/0x30 [ 752.927396] lru_add_drain_all+0x1a0/0x210 [ 752.931622] khugepaged+0x98/0x1df0 [ 752.935221] kthread+0x148/0x170 [ 752.938554] ret_from_fork+0x10/0x18 [ 752.942510] task:nvidia-modeset/ state:D stack: 0 pid: 1524 ppid: 2 flags:0x00000028 [ 752.951113] Call trace: [ 752.953633] __switch_to+0xc8/0x120 [ 752.957234] __schedule+0x3d0/0x910 [ 752.960836] schedule+0x74/0x100 [ 752.964165] schedule_timeout+0x2d4/0x330 [ 752.968301] __down+0x74/0xd0 [ 752.971367] down+0x54/0x70 [ 752.974334] nvKmsKapiGetFunctionsTable+0x6b0/0x770 [nvidia_modeset] [ 752.980955] nvkms_sema_up+0x25c/0x3e0 [nvidia_modeset] [ 752.986334] kthread+0x148/0x170 [ 752.989664] ret_from_fork+0x10/0x18 [ 752.993392] task:gnome-shell state:D stack: 0 pid: 1717 ppid: 1458 flags:0x00000000 [ 753.001994] Call trace: [ 753.004517] __switch_to+0xc8/0x120 [ 753.008117] __schedule+0x3d0/0x910 [ 753.011714] schedule+0x74/0x100 [ 753.015051] rpm_resume+0x154/0x740 [ 753.018654] __pm_runtime_resume+0x40/0x90 [ 753.023049] gk20a_busy+0x12c/0x1f0 [nvgpu] [ 753.027521] nvgpu_ioctl_tsg_dev_ioctl+0x8b8/0xf40 [nvgpu] [ 753.033169] __arm64_sys_ioctl+0xa8/0xf0 [ 753.037214] el0_svc_common.constprop.0+0x7c/0x1c0 [ 753.042150] do_el0_svc+0x34/0xa0 [ 753.045568] el0_svc+0x1c/0x30 [ 753.048719] el0_sync_handler+0xa8/0xb0 [ 753.052674] el0_sync+0x16c/0x180 [ 753.056329] rcu: ====For debug only: End Printing Blocked Tasks==== [ 739.726770] __ga10b__ [ 752.773732] rcu: INFO: rcu_preempt self-detected stall on CPU [ 752.782191] rcu: 0-....: (5232 ticks this GP) idle=71e/1/0x4000000000000004 softirq=11994/11995 fqs=2255 [ 752.792135] (t=5254 jiffies g=24765 q=664) [ 752.792141] Task dump for CPU 0: [ 752.792147] task:cat state:R running task stack: 0 pid: 2600 ppid: 2228 flags:0x00000002 [ 752.792160] Call trace: [ 752.792182] dump_backtrace+0x0/0x1d0 [ 752.792190] show_stack+0x2c/0x40 [ 752.792200] sched_show_task+0x144/0x170 [ 752.792216] dump_cpu_task+0x48/0x54 [ 752.792221] rcu_dump_cpu_stacks+0xb4/0xf0 [ 752.792232] rcu_sched_clock_irq+0xb1c/0xec0 [ 752.792239] update_process_times+0x64/0xa0 [ 752.792247] tick_sched_handle.isra.0+0x34/0x70 [ 752.792252] tick_sched_timer+0x50/0xa0 [ 752.792256] __hrtimer_run_queues+0x144/0x370 [ 752.792260] hrtimer_interrupt+0xec/0x240 [ 752.792271] arch_timer_handler_phys+0x3c/0x50 [ 752.792277] handle_percpu_devid_irq+0x8c/0x280 [ 752.792284] generic_handle_irq+0x3c/0x60 [ 752.792289] __handle_domain_irq+0x6c/0xc0 [ 752.792294] gic_handle_irq+0x64/0x130 [ 752.792297] el1_irq+0xd0/0x180 [ 752.792306] __clean_dcache_area_poc+0x34/0x38 [ 752.792309] dma_sync_single_for_device+0xb8/0x110 [ 752.792319] __arm_lpae_sync_pgtable.isra.0+0x6c/0xb0 [ 752.792323] __arm_lpae_dma_sync+0x1dc/0x200 [ 752.792327] __arm_lpae_dma_sync+0x148/0x200 [ 752.792331] __arm_lpae_dma_sync+0x148/0x200 [ 752.792336] __arm_lpae_dma_sync+0x148/0x200 [ 752.792340] arm_lpae_dma_sync+0x50/0x80 [ 752.792346] arm_smmu_dma_sync+0x3c/0x60 [ 752.792349] __iommu_unmap+0xd0/0x1d0 [ 752.792352] iommu_unmap_fast+0x3c/0x50 [ 752.792356] __iommu_dma_unmap+0x88/0x110 [ 752.792361] iommu_dma_unmap_page+0x4c/0x80 [ 752.792364] dma_unmap_page_attrs+0x60/0x200 [ 752.792373] usb_hcd_unmap_urb_for_dma+0x6c/0x110 [ 752.792377] unmap_urb_for_dma+0x44/0x50 [ 752.792382] __usb_hcd_giveback_urb+0x40/0x140 [ 752.792388] usb_giveback_urb_bh+0xb4/0x110 [ 752.792396] tasklet_action_common.isra.0+0x138/0x180 [ 752.792401] tasklet_hi_action+0x2c/0x40 [ 752.792404] __do_softirq+0x138/0x3e0 [ 752.792408] irq_exit+0xc0/0xe0 [ 752.792414] __handle_domain_irq+0x70/0xc0 [ 752.792418] gic_handle_irq+0x64/0x130 [ 752.792421] el1_irq+0xd0/0x180 [ 752.792427] preempt_schedule_irq+0x44/0x90 [ 752.792431] arm64_preempt_schedule_irq+0x18/0x20 [ 752.792434] el1_irq+0xe8/0x180 [ 752.792442] write_test+0x228/0x390 [ 752.792452] seq_read_iter+0x1cc/0x490 [ 752.792456] seq_read+0xf0/0x150 [ 752.792464] full_proxy_read+0x68/0xa0 [ 752.792470] vfs_read+0xb0/0x1c0 [ 752.792474] ksys_read+0x78/0x100 [ 752.792478] __arm64_sys_read+0x24/0x30 [ 752.792485] el0_svc_common.constprop.0+0x7c/0x1c0 [ 752.792489] do_el0_svc+0x34/0xa0 [ 752.792496] el0_svc+0x1c/0x30 [ 752.792502] el0_sync_handler+0xa8/0xb0 [ 752.792505] el0_sync+0x16c/0x180 [ 752.792509] rcu: ====For debug only: Start Printing Blocked Tasks==== [ 752.800702] task:kworker/4:0 state:D stack: 0 pid: 35 ppid: 2 flags:0x00000028 [ 752.809312] Workqueue: pm pm_runtime_work [ 752.813445] Call trace: [ 752.815967] __switch_to+0xc8/0x120 [ 752.819570] __schedule+0x3d0/0x910 [ 752.823168] schedule+0x74/0x100 [ 752.826503] synchronize_irq+0x84/0xc0 [ 752.830371] disable_irq+0x38/0x50 [ 752.834106] nvgpu_disable_irqs+0x40/0x80 [nvgpu] [ 752.839139] gk20a_pm_prepare_poweroff+0x98/0x120 [nvgpu] [ 752.844868] gk20a_pm_runtime_suspend+0x78/0xf0 [nvgpu] [ 752.850253] pm_generic_runtime_suspend+0x3c/0x60 [ 752.855099] genpd_runtime_suspend+0xac/0x260 [ 752.859589] __rpm_callback+0xfc/0x1a0 [ 752.863457] rpm_callback+0x34/0xa0 [ 752.867056] rpm_suspend+0xe0/0x620 [ 752.870656] pm_runtime_work+0xbc/0xc0 [ 752.874529] process_one_work+0x1c0/0x4a0 [ 752.878663] worker_thread+0x50/0x420 [ 752.882435] kthread+0x148/0x170 [ 752.885766] ret_from_fork+0x10/0x18 [ 752.889478] task:khugepaged state:D stack: 0 pid: 63 ppid: 2 flags:0x00000028 [ 752.898079] Call trace: [ 752.900601] __switch_to+0xc8/0x120 [ 752.904207] __schedule+0x3d0/0x910 [ 752.907805] schedule+0x74/0x100 [ 752.911133] schedule_timeout+0x2d4/0x330 [ 752.915269] wait_for_completion+0x88/0x110 [ 752.919577] __flush_work.isra.0+0x104/0x220 [ 752.923977] flush_work+0x20/0x30 [ 752.927396] lru_add_drain_all+0x1a0/0x210 [ 752.931622] khugepaged+0x98/0x1df0 [ 752.935221] kthread+0x148/0x170 [ 752.938554] ret_from_fork+0x10/0x18 [ 752.942510] task:nvidia-modeset/ state:D stack: 0 pid: 1524 ppid: 2 flags:0x00000028 [ 752.951113] Call trace: [ 752.953633] __switch_to+0xc8/0x120 [ 752.957234] __schedule+0x3d0/0x910 [ 752.960836] schedule+0x74/0x100 [ 752.964165] schedule_timeout+0x2d4/0x330 [ 752.968301] __down+0x74/0xd0 [ 752.971367] down+0x54/0x70 [ 752.974334] nvKmsKapiGetFunctionsTable+0x6b0/0x770 [nvidia_modeset] [ 752.980955] nvkms_sema_up+0x25c/0x3e0 [nvidia_modeset] [ 752.986334] kthread+0x148/0x170 [ 752.989664] ret_from_fork+0x10/0x18 [ 752.993392] task:gnome-shell state:D stack: 0 pid: 1717 ppid: 1458 flags:0x00000000 [ 753.001994] Call trace: [ 753.004517] __switch_to+0xc8/0x120 [ 753.008117] __schedule+0x3d0/0x910 [ 753.011714] schedule+0x74/0x100 [ 753.015051] rpm_resume+0x154/0x740 [ 753.018654] __pm_runtime_resume+0x40/0x90 [ 753.023049] gk20a_busy+0x12c/0x1f0 [nvgpu] [ 753.027521] nvgpu_ioctl_tsg_dev_ioctl+0x8b8/0xf40 [nvgpu] [ 753.033169] __arm64_sys_ioctl+0xa8/0xf0 [ 753.037214] el0_svc_common.constprop.0+0x7c/0x1c0 [ 753.042150] do_el0_svc+0x34/0xa0 [ 753.045568] el0_svc+0x1c/0x30 [ 753.048719] el0_sync_handler+0xa8/0xb0 [ 753.052674] el0_sync+0x16c/0x180 [ 753.056329] rcu: ====For debug only: End Printing Blocked Tasks==== [ 780.537464] watchdog: BUG: soft lockup - CPU#0 stuck for 22s! [cat:2600] [ 780.544378] Modules linked in: nvidia_modeset(OE) fuse lzo_rle lzo_compress zram ramoops reed_solomon loop nvgpu aes_ce_blk crypto_simd snd_soc_tegra186_asrc cryptd snd_soc_tegra210_ope snd_soc_tegra1] [ 780.627175] CPU: 0 PID: 2600 Comm: cat Tainted: G OE 5.10.104-tegra #145 [ 780.635329] Hardware name: Unknown Jetson AGX Orin/Jetson AGX Orin, BIOS 202210.0-Unknown 01/11/2023 [ 780.644738] pstate: 20400009 (nzCv daif +PAN -UAO -TCO BTYPE=--) [ 780.650941] pc : __clean_dcache_area_poc+0x34/0x38 [ 780.655879] lr : arch_sync_dma_for_device+0x40/0x50 [ 780.660897] sp : ffff800010003960 [ 780.664311] x29: ffff800010003960 x28: ffff313f06247000 [ 780.669785] x27: ffff314601c25d00 x26: 0000000000001000 [ 780.675264] x25: 0000000106247000 x24: 0000000000000000 [ 780.680736] x23: 0000007ffff23000 x22: 0000000000000008 [ 780.686205] x21: 0000000000000001 x20: 0000000000000008 [ 780.691680] x19: 0000000106247918 x18: 0000000000000017 [ 780.697155] x17: 0000000000000007 x16: ffffc3c3f30506cc [ 780.702627] x15: 0000000000000001 x14: 0000000000000019 [ 780.708100] x13: 0000000000000033 x12: 000000000000004c [ 780.713576] x11: 0000000000000000 x10: 0000000000000079 [ 780.719049] x9 : 0000000007ffff23 x8 : 0000000000001e80 [ 780.724525] x7 : ffffc3c3f392c060 x6 : 0000000000000123 [ 780.729997] x5 : 0000000000000009 x4 : ffff313f06247000 [ 780.735473] x3 : 000000000000003f x2 : 0000000000000040 [ 780.740942] x1 : ffff313f06247920 x0 : ffff313f06247940 [ 780.746415] Call trace: [ 780.748938] __clean_dcache_area_poc+0x34/0x38 [ 780.753519] dma_sync_single_for_device+0xb8/0x110 [ 780.758459] __arm_lpae_sync_pgtable.isra.0+0x6c/0xb0 [ 780.763661] __arm_lpae_dma_sync+0x1dc/0x200 [ 780.768060] __arm_lpae_dma_sync+0x148/0x200 [ 780.772462] __arm_lpae_dma_sync+0x148/0x200 [ 780.776864] __arm_lpae_dma_sync+0x148/0x200 [ 780.781265] arm_lpae_dma_sync+0x50/0x80 [ 780.785308] arm_smmu_dma_sync+0x3c/0x60 [ 780.789350] __iommu_unmap+0xd0/0x1d0 [ 780.793123] iommu_unmap_fast+0x3c/0x50 [ 780.797077] __iommu_dma_unmap+0x88/0x110 [ 780.801212] iommu_dma_unmap_page+0x4c/0x80 [ 780.805520] dma_unmap_page_attrs+0x60/0x200 [ 780.809924] usb_hcd_unmap_urb_for_dma+0x6c/0x110 [ 780.814767] unmap_urb_for_dma+0x44/0x50 [ 780.818807] __usb_hcd_giveback_urb+0x40/0x140 [ 780.823383] usb_giveback_urb_bh+0xb4/0x110 [ 780.827694] tasklet_action_common.isra.0+0x138/0x180 [ 780.832898] tasklet_hi_action+0x2c/0x40 [ 780.836940] __do_softirq+0x138/0x3e0 [ 780.840713] irq_exit+0xc0/0xe0 [ 780.843953] __handle_domain_irq+0x70/0xc0 [ 780.848176] gic_handle_irq+0x64/0x130 [ 780.852043] el1_irq+0xd0/0x180 [ 780.855281] preempt_schedule_irq+0x44/0x90 [ 780.859589] arm64_preempt_schedule_irq+0x18/0x20 [ 780.864429] el1_irq+0xe8/0x180 [ 780.867668] write_test+0x228/0x390 [ 780.871269] seq_read_iter+0x1cc/0x490 [ 780.875135] seq_read+0xf0/0x150 [ 780.878466] full_proxy_read+0x68/0xa0 [ 780.882331] vfs_read+0xb0/0x1c0 [ 780.885661] ksys_read+0x78/0x100 [ 780.889078] __arm64_sys_read+0x24/0x30 [ 780.893036] el0_svc_common.constprop.0+0x7c/0x1c0 [ 780.897974] do_el0_svc+0x34/0xa0 [ 780.901392] el0_svc+0x1c/0x30 [ 780.904540] el0_sync_handler+0xa8/0xb0 [ 780.908495] el0_sync+0x16c/0x180 [ 780.911912] watchdog: ====For debug only: Start Printing Blocked Tasks==== [ 780.920713] task:kworker/4:0 state:D stack: 0 pid: 35 ppid: 2 flags:0x00000028 [ 780.929323] Workqueue: pm pm_runtime_work [ 780.933457] Call trace: [ 780.935976] __switch_to+0xc8/0x120 [ 780.939575] __schedule+0x3d0/0x910 [ 780.943172] schedule+0x74/0x100 [ 780.946500] synchronize_irq+0x84/0xc0 [ 780.950367] disable_irq+0x38/0x50 [ 780.954041] nvgpu_disable_irqs+0x40/0x80 [nvgpu] [ 780.959042] gk20a_pm_prepare_poweroff+0x98/0x120 [nvgpu] [ 780.964757] gk20a_pm_runtime_suspend+0x78/0xf0 [nvgpu] [ 780.970129] pm_generic_runtime_suspend+0x3c/0x60 [ 780.974973] genpd_runtime_suspend+0xac/0x260 [ 780.979460] __rpm_callback+0xfc/0x1a0 [ 780.983329] rpm_callback+0x34/0xa0 [ 780.986925] rpm_suspend+0xe0/0x620 [ 780.990526] pm_runtime_work+0xbc/0xc0 [ 780.994397] process_one_work+0x1c0/0x4a0 [ 780.998531] worker_thread+0x50/0x420 [ 781.002303] kthread+0x148/0x170 [ 781.005636] ret_from_fork+0x10/0x18 [ 781.009330] task:khugepaged state:D stack: 0 pid: 63 ppid: 2 flags:0x00000028 [ 781.017925] Call trace: [ 781.020447] __switch_to+0xc8/0x120 [ 781.024046] __schedule+0x3d0/0x910 [ 781.027645] schedule+0x74/0x100 [ 781.030972] schedule_timeout+0x2d4/0x330 [ 781.035108] wait_for_completion+0x88/0x110 [ 781.039416] __flush_work.isra.0+0x104/0x220 [ 781.043817] flush_work+0x20/0x30 [ 781.047236] lru_add_drain_all+0x1a0/0x210 [ 781.051460] khugepaged+0x98/0x1df0 [ 781.055059] kthread+0x148/0x170 [ 781.058389] ret_from_fork+0x10/0x18 [ 781.062205] task:nvidia-modeset/ state:D stack: 0 pid: 1524 ppid: 2 flags:0x00000028 [ 781.070807] Call trace: [ 781.073328] __switch_to+0xc8/0x120 [ 781.076926] __schedule+0x3d0/0x910 [ 781.080524] schedule+0x74/0x100 [ 781.083854] schedule_timeout+0x2d4/0x330 [ 781.087987] __down+0x74/0xd0 [ 781.091052] down+0x54/0x70 [ 781.094014] nvKmsKapiGetFunctionsTable+0x6b0/0x770 [nvidia_modeset] [ 781.100631] nvkms_sema_up+0x25c/0x3e0 [nvidia_modeset] [ 781.106008] kthread+0x148/0x170 [ 781.109336] ret_from_fork+0x10/0x18 [ 781.113041] task:gnome-shell state:D stack: 0 pid: 1717 ppid: 1458 flags:0x00000000 [ 781.121640] Call trace: [ 781.124161] __switch_to+0xc8/0x120 [ 781.127755] __schedule+0x3d0/0x910 [ 781.131348] schedule+0x74/0x100 [ 781.134677] rpm_resume+0x154/0x740 [ 781.138272] __pm_runtime_resume+0x40/0x90 [ 781.142654] gk20a_busy+0x12c/0x1f0 [nvgpu] [ 781.147116] nvgpu_ioctl_tsg_dev_ioctl+0x8b8/0xf40 [nvgpu] [ 781.152762] __arm64_sys_ioctl+0xa8/0xf0 [ 781.156799] el0_svc_common.constprop.0+0x7c/0x1c0 [ 781.161732] do_el0_svc+0x34/0xa0 [ 781.165149] el0_svc+0x1c/0x30 [ 781.168300] el0_sync_handler+0xa8/0xb0 [ 781.172255] el0_sync+0x16c/0x180 [ 781.175764] task:kworker/4:2 state:D stack: 0 pid: 2575 ppid: 2 flags:0x00000028 [ 781.184365] Workqueue: pm pm_runtime_work [ 781.188494] Call trace: [ 781.191016] __switch_to+0xc8/0x120 [ 781.194614] __schedule+0x3d0/0x910 [ 781.198212] schedule+0x74/0x100 [ 781.201541] schedule_timeout+0x180/0x330 [ 781.205694] snd_hdac_bus_get_response+0x14c/0x190 [snd_hda_core] [ 781.211990] azx_rirb_get_response+0x48/0x150 [snd_hda_codec] [ 781.217913] azx_get_response+0x68/0x90 [snd_hda_codec] [ 781.223296] snd_hdac_bus_exec_verb_unlocked+0x194/0x270 [snd_hda_core] [ 781.230124] codec_exec_verb+0x7c/0x120 [snd_hda_codec] [ 781.235515] snd_hdac_exec_verb+0x44/0x80 [snd_hda_core] [ 781.241007] codec_read+0x68/0xa0 [snd_hda_core] [ 781.245772] snd_hdac_codec_read+0x48/0x60 [snd_hda_core] [ 781.251334] hda_set_power_state+0xe4/0x130 [snd_hda_codec] [ 781.257073] hda_call_codec_suspend+0xa8/0x120 [snd_hda_codec] [ 781.263082] hda_codec_runtime_suspend+0x34/0xe0 [snd_hda_codec] [ 781.269273] __rpm_callback+0xfc/0x1a0 [ 781.273140] rpm_callback+0x34/0xa0 [ 781.276739] rpm_suspend+0xe0/0x620 [ 781.280343] pm_runtime_work+0xbc/0xc0 [ 781.284212] process_one_work+0x1c0/0x4a0 [ 781.288347] worker_thread+0x50/0x420 [ 781.292119] kthread+0x148/0x170 [ 781.295449] ret_from_fork+0x10/0x18 [ 781.299140] watchdog: ====For debug only: End Printing Blocked Tasks==== [ 781.307734] Kernel panic - not syncing: softlockup: hung tasks [ 781.313754] CPU: 0 PID: 2600 Comm: cat Tainted: G OEL 5.10.104-tegra #145 [ 781.321902] Hardware name: Unknown Jetson AGX Orin/Jetson AGX Orin, BIOS 202210.0-Unknown 01/11/2023 [ 781.331308] Call trace: [ 781.333831] dump_backtrace+0x0/0x1d0 [ 781.337600] show_stack+0x2c/0x40 [ 781.341018] dump_stack+0xd8/0x138 [ 781.344523] panic+0x178/0x380 [ 781.347674] watchdog_timer_fn+0x26c/0x2b0 [ 781.351894] __hrtimer_run_queues+0x144/0x370 [ 781.356382] hrtimer_interrupt+0xec/0x240 [ 781.360523] arch_timer_handler_phys+0x3c/0x50 [ 781.365098] handle_percpu_devid_irq+0x8c/0x280 [ 781.369767] generic_handle_irq+0x3c/0x60 [ 781.373900] __handle_domain_irq+0x6c/0xc0 [ 781.378120] gic_handle_irq+0x64/0x130 [ 781.381987] el1_irq+0xd0/0x180 [ 781.385225] __clean_dcache_area_poc+0x34/0x38 [ 781.389800] dma_sync_single_for_device+0xb8/0x110 [ 781.394732] __arm_lpae_sync_pgtable.isra.0+0x6c/0xb0 [ 781.399929] __arm_lpae_dma_sync+0x1dc/0x200 [ 781.404330] __arm_lpae_dma_sync+0x148/0x200 [ 781.408729] __arm_lpae_dma_sync+0x148/0x200 [ 781.413127] __arm_lpae_dma_sync+0x148/0x200 [ 781.417527] arm_lpae_dma_sync+0x50/0x80 [ 781.421570] arm_smmu_dma_sync+0x3c/0x60 [ 781.425612] __iommu_unmap+0xd0/0x1d0 [ 781.429383] iommu_unmap_fast+0x3c/0x50 [ 781.433337] __iommu_dma_unmap+0x88/0x110 [ 781.437471] iommu_dma_unmap_page+0x4c/0x80 [ 781.441778] dma_unmap_page_attrs+0x60/0x200 [ 781.446181] usb_hcd_unmap_urb_for_dma+0x6c/0x110 [ 781.451022] unmap_urb_for_dma+0x44/0x50 [ 781.455064] __usb_hcd_giveback_urb+0x40/0x140 [ 781.459640] usb_giveback_urb_bh+0xb4/0x110 [ 781.463950] tasklet_action_common.isra.0+0x138/0x180 [ 781.469154] tasklet_hi_action+0x2c/0x40 [ 781.473190] __do_softirq+0x138/0x3e0 [ 781.476962] irq_exit+0xc0/0xe0 [ 781.480200] __handle_domain_irq+0x70/0xc0 [ 781.484417] gic_handle_irq+0x64/0x130 [ 781.488284] el1_irq+0xd0/0x180 [ 781.491521] preempt_schedule_irq+0x44/0x90 [ 781.495828] arm64_preempt_schedule_irq+0x18/0x20 [ 781.500674] el1_irq+0xe8/0x180 [ 781.503912] write_test+0x228/0x390 [ 781.507511] seq_read_iter+0x1cc/0x490 [ 781.511378] seq_read+0xf0/0x150 [ 781.514706] full_proxy_read+0x68/0xa0 [ 781.518573] vfs_read+0xb0/0x1c0 [ 781.521902] ksys_read+0x78/0x100 [ 781.525320] __arm64_sys_read+0x24/0x30 [ 781.529276] el0_svc_common.constprop.0+0x7c/0x1c0 [ 781.534211] do_el0_svc+0x34/0xa0 [ 781.537630] el0_svc+0x1c/0x30 [ 781.540781] el0_sync_handler+0xa8/0xb0 [ 781.544736] el0_sync+0x16c/0x180 [ 781.548162] SMP: stopping secondary CPUs [ 781.552214] Kernel Offset: 0x43c3e3020000 from 0xffff800010000000 [ 781.558494] PHYS_OFFSET: 0xffffcec200000000 [ 781.562802] CPU features: 0x0040006,4a80aa38 [ 781.567198] Memory Limit: none [ 781.585149] ---[ end Kernel panic - not syncing: softlockup: hung tasks ]---