[ 1579.589330] __ga10b__ NV_PGRAPH_PRI_FE_TPC_FS(0): 0xf7 [ 1579.594623] __ga10b__ NV_PGRAPH_PRI_CWD_GPC_TPC_ID: 0x1120213 [ 1579.600539] __ga10b__ NV_PGRAPH_PRI_CWD_SM_ID(0): 0x10306 [ 1579.606094] __ga10b__ NV_PGRAPH_PRI_FECS_CTXSW_STATUS_FE_0: 0x2 [ 1579.612188] __ga10b__ NV_PGRAPH_PRI_FECS_CTXSW_STATUS_1: 0x90 [ 1579.618113] __ga10b__ NV_PGRAPH_PRI_GPC0_GPCCS_CTXSW_STATUS_GPC_0: 0x0 [ 1579.624829] __ga10b__ NV_PGRAPH_PRI_GPC0_GPCCS_CTXSW_STATUS_1: 0x390 [ 1579.631389] __ga10b__ NV_PGRAPH_PRI_FECS_CTXSW_IDLESTATE : 0xf [ 1579.637402] __ga10b__ NV_PGRAPH_PRI_GPC0_GPCCS_CTXSW_IDLESTATE : 0xf [ 1579.643961] __ga10b__ NV_PGRAPH_PRI_FECS_CURRENT_CTX : 0xb013cdb1 [ 1579.650239] __ga10b__ NV_PGRAPH_PRI_FECS_NEW_CTX : 0xb01bdbab [ 1579.656156] __ga10b__ NV_PGRAPH_PRI_FECS_HOST_INT_ENABLE : 0x7f0003 [ 1579.662605] __ga10b__ NV_PGRAPH_PRI_FECS_HOST_INT_STATUS : 0x0 [ 1579.668614] __ga10b__ NV_PGRAPH_PRI_GPCS_ROP0_CROP_STATUS1 : 0x5f00000 [ 1579.675329] __ga10b__ NV_PGRAPH_PRI_GPCS_ROPS_CROP_STATUS1 : 0x5f00000 [ 1579.682065] __ga10b__ NV_PGRAPH_PRI_GPCS_ROP0_ZROP_STATUS : 0x0 [ 1579.688165] __ga10b__ NV_PGRAPH_PRI_GPCS_ROP0_ZROP_STATUS2 : 0x0 [ 1579.694347] __ga10b__ NV_PGRAPH_PRI_GPCS_ROP1_ZROP_STATUS: 0x0 [ 1579.700356] __ga10b__ NV_PGRAPH_PRI_GPCS_ROP1_ZROP_STATUS2: 0x0 [ 1579.706455] __ga10b__ NV_PGRAPH_PRI_GPCS_ROPS_ZROP_STATUS : 0x0 [ 1579.712546] __ga10b__ NV_PGRAPH_PRI_GPCS_ROPS_ZROP_STATUS2 : 0x0 [ 1579.718723] __ga10b__ NV_PGRAPH_PRI_GPC0_GPCCS_GPC_EXCEPTION: 0x0 [ 1579.725001] __ga10b__ NV_PGRAPH_PRI_GPC0_GPCCS_GPC_EXCEPTION_EN: 0xfc0f6004 [ 1579.732163] __ga10b__ NV_PGRAPH_PRI_GPC0_TPC0_TPCCS_TPC_EXCEPTION: 0x0 [ 1579.738872] __ga10b__ NV_PGRAPH_PRI_GPC0_TPC0_TPCCS_TPC_EXCEPTION_EN: 0x16 [ 1579.745973] __ga10b__ NV_PGRAPH_PRI_GPCS_TPCS_SMS_HWW_GLOBAL_ESR_REPORT_MASK: 0x1174 [ 1579.753960] __ga10b__ NV_PGRAPH_PRI_GPCS_TPCS_SMS_HWW_WARP_ESR_REPORT_MASK: 0xe812b60 [ 1579.762029] __ga10b__ NV_PGRAPH_PRI_GPCS_TPCS_SMS_HWW_GLOBAL_ESR: 0x0 [ 1579.768672] __ga10b__ NV_PGRAPH_PRI_GPCS_TPCS_SMS_DBGR_CONTROL0: 0x1c00 [ 1579.775496] __ga10b__ NV_PGRAPH_PRI_GPCS_TPCS_SMS_DBGR_STATUS0: 0x0 [ 1579.781942] __ga10b__ NV_PGRAPH_PRI_GPCS_TPCS_SMS_DBGR_BPT_PAUSE_MASK_0: 0x0 [ 1579.789204] __ga10b__ NV_PGRAPH_PRI_GPCS_TPCS_SMS_DBGR_BPT_PAUSE_MASK_1: 0x0 [ 1579.796471] __ga10b__ NV_PGRAPH_PRI_GPC0_TPC0_SM0_HWW_WARP_ESR: 0x0 [ 1579.802927] __ga10b__ NV_PGRAPH_PRI_GPC0_TPC0_SM0_HWW_WARP_ESR_REPORT_MASK: 0xe812b60 [ 1579.810992] __ga10b__ NV_PGRAPH_PRI_GPC0_TPC0_SM0_HWW_GLOBAL_ESR: 0x0 [ 1579.817620] __ga10b__ NV_PGRAPH_PRI_GPC0_TPC0_SM0_HWW_GLOBAL_ESR_REPORT_MASK: 0x0 [ 1579.825332] __ga10b__ NV_PGRAPH_PRI_GPC0_TPC0_SM0_DBGR_CONTROL0: 0x1c00 [ 1579.832154] __ga10b__ NV_PGRAPH_PRI_GPC0_TPC0_SM0_DBGR_STATUS0: 0x0 [ 1579.838605] __ga10b__ NV_PGRAPH_PRI_GPC0_TPC0_SM1_HWW_WARP_ESR: 0x0 [ 1579.845077] __ga10b__ NV_PGRAPH_PRI_GPC0_TPC0_SM1_HWW_WARP_ESR_REPORT_MASK: 0xe812b60 [ 1579.853145] __ga10b__ NV_PGRAPH_PRI_GPC0_TPC0_SM1_HWW_GLOBAL_ESR: 0x0 [ 1579.859783] __ga10b__ NV_PGRAPH_PRI_GPC0_TPC0_SM1_HWW_GLOBAL_ESR_REPORT_MASK: 0x0 [ 1579.867489] __ga10b__ NV_PGRAPH_PRI_GPC0_TPC0_SM1_DBGR_CONTROL0: 0x1c00 [ 1579.874310] __ga10b__ NV_PGRAPH_PRI_GPC0_TPC0_SM1_DBGR_STATUS0: 0x0 [ 1579.880776] __ga10b__ NV_PGRAPH_PRI_GPC0_TPC1_SM0_HWW_WARP_ESR: 0x0 [ 1579.887233] __ga10b__ NV_PGRAPH_PRI_GPC0_TPC1_SM0_HWW_WARP_ESR_REPORT_MASK: 0xe812b60 [ 1579.895292] __ga10b__ NV_PGRAPH_PRI_GPC0_TPC1_SM0_HWW_GLOBAL_ESR: 0x0 [ 1579.901919] __ga10b__ NV_PGRAPH_PRI_GPC0_TPC1_SM0_HWW_GLOBAL_ESR_REPORT_MASK: 0x0 [ 1579.909625] __ga10b__ NV_PGRAPH_PRI_GPC0_TPC1_SM0_DBGR_CONTROL0: 0x1c00 [ 1579.916442] __ga10b__ NV_PGRAPH_PRI_GPC0_TPC1_SM0_DBGR_STATUS0: 0x0 [ 1579.922903] __ga10b__ NV_PGRAPH_PRI_GPC0_TPC1_SM1_HWW_WARP_ESR: 0x0 [ 1579.929356] __ga10b__ NV_PGRAPH_PRI_GPC0_TPC1_SM1_HWW_WARP_ESR_REPORT_MASK: 0xe812b60 [ 1579.937423] __ga10b__ NV_PGRAPH_PRI_GPC0_TPC1_SM1_HWW_GLOBAL_ESR: 0x0 [ 1579.944066] __ga10b__ NV_PGRAPH_PRI_GPC0_TPC1_SM1_HWW_GLOBAL_ESR_REPORT_MASK: 0x1174 [ 1579.952045] __ga10b__ NV_PGRAPH_PRI_GPC0_TPC1_SM1_DBGR_CONTROL0: 0x1c00 [ 1579.958868] __ga10b__ NV_PGRAPH_PRI_GPC0_TPC1_SM1_DBGR_STATUS0: 0x0 [ 1579.965320] __ga10b__ NV_PGRAPH_PRI_GPC0_TPC2_SM0_HWW_WARP_ESR: 0x0 [ 1579.971776] __ga10b__ NV_PGRAPH_PRI_GPC0_TPC2_SM0_HWW_WARP_ESR_REPORT_MASK: 0xe812b60 [ 1579.979928] __ga10b__ NV_PGRAPH_PRI_GPC0_TPC2_SM0_HWW_GLOBAL_ESR: 0x0 [ 1579.986572] __ga10b__ NV_PGRAPH_PRI_GPC0_TPC2_SM0_HWW_GLOBAL_ESR_REPORT_MASK: 0x1174 [ 1579.994558] __ga10b__ NV_PGRAPH_PRI_GPC0_TPC2_SM0_DBGR_CONTROL0: 0x1c00 [ 1580.001386] __ga10b__ NV_PGRAPH_PRI_GPC0_TPC2_SM0_DBGR_STATUS0: 0x0 [ 1580.007832] __ga10b__ NV_PGRAPH_PRI_GPC0_TPC2_SM1_HWW_WARP_ESR: 0x0 [ 1580.014278] __ga10b__ NV_PGRAPH_PRI_GPC0_TPC2_SM1_HWW_WARP_ESR_REPORT_MASK: 0xe812b60 [ 1580.022343] __ga10b__ NV_PGRAPH_PRI_GPC0_TPC2_SM1_HWW_GLOBAL_ESR: 0x0 [ 1580.028981] __ga10b__ NV_PGRAPH_PRI_GPC0_TPC2_SM1_HWW_GLOBAL_ESR_REPORT_MASK: 0x1174 [ 1580.036972] __ga10b__ NV_PGRAPH_PRI_GPC0_TPC2_SM1_DBGR_CONTROL0: 0x1c00 [ 1580.043798] __ga10b__ NV_PGRAPH_PRI_GPC0_TPC2_SM1_DBGR_STATUS0: 0x0 [ 1580.050255] __ga10b__ NV_PGRAPH_PRI_GPC1_TPC0_SM0_HWW_WARP_ESR: 0x0 [ 1580.056714] __ga10b__ NV_PGRAPH_PRI_GPC1_TPC0_SM0_HWW_WARP_ESR_REPORT_MASK: 0x0 [ 1580.064247] __ga10b__ NV_PGRAPH_PRI_GPC1_TPC0_SM0_HWW_GLOBAL_ESR: 0x0 [ 1580.070893] __ga10b__ NV_PGRAPH_PRI_GPC1_TPC0_SM0_HWW_GLOBAL_ESR_REPORT_MASK: 0x1174 [ 1580.078871] __ga10b__ NV_PGRAPH_PRI_GPC1_TPC0_SM0_DBGR_CONTROL0: 0x1c00 [ 1580.085696] __ga10b__ NV_PGRAPH_PRI_GPC1_TPC0_SM0_DBGR_STATUS0: 0x0 [ 1580.092154] __ga10b__ NV_PGRAPH_PRI_GPC1_TPC0_SM1_HWW_WARP_ESR: 0x0 [ 1580.098611] __ga10b__ NV_PGRAPH_PRI_GPC1_TPC0_SM1_HWW_WARP_ESR_REPORT_MASK: 0xe812b60 [ 1580.106677] __ga10b__ NV_PGRAPH_PRI_GPC1_TPC0_SM1_HWW_GLOBAL_ESR: 0x0 [ 1580.113325] __ga10b__ NV_PGRAPH_PRI_GPC1_TPC0_SM1_HWW_GLOBAL_ESR_REPORT_MASK: 0x0 [ 1580.121034] __ga10b__ NV_PGRAPH_PRI_GPC1_TPC0_SM1_DBGR_CONTROL0: 0x1c00 [ 1580.127849] __ga10b__ NV_PGRAPH_PRI_GPC1_TPC0_SM1_DBGR_STATUS0: 0x0 [ 1580.134297] __ga10b__ NV_PGRAPH_PRI_GPC1_TPC1_SM0_HWW_WARP_ESR: 0x0 [ 1580.140746] __ga10b__ NV_PGRAPH_PRI_GPC1_TPC1_SM0_HWW_WARP_ESR_REPORT_MASK: 0xe812b60 [ 1580.148819] __ga10b__ NV_PGRAPH_PRI_GPC1_TPC1_SM0_HWW_GLOBAL_ESR: 0x0 [ 1580.155449] __ga10b__ NV_PGRAPH_PRI_GPC1_TPC1_SM0_HWW_GLOBAL_ESR_REPORT_MASK: 0x0 [ 1580.163158] __ga10b__ NV_PGRAPH_PRI_GPC1_TPC1_SM0_DBGR_CONTROL0: 0x1c00 [ 1580.169985] __ga10b__ NV_PGRAPH_PRI_GPC1_TPC1_SM0_DBGR_STATUS0: 0x0 [ 1580.176434] __ga10b__ NV_PGRAPH_PRI_GPC1_TPC1_SM1_HWW_WARP_ESR: 0x0 [ 1580.182883] __ga10b__ NV_PGRAPH_PRI_GPC1_TPC1_SM1_HWW_WARP_ESR_REPORT_MASK: 0x0 [ 1580.190412] __ga10b__ NV_PGRAPH_PRI_GPC1_TPC1_SM1_HWW_GLOBAL_ESR: 0x0 [ 1580.197042] __ga10b__ NV_PGRAPH_PRI_GPC1_TPC1_SM1_HWW_GLOBAL_ESR_REPORT_MASK: 0x1174 [ 1580.205031] __ga10b__ NV_PGRAPH_PRI_GPC1_TPC1_SM1_DBGR_CONTROL0: 0x1c00 [ 1580.211860] __ga10b__ NV_PGRAPH_PRI_GPC1_TPC1_SM1_DBGR_STATUS0: 0x0 [ 1580.218309] __ga10b__ NV_PGRAPH_PRI_GPC1_TPC2_SM0_HWW_WARP_ESR: 0x0 [ 1580.224894] __ga10b__ NV_PGRAPH_PRI_GPC1_TPC2_SM0_HWW_WARP_ESR_REPORT_MASK: 0xe812b60 [ 1580.232985] __ga10b__ NV_PGRAPH_PRI_GPC1_TPC2_SM0_HWW_GLOBAL_ESR: 0x0 [ 1580.239633] __ga10b__ NV_PGRAPH_PRI_GPC1_TPC2_SM0_HWW_GLOBAL_ESR_REPORT_MASK: 0x1174 [ 1580.247622] __ga10b__ NV_PGRAPH_PRI_GPC1_TPC2_SM0_DBGR_CONTROL0: 0x1c00 [ 1580.254450] __ga10b__ NV_PGRAPH_PRI_GPC1_TPC2_SM0_DBGR_STATUS0: 0x0 [ 1580.260914] __ga10b__ NV_PGRAPH_PRI_GPC1_TPC2_SM1_HWW_WARP_ESR: 0x0 [ 1580.267363] __ga10b__ NV_PGRAPH_PRI_GPC1_TPC2_SM1_HWW_WARP_ESR_REPORT_MASK: 0xe812b60 [ 1580.275436] __ga10b__ NV_PGRAPH_PRI_GPC1_TPC2_SM1_HWW_GLOBAL_ESR: 0x0 [ 1580.282078] __ga10b__ NV_PGRAPH_PRI_GPC1_TPC2_SM1_HWW_GLOBAL_ESR_REPORT_MASK: 0x1174 [ 1580.290060] __ga10b__ NV_PGRAPH_PRI_GPC1_TPC2_SM1_DBGR_CONTROL0: 0x1c00 [ 1580.296872] __ga10b__ NV_PGRAPH_PRI_GPC1_TPC2_SM1_DBGR_STATUS0: 0x0 [ 1580.303330] __ga10b__ NV_PGRAPH_PRI_GPC1_TPC3_SM0_HWW_WARP_ESR: 0x0 [ 1580.309785] __ga10b__ NV_PGRAPH_PRI_GPC1_TPC3_SM0_HWW_WARP_ESR_REPORT_MASK: 0xe812b60 [ 1580.317858] __ga10b__ NV_PGRAPH_PRI_GPC1_TPC3_SM0_HWW_GLOBAL_ESR: 0x0 [ 1580.324507] __ga10b__ NV_PGRAPH_PRI_GPC1_TPC3_SM0_HWW_GLOBAL_ESR_REPORT_MASK: 0x1174 [ 1580.332489] __ga10b__ NV_PGRAPH_PRI_GPC1_TPC3_SM0_DBGR_CONTROL0: 0x1c00 [ 1580.339314] __ga10b__ NV_PGRAPH_PRI_GPC1_TPC3_SM0_DBGR_STATUS0: 0x0 [ 1580.345780] __ga10b__ NV_PGRAPH_PRI_GPC1_TPC3_SM1_HWW_WARP_ESR: 0x0 [ 1580.352244] __ga10b__ NV_PGRAPH_PRI_GPC1_TPC3_SM1_HWW_WARP_ESR_REPORT_MASK: 0xe812b60 [ 1580.360323] __ga10b__ NV_PGRAPH_PRI_GPC1_TPC3_SM1_HWW_GLOBAL_ESR: 0x0 [ 1580.366958] __ga10b__ NV_PGRAPH_PRI_GPC1_TPC3_SM1_HWW_GLOBAL_ESR_REPORT_MASK: 0x1174 [ 1580.374939] __ga10b__ NV_PGRAPH_PRI_GPC1_TPC3_SM1_DBGR_CONTROL0: 0x1c00 [ 1580.381768] __ga10b__ NV_PGRAPH_PRI_GPC1_TPC3_SM1_DBGR_STATUS0: 0x0 [ 1580.388559] nvgpu: 17000000.ga10b nvgpu_tsg_force_reset_ch:469 [ERR] chid: 509 is not bound to tsg [ 1583.615522] cpufreq: cpu4,cur:1983000,set:2188800,set ndiv:171 [ 1586.264747] cpufreq: cpu4,cur:2073000,set:2188800,set ndiv:171 [ 1587.310308] cpufreq: cpu4,cur:2065000,set:2188800,set ndiv:171 [ 1588.214897] nvgpu: 17000000.ga10b ga10b_pbdma_handle_intr_0_legacy:437 [ERR] semaphore acquire timeout! [ 1588.225024] __ga10b__ Channel Status - chip ga10b [ 1588.225025] __ga10b__ --------------------------- [ 1588.229911] __ga10b__ 492-ga10b, TSG: 19, pid 3008, refs: 2, deterministic: no, domain name: (default) [ 1588.234786] __ga10b__ channel status: in use idle not busy [ 1588.244387] __ga10b__ RAMFC: TOP: 80000020068b2878 PUT: 0020068b2878 GET: 0020068b2878 FETCH: 000000000000 HEADER: 2140006c COUNT: 00000000 SEMAPHORE: addr 0020068a0000 payload 0000000000000000 execute 00100001 [ 1588.250153] __ga10b__ [ 1588.269449] __ga10b__ 493-ga10b, TSG: 18, pid 3008, refs: 2, deterministic: no, domain name: (default) [ 1588.271979] __ga10b__ channel status: in use idle not busy [ 1588.281595] __ga10b__ RAMFC: TOP: 800000200607ff58 PUT: 00200607ff58 GET: 00200607ff58 FETCH: 000000000000 HEADER: 2140006c COUNT: 00000000 SEMAPHORE: addr 0020063c0000 payload 0000000000000000 execute 00100001 [ 1588.287374] __ga10b__ [ 1588.306643] __ga10b__ 494-ga10b, TSG: 17, pid 3008, refs: 2, deterministic: no, domain name: (default) [ 1588.309202] __ga10b__ channel status: in use idle not busy [ 1588.318814] __ga10b__ RAMFC: TOP: 8000002006066d10 PUT: 002006066d10 GET: 002006066d10 FETCH: 000000000000 HEADER: 2140006c COUNT: 00000000 SEMAPHORE: addr 002006010000 payload 0000000000000000 execute 00100001 [ 1588.324596] __ga10b__ [ 1588.343877] __ga10b__ 495-ga10b, TSG: 16, pid 3008, refs: 2, deterministic: no, domain name: (default) [ 1588.346412] __ga10b__ channel status: in use idle not busy [ 1588.356003] __ga10b__ RAMFC: TOP: 8000002006002a58 PUT: 002006002a58 GET: 002006002a58 FETCH: 000000000000 HEADER: 2140006c COUNT: 00000000 SEMAPHORE: addr 002005d30000 payload 0000000000000000 execute 00100001 [ 1588.361745] __ga10b__ [ 1588.380981] __ga10b__ 496-ga10b, TSG: 15, pid 3090, refs: 2, deterministic: no, domain name: (default) [ 1588.383502] __ga10b__ channel status: in use idle not busy [ 1588.393090] __ga10b__ RAMFC: TOP: 80000020040080a0 PUT: 0020040080a0 GET: 0020040080a0 FETCH: 000000000000 HEADER: 2140006c COUNT: 00000000 SEMAPHORE: addr 002004010000 payload 0000000000000000 execute 00100001 [ 1588.398833] __ga10b__ [ 1588.418079] __ga10b__ 497-ga10b, TSG: 14, pid 3008, refs: 6, deterministic: no, domain name: (default) [ 1588.420599] __ga10b__ channel status: in use pending, acquire_fail busy [ 1588.430180] __ga10b__ RAMFC: TOP: 8000001ffac681d4 PUT: 001ffac681e8 GET: 001ffac681d4 FETCH: 000000000000 HEADER: 2140006c COUNT: 11110000 SEMAPHORE: addr 002000040000 payload 0000000000018f35 execute 00081003 [ 1588.437079] __ga10b__ [ 1588.456323] __ga10b__ 498-ga10b, TSG: 13, pid 3008, refs: 2, deterministic: no, domain name: (default) [ 1588.458843] __ga10b__ channel status: in use idle not busy [ 1588.468429] __ga10b__ RAMFC: TOP: 8000002005a30168 PUT: 002005a30168 GET: 002005a30168 FETCH: 000000000000 HEADER: 2140006c COUNT: 00000000 SEMAPHORE: addr 002005a20000 payload 0000000000000000 execute 00100001 [ 1588.474167] __ga10b__ [ 1588.493413] __ga10b__ 499-ga10b, TSG: 12, pid 3008, refs: 2, deterministic: no, domain name: (default) [ 1588.495950] __ga10b__ channel status: in use idle not busy [ 1588.505550] __ga10b__ RAMFC: TOP: 80000020051f8208 PUT: 0020051f8208 GET: 0020051f8208 FETCH: 000000000000 HEADER: 2140006c COUNT: 00000000 SEMAPHORE: addr 002005640000 payload 0000000000000000 execute 00100001 [ 1588.511302] __ga10b__ [ 1588.530547] __ga10b__ 500-ga10b, TSG: 11, pid 3008, refs: 2, deterministic: no, domain name: (default) [ 1588.533079] __ga10b__ channel status: in use idle not busy [ 1588.542658] __ga10b__ RAMFC: TOP: 8000002005198230 PUT: 002005198230 GET: 002005198230 FETCH: 000000000000 HEADER: 2140006c COUNT: 00000000 SEMAPHORE: addr 0020051e0000 payload 0000000000000000 execute 00100001 [ 1588.548403] __ga10b__ [ 1588.567641] __ga10b__ 501-ga10b, TSG: 10, pid 3008, refs: 2, deterministic: no, domain name: (default) [ 1588.570178] __ga10b__ channel status: in use idle not busy [ 1588.579757] __ga10b__ RAMFC: TOP: 8000002005180208 PUT: 002005180208 GET: 002005180208 FETCH: 000000000000 HEADER: 2140006c COUNT: 00000000 SEMAPHORE: addr 002004eb0000 payload 0000000000000000 execute 00100001 [ 1588.585493] __ga10b__ [ 1588.604744] __ga10b__ 502-ga10b, TSG: 9, pid 3008, refs: 2, deterministic: no, domain name: (default) [ 1588.607280] __ga10b__ channel status: in use idle not busy [ 1588.616778] __ga10b__ RAMFC: TOP: 8000002004e88230 PUT: 002004e88230 GET: 002004e88230 FETCH: 000000000000 HEADER: 2140006c COUNT: 00000000 SEMAPHORE: addr 002004e90000 payload 0000000000000000 execute 00100001 [ 1588.622519] __ga10b__ [ 1588.641771] __ga10b__ 503-ga10b, TSG: 8, pid 3008, refs: 2, deterministic: no, domain name: (default) [ 1588.644295] __ga10b__ channel status: in use idle not busy [ 1588.653786] __ga10b__ RAMFC: TOP: 8000002004bb0208 PUT: 002004bb0208 GET: 002004bb0208 FETCH: 000000000000 HEADER: 2140006c COUNT: 00000000 SEMAPHORE: addr 002004ba0000 payload 0000000000000000 execute 00100001 [ 1588.659526] __ga10b__ [ 1588.678765] __ga10b__ 504-ga10b, TSG: 7, pid 3008, refs: 2, deterministic: no, domain name: (default) [ 1588.681308] __ga10b__ channel status: in use idle not busy [ 1588.690807] __ga10b__ RAMFC: TOP: 80000020046d0230 PUT: 0020046d0230 GET: 0020046d0230 FETCH: 000000000000 HEADER: 2140006c COUNT: 00000000 SEMAPHORE: addr 0020046c0000 payload 0000000000000000 execute 00100001 [ 1588.696547] __ga10b__ [ 1588.715789] __ga10b__ 505-ga10b, TSG: 6, pid 3008, refs: 2, deterministic: no, domain name: (default) [ 1588.718315] __ga10b__ channel status: in use idle not busy [ 1588.727814] __ga10b__ RAMFC: TOP: 8000002004318050 PUT: 002004318050 GET: 002004318050 FETCH: 000000000000 HEADER: 2140006c COUNT: 00000000 SEMAPHORE: addr 002004360000 payload 0000000000000000 execute 00100001 [ 1588.733553] __ga10b__ [ 1588.752790] __ga10b__ 506-ga10b, TSG: 5, pid 3008, refs: 2, deterministic: no, domain name: (default) [ 1588.755323] __ga10b__ channel status: in use idle not busy [ 1588.764818] __ga10b__ RAMFC: TOP: 8000002004305230 PUT: 002004305230 GET: 002004305230 FETCH: 000000000000 HEADER: 2140006c COUNT: 00000000 SEMAPHORE: addr 002004030000 payload 0000000000000000 execute 00100001 [ 1588.770553] __ga10b__ [ 1588.789784] __ga10b__ 507-ga10b, TSG: 4, pid 3008, refs: 2, deterministic: no, domain name: (default) [ 1588.792324] __ga10b__ channel status: in use idle not busy [ 1588.801839] __ga10b__ RAMFC: TOP: 80000020040080a0 PUT: 0020040080a0 GET: 0020040080a0 FETCH: 000000000000 HEADER: 2140006c COUNT: 00000000 SEMAPHORE: addr 002004010000 payload 0000000000000000 execute 00100001 [ 1588.807578] __ga10b__ [ 1588.826830] __ga10b__ 508-ga10b, TSG: 3, pid 2755, refs: 2, deterministic: no, domain name: (default) [ 1588.829361] __ga10b__ channel status: in use idle not busy [ 1588.838858] __ga10b__ RAMFC: TOP: 80000020040080a0 PUT: 0020040080a0 GET: 0020040080a0 FETCH: 000000000000 HEADER: 2140006c COUNT: 00000000 SEMAPHORE: addr 002004010000 payload 0000000000000000 execute 00100001 [ 1588.844599] __ga10b__ [ 1588.863840] __ga10b__ 509-ga10b, TSG: 2, pid 2334, refs: 5, deterministic: no, domain name: (default) [ 1588.866364] __ga10b__ channel status: in use on_pbdma, pbdma_busy busy [ 1588.875865] __ga10b__ RAMFC: TOP: 8000001fffda065c PUT: 001fffda0670 GET: 001fffda065c FETCH: 000000000000 HEADER: 2140006c COUNT: 11110000 SEMAPHORE: addr 002000110000 payload 0000000000013168 execute 00081003 [ 1588.882686] __ga10b__ [ 1588.901936] __ga10b__ 510-ga10b, TSG: 1, pid 2080, refs: 2, deterministic: no, domain name: (default) [ 1588.904473] __ga10b__ channel status: in use idle not busy [ 1588.913977] __ga10b__ RAMFC: TOP: 8000002004033780 PUT: 002004033780 GET: 002004033780 FETCH: 000000000000 HEADER: 2140006c COUNT: 00000000 SEMAPHORE: addr 002004320000 payload 0000000000000000 execute 00000001 [ 1588.919716] __ga10b__ [ 1588.938954] __ga10b__ 511-ga10b, TSG: 0, pid 2080, refs: 3, deterministic: no, domain name: (default) [ 1588.941485] __ga10b__ channel status: in use on_eng, pending busy [ 1588.950987] __ga10b__ RAMFC: TOP: 800000200405ede8 PUT: 00200405ede8 GET: 00200405ede8 FETCH: 000000000000 HEADER: 2140006c COUNT: 00000000 SEMAPHORE: addr 002004020000 payload 0000000000000000 execute 00100001 [ 1588.957351] __ga10b__ [ 1588.976591] __ga10b__ PBDMA Status - chip ga10b [ 1588.979128] __ga10b__ ------------------------- [ 1588.983802] __ga10b__ pbdma 0: [ 1588.988475] __ga10b__ id: 2 - [tsg] next_id: - -1 [channel] | status: valid [ 1588.991629] __ga10b__ PBDMA_PUT 0000001fffda0670 PBDMA_GET 0000001fffda065c [ 1588.999347] __ga10b__ GP_PUT 000006d3 GP_GET 000006cb FETCH 000006cb HEADER 2140006c [ 1589.006707] __ga10b__ HDR 2001001b SHADOW0 0405edc0 SHADOW1 00002820 [ 1589.015573] __ga10b__ pbdma 1: [ 1589.023026] __ga10b__ id: -1 - [channel] next_id: - -1 [channel] | status: invalid [ 1589.026177] __ga10b__ PBDMA_PUT 00000020046d00a0 PBDMA_GET 00000020046d00a0 [ 1589.034155] __ga10b__ GP_PUT 0000000c GP_GET 0000000c FETCH 0000000c HEADER 2140006c [ 1589.041526] __ga10b__ HDR 2001001b SHADOW0 046d0078 SHADOW1 00002820 [ 1589.050399] __ga10b__ pbdma 2: [ 1589.057841] __ga10b__ id: -1 - [channel] next_id: - -1 [channel] | status: invalid [ 1589.060998] __ga10b__ PBDMA_PUT 0000002004318118 PBDMA_GET 0000002004318118 [ 1589.068983] __ga10b__ GP_PUT 0000001c GP_GET 0000001c FETCH 0000001c HEADER 2140006c [ 1589.076342] __ga10b__ HDR 2001001b SHADOW0 043180f0 SHADOW1 00002820 [ 1589.085213] __ga10b__ pbdma 3: [ 1589.092659] __ga10b__ id: -1 - [channel] next_id: - -1 [channel] | status: invalid [ 1589.095817] __ga10b__ PBDMA_PUT 00000067ef7f6fec PBDMA_GET 000000cd20b86794 [ 1589.103808] __ga10b__ GP_PUT 00000000 GP_GET 7e1826eb FETCH 00000000 HEADER e1955910 [ 1589.111167] __ga10b__ HDR 5f129d84 SHADOW0 61804464 SHADOW1 be122349 [ 1589.120034] __ga10b__ pbdma 4: [ 1589.127487] __ga10b__ id: -1 - [channel] next_id: - -1 [channel] | status: invalid [ 1589.130645] __ga10b__ PBDMA_PUT 0000000db7213608 PBDMA_GET 0000000db7213608 [ 1589.138630] __ga10b__ GP_PUT 00000000 GP_GET 6b8eb69b FETCH 00000000 HEADER 8086a79c [ 1589.145993] __ga10b__ HDR 145ec4b0 SHADOW0 bdc529ee SHADOW1 f4b979a2 [ 1589.154861] __ga10b__ pbdma 5: [ 1589.162321] __ga10b__ id: -1 - [channel] next_id: - -1 [channel] | status: invalid [ 1589.165473] __ga10b__ PBDMA_PUT 0000002004ea8118 PBDMA_GET 0000002004ea8118 [ 1589.173461] __ga10b__ GP_PUT 0000001c GP_GET 0000001c FETCH 0000001c HEADER 2140006c [ 1589.180820] __ga10b__ HDR 2001001b SHADOW0 04ea80f0 SHADOW1 00002820 [ 1589.189689] __ga10b__ [ 1589.197136] __ga10b__ ga10b eng 0: [ 1589.199659] __ga10b__ id: 0 (tsg), next_id: -1 (channel), ctx status: valid [ 1589.203257] __ga10b__ [ 1589.210526] __ga10b__ ga10b eng 1: [ 1589.213064] __ga10b__ id: 0 (tsg), next_id: -1 (channel), ctx status: valid [ 1589.216665] __ga10b__ [ 1589.223917] __ga10b__ ga10b eng 2: [ 1589.226434] __ga10b__ id: -1 (channel), next_id: -1 (channel), ctx status: invalid [ 1589.230033] __ga10b__ [ 1589.237930] __ga10b__ ga10b eng 3: [ 1589.240450] __ga10b__ id: -1 (channel), next_id: -1 (channel), ctx status: invalid [ 1589.244050] __ga10b__ [ 1589.251952] __ga10b__ ga10b eng 4: [ 1589.254473] __ga10b__ id: -1 (channel), next_id: -1 (channel), ctx status: invalid [ 1589.258078] __ga10b__ [ 1589.265980] __ga10b__ ga10b eng 5: [ 1589.268512] __ga10b__ id: -1 (channel), next_id: -1 (channel), ctx status: invalid [ 1589.272110] __ga10b__ [ 1589.279736] nvgpu: 17000000.ga10b nvgpu_channel_recover_from_wdt:107 [ERR] Job on channel 509 timed out [ 1589.279997] __ga10b__ [ 1589.283306] __ga10b__ NV_PGRAPH_STATUS: 0x0 [ 1589.292638] nvgpu: 17000000.ga10b report_pbdma_error:330 [ERR] pbdma_intr_0(0)= 0x04000000 [ 1589.292644] nvgpu: 17000000.ga10b nvgpu_cic_mon_report_err_safety_services:55 [ERR] Error reporting is not supported in this platform [ 1589.295159] __ga10b__ NV_PGRAPH_STATUS1: 0x0 [ 1589.295162] __ga10b__ NV_PGRAPH_ENGINE_STATUS: 0x0 [ 1589.299476] nvgpu: 17000000.ga10b nvgpu_set_err_notifier_locked:149 [ERR] error notifier set to 24 for ch 509 [ 1589.309593] __ga10b__ NV_PGRAPH_GRFIFO_STATUS : 0x1010001 [ 1589.347848] __ga10b__ NV_PGRAPH_GRFIFO_CONTROL : 0x10001 [ 1589.353320] __ga10b__ NV_PGRAPH_PRI_FECS_HOST_INT_STATUS : 0x0 [ 1589.359332] __ga10b__ NV_PGRAPH_EXCEPTION : 0x0 [ 1589.364086] __ga10b__ NV_PGRAPH_FECS_INTR : 0x0 [ 1589.368852] __ga10b__ NV_PFIFO_ENGINE_STATUS(GR) : 0x102010 [ 1589.374598] __ga10b__ NV_PGRAPH_ACTIVITY0: 0x0 [ 1589.379178] __ga10b__ NV_PGRAPH_ACTIVITY1: 0x1000 [ 1589.384025] __ga10b__ NV_PGRAPH_ACTIVITY4: 0x0 [ 1589.388606] __ga10b__ NV_PGRAPH_PRI_SKED_ACTIVITY: 0x0 [ 1589.393903] __ga10b__ NV_PGRAPH_PRI_GPC0_GPCCS_GPC_ACTIVITY0: 0x0 [ 1589.400244] __ga10b__ NV_PGRAPH_PRI_GPC0_GPCCS_GPC_ACTIVITY1: 0x0 [ 1589.406545] __ga10b__ NV_PGRAPH_PRI_GPC0_GPCCS_GPC_ACTIVITY2: 0x0 [ 1589.412826] __ga10b__ NV_PGRAPH_PRI_GPC0_GPCCS_GPC_ACTIVITY3: 0x0 [ 1589.419110] __ga10b__ NV_PGRAPH_PRI_GPC0_GPCCS_GPC_ACTIVITY4: 0x0 [ 1589.425390] __ga10b__ NV_PGRAPH_PRI_GPC0_TPC0_TPCCS_TPC_ACTIVITY0: 0x0 [ 1589.432108] __ga10b__ NV_PGRAPH_PRI_GPC0_TPC1_TPCCS_TPC_ACTIVITY0: 0x0 [ 1589.438824] __ga10b__ NV_PGRAPH_PRI_GPC0_TPC2_TPCCS_TPC_ACTIVITY0: 0x0 [ 1589.445560] __ga10b__ NV_PGRAPH_PRI_GPCS_GPCCS_GPC_ACTIVITY0: 0x0 [ 1589.451853] __ga10b__ NV_PGRAPH_PRI_GPCS_GPCCS_GPC_ACTIVITY1: 0x0 [ 1589.458131] __ga10b__ NV_PGRAPH_PRI_GPCS_GPCCS_GPC_ACTIVITY2: 0x0 [ 1589.464413] __ga10b__ NV_PGRAPH_PRI_GPCS_GPCCS_GPC_ACTIVITY3: 0x0 [ 1589.470686] __ga10b__ NV_PGRAPH_PRI_GPCS_GPCCS_GPC_ACTIVITY4: 0x0 [ 1589.476967] __ga10b__ NV_PGRAPH_PRI_GPCS_TPCS_TPCCS_TPC_ACTIVITY0: 0x0 [ 1589.483683] __ga10b__ NV_PGRAPH_PRI_DS_MPIPE_STATUS: 0x0 [ 1589.489162] __ga10b__ NV_PGRAPH_PRI_FE_GO_IDLE_TIMEOUT : 0x1800 [ 1589.495265] __ga10b__ NV_PGRAPH_PRI_FE_GO_IDLE_INFO : 0x1000700 [ 1589.501365] __ga10b__ NV_PGRAPH_PRI_GPC0_TPC0_TEX_M_TEX_SUBUNITS_STATUS: 0x0 [ 1589.508629] __ga10b__ NV_PGRAPH_PRI_CWD_FS: 0x702 [ 1589.513480] __ga10b__ NV_PGRAPH_PRI_FE_TPC_FS(0): 0xf7 [ 1589.520272] __ga10b__ NV_PGRAPH_PRI_CWD_GPC_TPC_ID: 0x1120213 [ 1589.526211] __ga10b__ NV_PGRAPH_PRI_CWD_SM_ID(0): 0x10306 [ 1589.531783] __ga10b__ NV_PGRAPH_PRI_FECS_CTXSW_STATUS_FE_0: 0x3 [ 1589.537885] __ga10b__ NV_PGRAPH_PRI_FECS_CTXSW_STATUS_1: 0x10b4 [ 1589.543974] __ga10b__ NV_PGRAPH_PRI_GPC0_GPCCS_CTXSW_STATUS_GPC_0: 0x0 [ 1589.550706] __ga10b__ NV_PGRAPH_PRI_GPC0_GPCCS_CTXSW_STATUS_1: 0x390 [ 1589.557253] __ga10b__ NV_PGRAPH_PRI_FECS_CTXSW_IDLESTATE : 0xf [ 1589.563260] __ga10b__ NV_PGRAPH_PRI_GPC0_GPCCS_CTXSW_IDLESTATE : 0xf [ 1589.569809] __ga10b__ NV_PGRAPH_PRI_FECS_CURRENT_CTX : 0xb01b4fc1 [ 1589.576095] __ga10b__ NV_PGRAPH_PRI_FECS_NEW_CTX : 0x3013cdb1 [ 1589.582014] __ga10b__ NV_PGRAPH_PRI_FECS_HOST_INT_ENABLE : 0x7f0003 [ 1589.588470] __ga10b__ NV_PGRAPH_PRI_FECS_HOST_INT_STATUS : 0x0 [ 1589.594482] __ga10b__ NV_PGRAPH_PRI_GPCS_ROP0_CROP_STATUS1 : 0x700000 [ 1589.601116] __ga10b__ NV_PGRAPH_PRI_GPCS_ROPS_CROP_STATUS1 : 0x5f00000 [ 1589.607852] __ga10b__ NV_PGRAPH_PRI_GPCS_ROP0_ZROP_STATUS : 0x0 [ 1589.613942] __ga10b__ NV_PGRAPH_PRI_GPCS_ROP0_ZROP_STATUS2 : 0x0 [ 1589.620120] __ga10b__ NV_PGRAPH_PRI_GPCS_ROP1_ZROP_STATUS: 0x0 [ 1589.626147] __ga10b__ NV_PGRAPH_PRI_GPCS_ROP1_ZROP_STATUS2: 0x0 [ 1589.632240] __ga10b__ NV_PGRAPH_PRI_GPCS_ROPS_ZROP_STATUS : 0x0 [ 1589.638577] __ga10b__ NV_PGRAPH_PRI_GPCS_ROPS_ZROP_STATUS2 : 0x0 [ 1589.644787] __ga10b__ NV_PGRAPH_PRI_GPC0_GPCCS_GPC_EXCEPTION: 0x0 [ 1589.651076] __ga10b__ NV_PGRAPH_PRI_GPC0_GPCCS_GPC_EXCEPTION_EN: 0xfc0f6004 [ 1589.658255] __ga10b__ NV_PGRAPH_PRI_GPC0_TPC0_TPCCS_TPC_EXCEPTION: 0x0 [ 1589.664987] __ga10b__ NV_PGRAPH_PRI_GPC0_TPC0_TPCCS_TPC_EXCEPTION_EN: 0x16 [ 1589.672086] __ga10b__ NV_PGRAPH_PRI_GPCS_TPCS_SMS_HWW_GLOBAL_ESR_REPORT_MASK: 0x1174 [ 1589.680065] __ga10b__ NV_PGRAPH_PRI_GPCS_TPCS_SMS_HWW_WARP_ESR_REPORT_MASK: 0xe812b60 [ 1589.688134] __ga10b__ NV_PGRAPH_PRI_GPCS_TPCS_SMS_HWW_GLOBAL_ESR: 0x0 [ 1589.694763] __ga10b__ NV_PGRAPH_PRI_GPCS_TPCS_SMS_DBGR_CONTROL0: 0x1c00 [ 1589.701598] __ga10b__ NV_PGRAPH_PRI_GPCS_TPCS_SMS_DBGR_STATUS0: 0x0 [ 1589.708044] __ga10b__ NV_PGRAPH_PRI_GPCS_TPCS_SMS_DBGR_BPT_PAUSE_MASK_0: 0x0 [ 1589.715305] __ga10b__ NV_PGRAPH_PRI_GPCS_TPCS_SMS_DBGR_BPT_PAUSE_MASK_1: 0x0 [ 1589.722561] __ga10b__ NV_PGRAPH_PRI_GPC0_TPC0_SM0_HWW_WARP_ESR: 0x0 [ 1589.729029] __ga10b__ NV_PGRAPH_PRI_GPC0_TPC0_SM0_HWW_WARP_ESR_REPORT_MASK: 0x0 [ 1589.736559] __ga10b__ NV_PGRAPH_PRI_GPC0_TPC0_SM0_HWW_GLOBAL_ESR: 0x0 [ 1589.743204] __ga10b__ NV_PGRAPH_PRI_GPC0_TPC0_SM0_HWW_GLOBAL_ESR_REPORT_MASK: 0x0 [ 1589.750926] __ga10b__ NV_PGRAPH_PRI_GPC0_TPC0_SM0_DBGR_CONTROL0: 0x1c00 [ 1589.757760] __ga10b__ NV_PGRAPH_PRI_GPC0_TPC0_SM0_DBGR_STATUS0: 0x0 [ 1589.764354] __ga10b__ NV_PGRAPH_PRI_GPC0_TPC0_SM1_HWW_WARP_ESR: 0x0 [ 1589.770807] __ga10b__ NV_PGRAPH_PRI_GPC0_TPC0_SM1_HWW_WARP_ESR_REPORT_MASK: 0xe812b60 [ 1589.778883] __ga10b__ NV_PGRAPH_PRI_GPC0_TPC0_SM1_HWW_GLOBAL_ESR: 0x0 [ 1589.785514] __ga10b__ NV_PGRAPH_PRI_GPC0_TPC0_SM1_HWW_GLOBAL_ESR_REPORT_MASK: 0x0 [ 1589.793228] __ga10b__ NV_PGRAPH_PRI_GPC0_TPC0_SM1_DBGR_CONTROL0: 0x1c00 [ 1589.800053] __ga10b__ NV_PGRAPH_PRI_GPC0_TPC0_SM1_DBGR_STATUS0: 0x0 [ 1589.806508] __ga10b__ NV_PGRAPH_PRI_GPC0_TPC1_SM0_HWW_WARP_ESR: 0x0 [ 1589.812962] __ga10b__ NV_PGRAPH_PRI_GPC0_TPC1_SM0_HWW_WARP_ESR_REPORT_MASK: 0x0 [ 1589.820499] __ga10b__ NV_PGRAPH_PRI_GPC0_TPC1_SM0_HWW_GLOBAL_ESR: 0x0 [ 1589.827153] __ga10b__ NV_PGRAPH_PRI_GPC0_TPC1_SM0_HWW_GLOBAL_ESR_REPORT_MASK: 0x0 [ 1589.834868] __ga10b__ NV_PGRAPH_PRI_GPC0_TPC1_SM0_DBGR_CONTROL0: 0x1c00 [ 1589.841681] __ga10b__ NV_PGRAPH_PRI_GPC0_TPC1_SM0_DBGR_STATUS0: 0x0 [ 1589.848153] __ga10b__ NV_PGRAPH_PRI_GPC0_TPC1_SM1_HWW_WARP_ESR: 0x0 [ 1589.854610] __ga10b__ NV_PGRAPH_PRI_GPC0_TPC1_SM1_HWW_WARP_ESR_REPORT_MASK: 0xe812b60 [ 1589.862678] __ga10b__ NV_PGRAPH_PRI_GPC0_TPC1_SM1_HWW_GLOBAL_ESR: 0x0 [ 1589.869310] __ga10b__ NV_PGRAPH_PRI_GPC0_TPC1_SM1_HWW_GLOBAL_ESR_REPORT_MASK: 0x1174 [ 1589.877295] __ga10b__ NV_PGRAPH_PRI_GPC0_TPC1_SM1_DBGR_CONTROL0: 0x1c00 [ 1589.884121] __ga10b__ NV_PGRAPH_PRI_GPC0_TPC1_SM1_DBGR_STATUS0: 0x0 [ 1589.890577] __ga10b__ NV_PGRAPH_PRI_GPC0_TPC2_SM0_HWW_WARP_ESR: 0x0 [ 1589.897034] __ga10b__ NV_PGRAPH_PRI_GPC0_TPC2_SM0_HWW_WARP_ESR_REPORT_MASK: 0x0 [ 1589.904569] __ga10b__ NV_PGRAPH_PRI_GPC0_TPC2_SM0_HWW_GLOBAL_ESR: 0x0 [ 1589.911203] __ga10b__ NV_PGRAPH_PRI_GPC0_TPC2_SM0_HWW_GLOBAL_ESR_REPORT_MASK: 0x1174 [ 1589.919192] __ga10b__ NV_PGRAPH_PRI_GPC0_TPC2_SM0_DBGR_CONTROL0: 0x1c00 [ 1589.926007] __ga10b__ NV_PGRAPH_PRI_GPC0_TPC2_SM0_DBGR_STATUS0: 0x0 [ 1589.932464] __ga10b__ NV_PGRAPH_PRI_GPC0_TPC2_SM1_HWW_WARP_ESR: 0x0 [ 1589.938924] __ga10b__ NV_PGRAPH_PRI_GPC0_TPC2_SM1_HWW_WARP_ESR_REPORT_MASK: 0xe812b60 [ 1589.946997] __ga10b__ NV_PGRAPH_PRI_GPC0_TPC2_SM1_HWW_GLOBAL_ESR: 0x0 [ 1589.953631] __ga10b__ NV_PGRAPH_PRI_GPC0_TPC2_SM1_HWW_GLOBAL_ESR_REPORT_MASK: 0x1174 [ 1589.961613] __ga10b__ NV_PGRAPH_PRI_GPC0_TPC2_SM1_DBGR_CONTROL0: 0x1c00 [ 1589.968426] __ga10b__ NV_PGRAPH_PRI_GPC0_TPC2_SM1_DBGR_STATUS0: 0x0 [ 1589.974901] __ga10b__ NV_PGRAPH_PRI_GPC1_TPC0_SM0_HWW_WARP_ESR: 0x0 [ 1589.981354] __ga10b__ NV_PGRAPH_PRI_GPC1_TPC0_SM0_HWW_WARP_ESR_REPORT_MASK: 0xe812b60 [ 1589.989417] __ga10b__ NV_PGRAPH_PRI_GPC1_TPC0_SM0_HWW_GLOBAL_ESR: 0x0 [ 1589.996062] __ga10b__ NV_PGRAPH_PRI_GPC1_TPC0_SM0_HWW_GLOBAL_ESR_REPORT_MASK: 0x1174 [ 1590.004048] __ga10b__ NV_PGRAPH_PRI_GPC1_TPC0_SM0_DBGR_CONTROL0: 0x1c00 [ 1590.010877] __ga10b__ NV_PGRAPH_PRI_GPC1_TPC0_SM0_DBGR_STATUS0: 0x0 [ 1590.017349] __ga10b__ NV_PGRAPH_PRI_GPC1_TPC0_SM1_HWW_WARP_ESR: 0x0 [ 1590.023820] __ga10b__ NV_PGRAPH_PRI_GPC1_TPC0_SM1_HWW_WARP_ESR_REPORT_MASK: 0x781eb60 [ 1590.031891] __ga10b__ NV_PGRAPH_PRI_GPC1_TPC0_SM1_HWW_GLOBAL_ESR: 0x0 [ 1590.038539] __ga10b__ NV_PGRAPH_PRI_GPC1_TPC0_SM1_HWW_GLOBAL_ESR_REPORT_MASK: 0x1174 [ 1590.046516] __ga10b__ NV_PGRAPH_PRI_GPC1_TPC0_SM1_DBGR_CONTROL0: 0x1c00 [ 1590.053346] __ga10b__ NV_PGRAPH_PRI_GPC1_TPC0_SM1_DBGR_STATUS0: 0x0 [ 1590.059798] __ga10b__ NV_PGRAPH_PRI_GPC1_TPC1_SM0_HWW_WARP_ESR: 0x0 [ 1590.066244] __ga10b__ NV_PGRAPH_PRI_GPC1_TPC1_SM0_HWW_WARP_ESR_REPORT_MASK: 0xe812b60 [ 1590.074314] __ga10b__ NV_PGRAPH_PRI_GPC1_TPC1_SM0_HWW_GLOBAL_ESR: 0x0 [ 1590.080967] __ga10b__ NV_PGRAPH_PRI_GPC1_TPC1_SM0_HWW_GLOBAL_ESR_REPORT_MASK: 0x1174 [ 1590.088954] __ga10b__ NV_PGRAPH_PRI_GPC1_TPC1_SM0_DBGR_CONTROL0: 0x1c00 [ 1590.095780] __ga10b__ NV_PGRAPH_PRI_GPC1_TPC1_SM0_DBGR_STATUS0: 0x0 [ 1590.102228] __ga10b__ NV_PGRAPH_PRI_GPC1_TPC1_SM1_HWW_WARP_ESR: 0x0 [ 1590.108686] __ga10b__ NV_PGRAPH_PRI_GPC1_TPC1_SM1_HWW_WARP_ESR_REPORT_MASK: 0xe812b60 [ 1590.116748] __ga10b__ NV_PGRAPH_PRI_GPC1_TPC1_SM1_HWW_GLOBAL_ESR: 0x0 [ 1590.123380] __ga10b__ NV_PGRAPH_PRI_GPC1_TPC1_SM1_HWW_GLOBAL_ESR_REPORT_MASK: 0x1174 [ 1590.131369] __ga10b__ NV_PGRAPH_PRI_GPC1_TPC1_SM1_DBGR_CONTROL0: 0x1c00 [ 1590.138194] __ga10b__ NV_PGRAPH_PRI_GPC1_TPC1_SM1_DBGR_STATUS0: 0x0 [ 1590.144642] __ga10b__ NV_PGRAPH_PRI_GPC1_TPC2_SM0_HWW_WARP_ESR: 0x0 [ 1590.151097] __ga10b__ NV_PGRAPH_PRI_GPC1_TPC2_SM0_HWW_WARP_ESR_REPORT_MASK: 0x0 [ 1590.158629] __ga10b__ NV_PGRAPH_PRI_GPC1_TPC2_SM0_HWW_GLOBAL_ESR: 0x0 [ 1590.165275] __ga10b__ NV_PGRAPH_PRI_GPC1_TPC2_SM0_HWW_GLOBAL_ESR_REPORT_MASK: 0x1174 [ 1590.173255] __ga10b__ NV_PGRAPH_PRI_GPC1_TPC2_SM0_DBGR_CONTROL0: 0x1c00 [ 1590.180082] __ga10b__ NV_PGRAPH_PRI_GPC1_TPC2_SM0_DBGR_STATUS0: 0x0 [ 1590.186546] __ga10b__ NV_PGRAPH_PRI_GPC1_TPC2_SM1_HWW_WARP_ESR: 0x0 [ 1590.193003] __ga10b__ NV_PGRAPH_PRI_GPC1_TPC2_SM1_HWW_WARP_ESR_REPORT_MASK: 0xe812b60 [ 1590.201072] __ga10b__ NV_PGRAPH_PRI_GPC1_TPC2_SM1_HWW_GLOBAL_ESR: 0x0 [ 1590.207715] __ga10b__ NV_PGRAPH_PRI_GPC1_TPC2_SM1_HWW_GLOBAL_ESR_REPORT_MASK: 0x1174 [ 1590.215699] __ga10b__ NV_PGRAPH_PRI_GPC1_TPC2_SM1_DBGR_CONTROL0: 0x1c00 [ 1590.222521] __ga10b__ NV_PGRAPH_PRI_GPC1_TPC2_SM1_DBGR_STATUS0: 0x0 [ 1590.228987] __ga10b__ NV_PGRAPH_PRI_GPC1_TPC3_SM0_HWW_WARP_ESR: 0x0 [ 1590.235442] __ga10b__ NV_PGRAPH_PRI_GPC1_TPC3_SM0_HWW_WARP_ESR_REPORT_MASK: 0xe812b60 [ 1590.243512] __ga10b__ NV_PGRAPH_PRI_GPC1_TPC3_SM0_HWW_GLOBAL_ESR: 0x0 [ 1590.250162] __ga10b__ NV_PGRAPH_PRI_GPC1_TPC3_SM0_HWW_GLOBAL_ESR_REPORT_MASK: 0x0 [ 1590.257882] __ga10b__ NV_PGRAPH_PRI_GPC1_TPC3_SM0_DBGR_CONTROL0: 0x1c00 [ 1590.264701] __ga10b__ NV_PGRAPH_PRI_GPC1_TPC3_SM0_DBGR_STATUS0: 0x0 [ 1590.271155] __ga10b__ NV_PGRAPH_PRI_GPC1_TPC3_SM1_HWW_WARP_ESR: 0x0 [ 1590.277611] __ga10b__ NV_PGRAPH_PRI_GPC1_TPC3_SM1_HWW_WARP_ESR_REPORT_MASK: 0xe812b60 [ 1590.285682] __ga10b__ NV_PGRAPH_PRI_GPC1_TPC3_SM1_HWW_GLOBAL_ESR: 0x0 [ 1590.292324] __ga10b__ NV_PGRAPH_PRI_GPC1_TPC3_SM1_HWW_GLOBAL_ESR_REPORT_MASK: 0x0 [ 1590.300041] __ga10b__ NV_PGRAPH_PRI_GPC1_TPC3_SM1_DBGR_CONTROL0: 0x1c00 [ 1590.306868] __ga10b__ NV_PGRAPH_PRI_GPC1_TPC3_SM1_DBGR_STATUS0: 0x0 [ 1590.313680] nvgpu: 17000000.ga10b nvgpu_tsg_force_reset_ch:469 [ERR] chid: 509 is not bound to tsg [ 1590.369528] cpufreq: cpu0,cur:2058000,set:2188800,set ndiv:171 [ 1592.999489] cpufreq: cpu4,cur:2023000,set:2188800,set ndiv:171 [ 1598.092402] nvgpu: 17000000.ga10b ga10b_pbdma_handle_intr_0_legacy:437 [ERR] semaphore acquire timeout! [ 1598.102540] __ga10b__ Channel Status - chip ga10b [ 1598.102542] __ga10b__ --------------------------- [ 1598.107446] __ga10b__ 492-ga10b, TSG: 19, pid 3008, refs: 2, deterministic: no, domain name: (default) [ 1598.112326] __ga10b__ channel status: in use idle not busy [ 1598.121950] __ga10b__ RAMFC: TOP: 80000020068b7080 PUT: 0020068b7080 GET: 0020068b7080 FETCH: 000000000000 HEADER: 2140006c COUNT: 00000000 SEMAPHORE: addr 0020068a0000 payload 0000000000000000 execute 00100001 [ 1598.127734] __ga10b__ [ 1598.147061] __ga10b__ 493-ga10b, TSG: 18, pid 3008, refs: 2, deterministic: no, domain name: (default) [ 1598.149619] __ga10b__ channel status: in use idle not busy [ 1598.159218] __ga10b__ RAMFC: TOP: 800000200607db40 PUT: 00200607db40 GET: 00200607db40 FETCH: 000000000000 HEADER: 2140006c COUNT: 00000000 SEMAPHORE: addr 0020063c0000 payload 0000000000000000 execute 00100001 [ 1598.164957] __ga10b__ [ 1598.184255] __ga10b__ 494-ga10b, TSG: 17, pid 3008, refs: 5, deterministic: no, domain name: (default) [ 1598.186842] __ga10b__ channel status: in use pending busy [ 1598.196448] __ga10b__ RAMFC: TOP: 8000002006063cf0 PUT: 002006063cf0 GET: 002006063cf0 FETCH: 000000000000 HEADER: 2140006c COUNT: 00000000 SEMAPHORE: addr 002006010000 payload 0000000000000000 execute 00100001 [ 1598.202107] __ga10b__ [ 1598.221413] __ga10b__ 495-ga10b, TSG: 16, pid 3008, refs: 4, deterministic: no, domain name: (default) [ 1598.223965] __ga10b__ channel status: in use pending busy [ 1598.233579] __ga10b__ RAMFC: TOP: 8000002006000f78 PUT: 002006000f78 GET: 002006000f78 FETCH: 000000000000 HEADER: 2140006c COUNT: 00000000 SEMAPHORE: addr 002005d30000 payload 0000000000000000 execute 00100001 [ 1598.239239] __ga10b__ [ 1598.258485] __ga10b__ 496-ga10b, TSG: 15, pid 3090, refs: 2, deterministic: no, domain name: (default) [ 1598.261015] __ga10b__ channel status: in use idle not busy [ 1598.270604] __ga10b__ RAMFC: TOP: 80000020040080a0 PUT: 0020040080a0 GET: 0020040080a0 FETCH: 000000000000 HEADER: 2140006c COUNT: 00000000 SEMAPHORE: addr 002004010000 payload 0000000000000000 execute 00100001 [ 1598.276355] __ga10b__ [ 1598.295582] __ga10b__ 497-ga10b, TSG: 14, pid 3008, refs: 5, deterministic: no, domain name: (default) [ 1598.298115] __ga10b__ channel status: in use pending, acquire_fail busy [ 1598.307705] __ga10b__ RAMFC: TOP: 8000001ffacedd10 PUT: 001ffacedd24 GET: 001ffacedd10 FETCH: 000000000000 HEADER: 2140006c COUNT: 11110000 SEMAPHORE: addr 002000040000 payload 0000000000019186 execute 00081003 [ 1598.314610] __ga10b__ [ 1598.333851] __ga10b__ 498-ga10b, TSG: 13, pid 3008, refs: 2, deterministic: no, domain name: (default) [ 1598.336379] __ga10b__ channel status: in use idle not busy [ 1598.345971] __ga10b__ RAMFC: TOP: 8000002005a30168 PUT: 002005a30168 GET: 002005a30168 FETCH: 000000000000 HEADER: 2140006c COUNT: 00000000 SEMAPHORE: addr 002005a20000 payload 0000000000000000 execute 00100001 [ 1598.351728] __ga10b__ [ 1598.370952] __ga10b__ 499-ga10b, TSG: 12, pid 3008, refs: 2, deterministic: no, domain name: (default) [ 1598.373478] __ga10b__ channel status: in use idle not busy [ 1598.383070] __ga10b__ RAMFC: TOP: 80000020051f8208 PUT: 0020051f8208 GET: 0020051f8208 FETCH: 000000000000 HEADER: 2140006c COUNT: 00000000 SEMAPHORE: addr 002005640000 payload 0000000000000000 execute 00100001 [ 1598.388813] __ga10b__ [ 1598.408059] __ga10b__ 500-ga10b, TSG: 11, pid 3008, refs: 2, deterministic: no, domain name: (default) [ 1598.410587] __ga10b__ channel status: in use idle not busy [ 1598.420171] __ga10b__ RAMFC: TOP: 8000002005198230 PUT: 002005198230 GET: 002005198230 FETCH: 000000000000 HEADER: 2140006c COUNT: 00000000 SEMAPHORE: addr 0020051e0000 payload 0000000000000000 execute 00100001 [ 1598.425915] __ga10b__ [ 1598.445159] __ga10b__ 501-ga10b, TSG: 10, pid 3008, refs: 2, deterministic: no, domain name: (default) [ 1598.447686] __ga10b__ channel status: in use idle not busy [ 1598.457276] __ga10b__ RAMFC: TOP: 8000002005180208 PUT: 002005180208 GET: 002005180208 FETCH: 000000000000 HEADER: 2140006c COUNT: 00000000 SEMAPHORE: addr 002004eb0000 payload 0000000000000000 execute 00100001 [ 1598.463019] __ga10b__ [ 1598.482276] __ga10b__ 502-ga10b, TSG: 9, pid 3008, refs: 2, deterministic: no, domain name: (default) [ 1598.484800] __ga10b__ channel status: in use idle not busy [ 1598.494313] __ga10b__ RAMFC: TOP: 8000002004e88230 PUT: 002004e88230 GET: 002004e88230 FETCH: 000000000000 HEADER: 2140006c COUNT: 00000000 SEMAPHORE: addr 002004e90000 payload 0000000000000000 execute 00100001 [ 1598.500064] __ga10b__ [ 1598.519309] __ga10b__ 503-ga10b, TSG: 8, pid 3008, refs: 2, deterministic: no, domain name: (default) [ 1598.521837] __ga10b__ channel status: in use idle not busy [ 1598.531343] __ga10b__ RAMFC: TOP: 8000002004bb0208 PUT: 002004bb0208 GET: 002004bb0208 FETCH: 000000000000 HEADER: 2140006c COUNT: 00000000 SEMAPHORE: addr 002004ba0000 payload 0000000000000000 execute 00100001 [ 1598.537084] __ga10b__ [ 1598.556336] __ga10b__ 504-ga10b, TSG: 7, pid 3008, refs: 2, deterministic: no, domain name: (default) [ 1598.558864] __ga10b__ channel status: in use idle not busy [ 1598.568358] __ga10b__ RAMFC: TOP: 80000020046d0230 PUT: 0020046d0230 GET: 0020046d0230 FETCH: 000000000000 HEADER: 2140006c COUNT: 00000000 SEMAPHORE: addr 0020046c0000 payload 0000000000000000 execute 00100001 [ 1598.574094] __ga10b__ [ 1598.593345] __ga10b__ 505-ga10b, TSG: 6, pid 3008, refs: 2, deterministic: no, domain name: (default) [ 1598.595886] __ga10b__ channel status: in use idle not busy [ 1598.605382] __ga10b__ RAMFC: TOP: 8000002004318050 PUT: 002004318050 GET: 002004318050 FETCH: 000000000000 HEADER: 2140006c COUNT: 00000000 SEMAPHORE: addr 002004360000 payload 0000000000000000 execute 00100001 [ 1598.611121] __ga10b__ [ 1598.630360] __ga10b__ 506-ga10b, TSG: 5, pid 3008, refs: 2, deterministic: no, domain name: (default) [ 1598.632878] __ga10b__ channel status: in use idle not busy [ 1598.642382] __ga10b__ RAMFC: TOP: 8000002004305230 PUT: 002004305230 GET: 002004305230 FETCH: 000000000000 HEADER: 2140006c COUNT: 00000000 SEMAPHORE: addr 002004030000 payload 0000000000000000 execute 00100001 [ 1598.648118] __ga10b__ [ 1598.667356] __ga10b__ 507-ga10b, TSG: 4, pid 3008, refs: 2, deterministic: no, domain name: (default) [ 1598.669893] __ga10b__ channel status: in use idle not busy [ 1598.679390] __ga10b__ RAMFC: TOP: 80000020040080a0 PUT: 0020040080a0 GET: 0020040080a0 FETCH: 000000000000 HEADER: 2140006c COUNT: 00000000 SEMAPHORE: addr 002004010000 payload 0000000000000000 execute 00100001 [ 1598.685132] __ga10b__ [ 1598.704368] __ga10b__ 508-ga10b, TSG: 3, pid 2755, refs: 2, deterministic: no, domain name: (default) [ 1598.706893] __ga10b__ channel status: in use idle not busy [ 1598.716389] __ga10b__ RAMFC: TOP: 80000020040080a0 PUT: 0020040080a0 GET: 0020040080a0 FETCH: 000000000000 HEADER: 2140006c COUNT: 00000000 SEMAPHORE: addr 002004010000 payload 0000000000000000 execute 00100001 [ 1598.722130] __ga10b__ [ 1598.741372] __ga10b__ 509-ga10b, TSG: 2, pid 2334, refs: 5, deterministic: no, domain name: (default) [ 1598.743892] __ga10b__ channel status: in use on_pbdma, pbdma_busy busy [ 1598.753386] __ga10b__ RAMFC: TOP: 8000001fff3a065c PUT: 001fff3a0670 GET: 001fff3a065c FETCH: 000000000000 HEADER: 2140006c COUNT: 11110000 SEMAPHORE: addr 002000110000 payload 000000000001316a execute 00081003 [ 1598.760190] __ga10b__ [ 1598.779430] __ga10b__ 510-ga10b, TSG: 1, pid 2080, refs: 2, deterministic: no, domain name: (default) [ 1598.781967] __ga10b__ channel status: in use idle not busy [ 1598.791458] __ga10b__ RAMFC: TOP: 80000020040339b0 PUT: 0020040339b0 GET: 0020040339b0 FETCH: 000000000000 HEADER: 2140006c COUNT: 00000000 SEMAPHORE: addr 002004320000 payload 0000000000000000 execute 00000001 [ 1598.797206] __ga10b__ [ 1598.816460] __ga10b__ 511-ga10b, TSG: 0, pid 2080, refs: 3, deterministic: no, domain name: (default) [ 1598.818988] __ga10b__ channel status: in use on_eng, pending busy [ 1598.828480] __ga10b__ RAMFC: TOP: 8000002004044c18 PUT: 002004044c18 GET: 002004044c18 FETCH: 000000000000 HEADER: 2140006c COUNT: 00000000 SEMAPHORE: addr 002004020000 payload 0000000000000000 execute 00100001 [ 1598.834845] __ga10b__ [ 1598.854072] __ga10b__ PBDMA Status - chip ga10b [ 1598.856589] __ga10b__ ------------------------- [ 1598.861261] __ga10b__ pbdma 0: [ 1598.865936] __ga10b__ id: 2 - [tsg] next_id: - -1 [channel] | status: valid [ 1598.869096] __ga10b__ PBDMA_PUT 0000001fff3a0670 PBDMA_GET 0000001fff3a065c [ 1598.876808] __ga10b__ GP_PUT 000006e7 GP_GET 000006df FETCH 000006df HEADER 2140006c [ 1598.884167] __ga10b__ HDR 2001001b SHADOW0 04044bf0 SHADOW1 00002820 [ 1598.893032] __ga10b__ pbdma 1: [ 1598.900476] __ga10b__ id: -1 - [channel] next_id: - -1 [channel] | status: invalid [ 1598.903614] __ga10b__ PBDMA_PUT 00000020046d00a0 PBDMA_GET 00000020046d00a0 [ 1598.911600] __ga10b__ GP_PUT 0000000c GP_GET 0000000c FETCH 0000000c HEADER 2140006c [ 1598.918962] __ga10b__ HDR 2001001b SHADOW0 046d0078 SHADOW1 00002820 [ 1598.927836] __ga10b__ pbdma 2: [ 1598.935286] __ga10b__ id: -1 - [channel] next_id: - -1 [channel] | status: invalid [ 1598.938434] __ga10b__ PBDMA_PUT 0000002004318118 PBDMA_GET 0000002004318118 [ 1598.946412] __ga10b__ GP_PUT 0000001c GP_GET 0000001c FETCH 0000001c HEADER 2140006c [ 1598.953767] __ga10b__ HDR 2001001b SHADOW0 043180f0 SHADOW1 00002820 [ 1598.962638] __ga10b__ pbdma 3: [ 1598.970083] __ga10b__ id: -1 - [channel] next_id: - -1 [channel] | status: invalid [ 1598.973226] __ga10b__ PBDMA_PUT 00000067ef7f6fec PBDMA_GET 000000cd20b86794 [ 1598.981211] __ga10b__ GP_PUT 00000000 GP_GET 7e1826eb FETCH 00000000 HEADER e1955910 [ 1598.988569] __ga10b__ HDR 5f129d84 SHADOW0 61804464 SHADOW1 be122349 [ 1598.997435] __ga10b__ pbdma 4: [ 1599.004892] __ga10b__ id: -1 - [channel] next_id: - -1 [channel] | status: invalid [ 1599.008050] __ga10b__ PBDMA_PUT 0000000db7213608 PBDMA_GET 0000000db7213608 [ 1599.016028] __ga10b__ GP_PUT 00000000 GP_GET 6b8eb69b FETCH 00000000 HEADER 8086a79c [ 1599.023387] __ga10b__ HDR 145ec4b0 SHADOW0 bdc529ee SHADOW1 f4b979a2 [ 1599.032260] __ga10b__ pbdma 5: [ 1599.039707] __ga10b__ id: -1 - [channel] next_id: - -1 [channel] | status: invalid [ 1599.042861] __ga10b__ PBDMA_PUT 0000002004ea8118 PBDMA_GET 0000002004ea8118 [ 1599.050844] __ga10b__ GP_PUT 0000001c GP_GET 0000001c FETCH 0000001c HEADER 2140006c [ 1599.058204] __ga10b__ HDR 2001001b SHADOW0 04ea80f0 SHADOW1 00002820 [ 1599.067075] __ga10b__ [ 1599.074526] __ga10b__ ga10b eng 0: [ 1599.077050] __ga10b__ id: 0 (tsg), next_id: -1 (channel), ctx status: valid [ 1599.080652] __ga10b__ [ 1599.087919] __ga10b__ ga10b eng 1: [ 1599.090435] __ga10b__ id: -1 (channel), next_id: -1 (channel), ctx status: invalid [ 1599.094035] __ga10b__ [ 1599.101942] __ga10b__ ga10b eng 2: [ 1599.104470] __ga10b__ id: -1 (channel), next_id: -1 (channel), ctx status: invalid [ 1599.108076] __ga10b__ [ 1599.115983] __ga10b__ ga10b eng 3: [ 1599.118506] __ga10b__ id: -1 (channel), next_id: -1 (channel), ctx status: invalid [ 1599.122112] __ga10b__ [ 1599.130014] __ga10b__ ga10b eng 4: [ 1599.132540] __ga10b__ id: -1 (channel), next_id: -1 (channel), ctx status: invalid [ 1599.136146] __ga10b__ [ 1599.144041] __ga10b__ ga10b eng 5: [ 1599.146567] __ga10b__ id: -1 (channel), next_id: -1 (channel), ctx status: invalid [ 1599.150167] __ga10b__ [ 1599.158064] __ga10b__ [ 1599.160596] nvgpu: 17000000.ga10b report_pbdma_error:330 [ERR] pbdma_intr_0(0)= 0x04000000 [ 1599.173238] nvgpu: 17000000.ga10b nvgpu_cic_mon_report_err_safety_services:55 [ERR] Error reporting is not supported in this platform [ 1599.185883] nvgpu: 17000000.ga10b nvgpu_set_err_notifier_locked:149 [ERR] error notifier set to 24 for ch 509 [ 1601.313915] cpufreq: cpu4,cur:2052000,set:2188800,set ndiv:171 [ 1602.874403] cpufreq: cpu0,cur:1313000,set:2188800,set ndiv:171 [ 1606.984265] nvgpu: 17000000.ga10b ga10b_pbdma_handle_intr_0_legacy:437 [ERR] semaphore acquire timeout! [ 1606.994337] __ga10b__ Channel Status - chip ga10b [ 1606.994338] __ga10b__ --------------------------- [ 1606.999216] __ga10b__ 492-ga10b, TSG: 19, pid 3008, refs: 2, deterministic: no, domain name: (default) [ 1607.004083] __ga10b__ channel status: in use on_eng not busy [ 1607.013706] __ga10b__ RAMFC: TOP: 80000020068b2eb8 PUT: 0020068b2eb8 GET: 0020068b2eb8 FETCH: 000000000000 HEADER: 2140006c COUNT: 00000000 SEMAPHORE: addr 0020068a0000 payload 0000000000000000 execute 00100001 [ 1607.019653] __ga10b__ [ 1607.038902] __ga10b__ 493-ga10b, TSG: 18, pid 3008, refs: 2, deterministic: no, domain name: (default) [ 1607.041448] __ga10b__ channel status: in use idle not busy [ 1607.051026] __ga10b__ RAMFC: TOP: 800000200607ad50 PUT: 00200607ad50 GET: 00200607ad50 FETCH: 000000000000 HEADER: 2140006c COUNT: 00000000 SEMAPHORE: addr 0020063c0000 payload 0000000000000000 execute 00100001 [ 1607.056765] __ga10b__ [ 1607.076005] __ga10b__ 494-ga10b, TSG: 17, pid 3008, refs: 2, deterministic: no, domain name: (default) [ 1607.078531] __ga10b__ channel status: in use idle not busy [ 1607.088117] __ga10b__ RAMFC: TOP: 8000002006060140 PUT: 002006060140 GET: 002006060140 FETCH: 000000000000 HEADER: 2140006c COUNT: 00000000 SEMAPHORE: addr 002006010000 payload 0000000000000000 execute 00100001 [ 1607.093859] __ga10b__ [ 1607.113100] __ga10b__ 495-ga10b, TSG: 16, pid 3008, refs: 2, deterministic: no, domain name: (default) [ 1607.115638] __ga10b__ channel status: in use idle not busy [ 1607.125229] __ga10b__ RAMFC: TOP: 8000002006006950 PUT: 002006006950 GET: 002006006950 FETCH: 000000000000 HEADER: 2140006c COUNT: 00000000 SEMAPHORE: addr 002005d30000 payload 0000000000000000 execute 00100001 [ 1607.130987] __ga10b__ [ 1607.150228] __ga10b__ 496-ga10b, TSG: 15, pid 3090, refs: 2, deterministic: no, domain name: (default) [ 1607.152745] __ga10b__ channel status: in use idle not busy [ 1607.162331] __ga10b__ RAMFC: TOP: 80000020040080a0 PUT: 0020040080a0 GET: 0020040080a0 FETCH: 000000000000 HEADER: 2140006c COUNT: 00000000 SEMAPHORE: addr 002004010000 payload 0000000000000000 execute 00100001 [ 1607.168075] __ga10b__ [ 1607.187314] __ga10b__ 497-ga10b, TSG: 14, pid 3008, refs: 5, deterministic: no, domain name: (default) [ 1607.189847] __ga10b__ channel status: in use pending, acquire_fail busy [ 1607.199433] __ga10b__ RAMFC: TOP: 8000001ffac40b64 PUT: 001ffac40b78 GET: 001ffac40b64 FETCH: 000000000000 HEADER: 2140006c COUNT: 11110000 SEMAPHORE: addr 002000040000 payload 000000000001939b execute 00081003 [ 1607.206347] __ga10b__ [ 1607.225603] __ga10b__ 498-ga10b, TSG: 13, pid 3008, refs: 2, deterministic: no, domain name: (default) [ 1607.228132] __ga10b__ channel status: in use idle not busy [ 1607.237724] __ga10b__ RAMFC: TOP: 8000002005a30168 PUT: 002005a30168 GET: 002005a30168 FETCH: 000000000000 HEADER: 2140006c COUNT: 00000000 SEMAPHORE: addr 002005a20000 payload 0000000000000000 execute 00100001 [ 1607.243475] __ga10b__ [ 1607.262715] __ga10b__ 499-ga10b, TSG: 12, pid 3008, refs: 2, deterministic: no, domain name: (default) [ 1607.265241] __ga10b__ channel status: in use idle not busy [ 1607.274832] __ga10b__ RAMFC: TOP: 80000020051f8208 PUT: 0020051f8208 GET: 0020051f8208 FETCH: 000000000000 HEADER: 2140006c COUNT: 00000000 SEMAPHORE: addr 002005640000 payload 0000000000000000 execute 00100001 [ 1607.280570] __ga10b__ [ 1607.299791] __ga10b__ 500-ga10b, TSG: 11, pid 3008, refs: 2, deterministic: no, domain name: (default) [ 1607.302315] __ga10b__ channel status: in use idle not busy [ 1607.311900] __ga10b__ RAMFC: TOP: 8000002005198230 PUT: 002005198230 GET: 002005198230 FETCH: 000000000000 HEADER: 2140006c COUNT: 00000000 SEMAPHORE: addr 0020051e0000 payload 0000000000000000 execute 00100001 [ 1607.317662] __ga10b__ [ 1607.336895] __ga10b__ 501-ga10b, TSG: 10, pid 3008, refs: 2, deterministic: no, domain name: (default) [ 1607.339411] __ga10b__ channel status: in use idle not busy [ 1607.348996] __ga10b__ RAMFC: TOP: 8000002005180208 PUT: 002005180208 GET: 002005180208 FETCH: 000000000000 HEADER: 2140006c COUNT: 00000000 SEMAPHORE: addr 002004eb0000 payload 0000000000000000 execute 00100001 [ 1607.354737] __ga10b__ [ 1607.373985] __ga10b__ 502-ga10b, TSG: 9, pid 3008, refs: 2, deterministic: no, domain name: (default) [ 1607.376523] __ga10b__ channel status: in use idle not busy [ 1607.386022] __ga10b__ RAMFC: TOP: 8000002004e88230 PUT: 002004e88230 GET: 002004e88230 FETCH: 000000000000 HEADER: 2140006c COUNT: 00000000 SEMAPHORE: addr 002004e90000 payload 0000000000000000 execute 00100001 [ 1607.391769] __ga10b__ [ 1607.411005] __ga10b__ 503-ga10b, TSG: 8, pid 3008, refs: 2, deterministic: no, domain name: (default) [ 1607.413533] __ga10b__ channel status: in use idle not busy [ 1607.423033] __ga10b__ RAMFC: TOP: 8000002004bb0208 PUT: 002004bb0208 GET: 002004bb0208 FETCH: 000000000000 HEADER: 2140006c COUNT: 00000000 SEMAPHORE: addr 002004ba0000 payload 0000000000000000 execute 00100001 [ 1607.428782] __ga10b__ [ 1607.448017] __ga10b__ 504-ga10b, TSG: 7, pid 3008, refs: 2, deterministic: no, domain name: (default) [ 1607.450546] __ga10b__ channel status: in use idle not busy [ 1607.460042] __ga10b__ RAMFC: TOP: 80000020046d0230 PUT: 0020046d0230 GET: 0020046d0230 FETCH: 000000000000 HEADER: 2140006c COUNT: 00000000 SEMAPHORE: addr 0020046c0000 payload 0000000000000000 execute 00100001 [ 1607.465784] __ga10b__ [ 1607.485035] __ga10b__ 505-ga10b, TSG: 6, pid 3008, refs: 2, deterministic: no, domain name: (default) [ 1607.487561] __ga10b__ channel status: in use idle not busy [ 1607.497062] __ga10b__ RAMFC: TOP: 8000002004318050 PUT: 002004318050 GET: 002004318050 FETCH: 000000000000 HEADER: 2140006c COUNT: 00000000 SEMAPHORE: addr 002004360000 payload 0000000000000000 execute 00100001 [ 1607.502804] __ga10b__ [ 1607.522046] __ga10b__ 506-ga10b, TSG: 5, pid 3008, refs: 2, deterministic: no, domain name: (default) [ 1607.524570] __ga10b__ channel status: in use idle not busy [ 1607.534073] __ga10b__ RAMFC: TOP: 8000002004305230 PUT: 002004305230 GET: 002004305230 FETCH: 000000000000 HEADER: 2140006c COUNT: 00000000 SEMAPHORE: addr 002004030000 payload 0000000000000000 execute 00100001 [ 1607.539815] __ga10b__ [ 1607.559069] __ga10b__ 507-ga10b, TSG: 4, pid 3008, refs: 2, deterministic: no, domain name: (default) [ 1607.561595] __ga10b__ channel status: in use idle not busy [ 1607.571093] __ga10b__ RAMFC: TOP: 80000020040080a0 PUT: 0020040080a0 GET: 0020040080a0 FETCH: 000000000000 HEADER: 2140006c COUNT: 00000000 SEMAPHORE: addr 002004010000 payload 0000000000000000 execute 00100001 [ 1607.576844] __ga10b__ [ 1607.596087] __ga10b__ 508-ga10b, TSG: 3, pid 2755, refs: 2, deterministic: no, domain name: (default) [ 1607.598604] __ga10b__ channel status: in use idle not busy [ 1607.608103] __ga10b__ RAMFC: TOP: 80000020040080a0 PUT: 0020040080a0 GET: 0020040080a0 FETCH: 000000000000 HEADER: 2140006c COUNT: 00000000 SEMAPHORE: addr 002004010000 payload 0000000000000000 execute 00100001 [ 1607.613842] __ga10b__ [ 1607.633081] __ga10b__ 509-ga10b, TSG: 2, pid 2334, refs: 5, deterministic: no, domain name: (default) [ 1607.635615] __ga10b__ channel status: in use on_pbdma, pbdma_busy busy [ 1607.645117] __ga10b__ RAMFC: TOP: 8000001fff3a065c PUT: 001fff3a0670 GET: 001fff3a065c FETCH: 000000000000 HEADER: 2140006c COUNT: 11110000 SEMAPHORE: addr 002000110000 payload 000000000001316c execute 00081003 [ 1607.651928] __ga10b__ [ 1607.671172] __ga10b__ 510-ga10b, TSG: 1, pid 2080, refs: 2, deterministic: no, domain name: (default) [ 1607.673698] __ga10b__ channel status: in use idle not busy [ 1607.683194] __ga10b__ RAMFC: TOP: 8000002004033be0 PUT: 002004033be0 GET: 002004033be0 FETCH: 000000000000 HEADER: 2140006c COUNT: 00000000 SEMAPHORE: addr 002004320000 payload 0000000000000000 execute 00000001 [ 1607.688937] __ga10b__ [ 1607.708165] __ga10b__ 511-ga10b, TSG: 0, pid 2080, refs: 2, deterministic: no, domain name: (default) [ 1607.710684] __ga10b__ channel status: in use idle not busy [ 1607.720189] __ga10b__ RAMFC: TOP: 8000002004046bf8 PUT: 002004046bf8 GET: 002004046bf8 FETCH: 000000000000 HEADER: 2140006c COUNT: 00000000 SEMAPHORE: addr 002004020000 payload 0000000000000000 execute 00100001 [ 1607.725934] __ga10b__ [ 1607.745186] __ga10b__ PBDMA Status - chip ga10b [ 1607.747724] __ga10b__ ------------------------- [ 1607.752398] __ga10b__ pbdma 0: [ 1607.757071] __ga10b__ id: 2 - [tsg] next_id: - -1 [channel] | status: valid [ 1607.760230] __ga10b__ PBDMA_PUT 0000001fff3a0670 PBDMA_GET 0000001fff3a065c [ 1607.767941] __ga10b__ GP_PUT 000006f7 GP_GET 000006ef FETCH 000006ef HEADER 2140006c [ 1607.775299] __ga10b__ HDR 2001001b SHADOW0 068b2e90 SHADOW1 00002820 [ 1607.784174] __ga10b__ pbdma 1: [ 1607.791634] __ga10b__ id: -1 - [channel] next_id: - -1 [channel] | status: invalid [ 1607.794791] __ga10b__ PBDMA_PUT 00000020046d00a0 PBDMA_GET 00000020046d00a0 [ 1607.802779] __ga10b__ GP_PUT 0000000c GP_GET 0000000c FETCH 0000000c HEADER 2140006c [ 1607.810136] __ga10b__ HDR 2001001b SHADOW0 046d0078 SHADOW1 00002820 [ 1607.819014] __ga10b__ pbdma 2: [ 1607.826459] __ga10b__ id: -1 - [channel] next_id: - -1 [channel] | status: invalid [ 1607.829623] __ga10b__ PBDMA_PUT 0000002004318118 PBDMA_GET 0000002004318118 [ 1607.837607] __ga10b__ GP_PUT 0000001c GP_GET 0000001c FETCH 0000001c HEADER 2140006c [ 1607.844969] __ga10b__ HDR 2001001b SHADOW0 043180f0 SHADOW1 00002820 [ 1607.853840] __ga10b__ pbdma 3: [ 1607.861281] __ga10b__ id: -1 - [channel] next_id: - -1 [channel] | status: invalid [ 1607.864431] __ga10b__ PBDMA_PUT 00000067ef7f6fec PBDMA_GET 000000cd20b86794 [ 1607.872416] __ga10b__ GP_PUT 00000000 GP_GET 7e1826eb FETCH 00000000 HEADER e1955910 [ 1607.879780] __ga10b__ HDR 5f129d84 SHADOW0 61804464 SHADOW1 be122349 [ 1607.888648] __ga10b__ pbdma 4: [ 1607.896090] __ga10b__ id: -1 - [channel] next_id: - -1 [channel] | status: invalid [ 1607.899247] __ga10b__ PBDMA_PUT 0000000db7213608 PBDMA_GET 0000000db7213608 [ 1607.907236] __ga10b__ GP_PUT 00000000 GP_GET 6b8eb69b FETCH 00000000 HEADER 8086a79c [ 1607.914599] __ga10b__ HDR 145ec4b0 SHADOW0 bdc529ee SHADOW1 f4b979a2 [ 1607.923479] __ga10b__ pbdma 5: [ 1607.930933] __ga10b__ id: -1 - [channel] next_id: - -1 [channel] | status: invalid [ 1607.934075] __ga10b__ PBDMA_PUT 0000002004ea8118 PBDMA_GET 0000002004ea8118 [ 1607.942061] __ga10b__ GP_PUT 0000001c GP_GET 0000001c FETCH 0000001c HEADER 2140006c [ 1607.949422] __ga10b__ HDR 2001001b SHADOW0 04ea80f0 SHADOW1 00002820 [ 1607.958280] __ga10b__ [ 1607.965726] __ga10b__ ga10b eng 0: [ 1607.968254] __ga10b__ id: 19 (tsg), next_id: -1 (channel), ctx status: valid [ 1607.971857] __ga10b__ [ 1607.979218] __ga10b__ ga10b eng 1: [ 1607.981741] __ga10b__ id: -1 (channel), next_id: -1 (channel), ctx status: invalid [ 1607.985344] __ga10b__ [ 1607.993232] __ga10b__ ga10b eng 2: [ 1607.995752] __ga10b__ id: -1 (channel), next_id: -1 (channel), ctx status: invalid [ 1607.999354] __ga10b__ [ 1608.007252] __ga10b__ ga10b eng 3: [ 1608.009776] __ga10b__ id: -1 (channel), next_id: -1 (channel), ctx status: invalid [ 1608.013376] __ga10b__ [ 1608.021272] __ga10b__ ga10b eng 4: [ 1608.023792] __ga10b__ id: -1 (channel), next_id: -1 (channel), ctx status: invalid [ 1608.027392] __ga10b__ [ 1608.035288] __ga10b__ ga10b eng 5: [ 1608.037802] __ga10b__ id: -1 (channel), next_id: -1 (channel), ctx status: invalid [ 1608.041406] __ga10b__ [ 1608.049301] __ga10b__ [ 1608.051824] nvgpu: 17000000.ga10b report_pbdma_error:330 [ERR] pbdma_intr_0(0)= 0x04000000 [ 1608.064460] nvgpu: 17000000.ga10b nvgpu_cic_mon_report_err_safety_services:55 [ERR] Error reporting is not supported in this platform [ 1608.067240] nvgpu: 17000000.ga10b nvgpu_channel_recover_from_wdt:107 [ERR] Job on channel 509 timed out [ 1608.077082] nvgpu: 17000000.ga10b nvgpu_set_err_notifier_locked:149 [ERR] error notifier set to 24 for ch 509 [ 1608.089075] __ga10b__ NV_PGRAPH_STATUS: 0x0 [ 1608.105460] __ga10b__ NV_PGRAPH_STATUS1: 0x0 [ 1608.109919] __ga10b__ NV_PGRAPH_ENGINE_STATUS: 0x1 [ 1608.114883] __ga10b__ NV_PGRAPH_GRFIFO_STATUS : 0x4040001 [ 1608.122904] __ga10b__ NV_PGRAPH_GRFIFO_CONTROL : 0x10001 [ 1608.128546] __ga10b__ NV_PGRAPH_PRI_FECS_HOST_INT_STATUS : 0x0 [ 1608.134590] __ga10b__ NV_PGRAPH_EXCEPTION : 0x0 [ 1608.139405] __ga10b__ NV_PGRAPH_FECS_INTR : 0x0 [ 1608.144173] __ga10b__ NV_PFIFO_ENGINE_STATUS(GR) : 0x80102010 [ 1608.150112] __ga10b__ NV_PGRAPH_ACTIVITY0: 0x0 [ 1608.154717] __ga10b__ NV_PGRAPH_ACTIVITY1: 0x0 [ 1608.159365] __ga10b__ NV_PGRAPH_ACTIVITY4: 0x0 [ 1608.163954] __ga10b__ NV_PGRAPH_PRI_SKED_ACTIVITY: 0x0 [ 1608.169252] __ga10b__ NV_PGRAPH_PRI_GPC0_GPCCS_GPC_ACTIVITY0: 0x1000000 [ 1608.176209] __ga10b__ NV_PGRAPH_PRI_GPC0_GPCCS_GPC_ACTIVITY1: 0x0 [ 1608.182538] __ga10b__ NV_PGRAPH_PRI_GPC0_GPCCS_GPC_ACTIVITY2: 0x0 [ 1608.188846] __ga10b__ NV_PGRAPH_PRI_GPC0_GPCCS_GPC_ACTIVITY3: 0x0 [ 1608.195134] __ga10b__ NV_PGRAPH_PRI_GPC0_GPCCS_GPC_ACTIVITY4: 0x0 [ 1608.201416] __ga10b__ NV_PGRAPH_PRI_GPC0_TPC0_TPCCS_TPC_ACTIVITY0: 0x0 [ 1608.208153] __ga10b__ NV_PGRAPH_PRI_GPC0_TPC1_TPCCS_TPC_ACTIVITY0: 0x0 [ 1608.214881] __ga10b__ NV_PGRAPH_PRI_GPC0_TPC2_TPCCS_TPC_ACTIVITY0: 0x0 [ 1608.221605] __ga10b__ NV_PGRAPH_PRI_GPCS_GPCCS_GPC_ACTIVITY0: 0x0 [ 1608.227901] __ga10b__ NV_PGRAPH_PRI_GPCS_GPCCS_GPC_ACTIVITY1: 0x0 [ 1608.234204] __ga10b__ NV_PGRAPH_PRI_GPCS_GPCCS_GPC_ACTIVITY2: 0x0 [ 1608.240504] __ga10b__ NV_PGRAPH_PRI_GPCS_GPCCS_GPC_ACTIVITY3: 0x0 [ 1608.246799] __ga10b__ NV_PGRAPH_PRI_GPCS_GPCCS_GPC_ACTIVITY4: 0x0 [ 1608.253098] __ga10b__ NV_PGRAPH_PRI_GPCS_TPCS_TPCCS_TPC_ACTIVITY0: 0x0 [ 1608.259845] __ga10b__ NV_PGRAPH_PRI_DS_MPIPE_STATUS: 0x0 [ 1608.265329] __ga10b__ NV_PGRAPH_PRI_FE_GO_IDLE_TIMEOUT : 0x800 [ 1608.271353] __ga10b__ NV_PGRAPH_PRI_FE_GO_IDLE_INFO : 0x1000710 [ 1608.277449] __ga10b__ NV_PGRAPH_PRI_GPC0_TPC0_TEX_M_TEX_SUBUNITS_STATUS: 0x0 [ 1608.284728] __ga10b__ NV_PGRAPH_PRI_CWD_FS: 0x702 [ 1608.289583] __ga10b__ NV_PGRAPH_PRI_FE_TPC_FS(0): 0xf7 [ 1608.294884] __ga10b__ NV_PGRAPH_PRI_CWD_GPC_TPC_ID: 0x1120213 [ 1608.300813] __ga10b__ NV_PGRAPH_PRI_CWD_SM_ID(0): 0x10306 [ 1608.306388] __ga10b__ NV_PGRAPH_PRI_FECS_CTXSW_STATUS_FE_0: 0x2 [ 1608.312513] __ga10b__ NV_PGRAPH_PRI_FECS_CTXSW_STATUS_1: 0x190 [ 1608.318529] __ga10b__ NV_PGRAPH_PRI_GPC0_GPCCS_CTXSW_STATUS_GPC_0: 0x0 [ 1608.325267] __ga10b__ NV_PGRAPH_PRI_GPC0_GPCCS_CTXSW_STATUS_1: 0x390 [ 1608.331830] __ga10b__ NV_PGRAPH_PRI_FECS_CTXSW_IDLESTATE : 0xf [ 1608.337867] __ga10b__ NV_PGRAPH_PRI_GPC0_GPCCS_CTXSW_IDLESTATE : 0xe [ 1608.344418] __ga10b__ NV_PGRAPH_PRI_FECS_CURRENT_CTX : 0xb013cdb1 [ 1608.350717] __ga10b__ NV_PGRAPH_PRI_FECS_NEW_CTX : 0xb01b354a [ 1608.356652] __ga10b__ NV_PGRAPH_PRI_FECS_HOST_INT_ENABLE : 0x7f0003 [ 1608.363129] __ga10b__ NV_PGRAPH_PRI_FECS_HOST_INT_STATUS : 0x0 [ 1608.369141] __ga10b__ NV_PGRAPH_PRI_GPCS_ROP0_CROP_STATUS1 : 0x5f00000 [ 1608.375868] __ga10b__ NV_PGRAPH_PRI_GPCS_ROPS_CROP_STATUS1 : 0x5f00000 [ 1608.382613] __ga10b__ NV_PGRAPH_PRI_GPCS_ROP0_ZROP_STATUS : 0x0 [ 1608.388732] __ga10b__ NV_PGRAPH_PRI_GPCS_ROP0_ZROP_STATUS2 : 0x0 [ 1608.394937] __ga10b__ NV_PGRAPH_PRI_GPCS_ROP1_ZROP_STATUS: 0x0 [ 1608.400967] __ga10b__ NV_PGRAPH_PRI_GPCS_ROP1_ZROP_STATUS2: 0x0 [ 1608.407087] __ga10b__ NV_PGRAPH_PRI_GPCS_ROPS_ZROP_STATUS : 0x0 [ 1608.413210] __ga10b__ NV_PGRAPH_PRI_GPCS_ROPS_ZROP_STATUS2 : 0x0 [ 1608.419416] __ga10b__ NV_PGRAPH_PRI_GPC0_GPCCS_GPC_EXCEPTION: 0x0 [ 1608.425709] __ga10b__ NV_PGRAPH_PRI_GPC0_GPCCS_GPC_EXCEPTION_EN: 0xfc0f6004 [ 1608.432896] __ga10b__ NV_PGRAPH_PRI_GPC0_TPC0_TPCCS_TPC_EXCEPTION: 0x0 [ 1608.439636] __ga10b__ NV_PGRAPH_PRI_GPC0_TPC0_TPCCS_TPC_EXCEPTION_EN: 0x16 [ 1608.446746] __ga10b__ NV_PGRAPH_PRI_GPCS_TPCS_SMS_HWW_GLOBAL_ESR_REPORT_MASK: 0x1174 [ 1608.454733] __ga10b__ NV_PGRAPH_PRI_GPCS_TPCS_SMS_HWW_WARP_ESR_REPORT_MASK: 0xe812b60 [ 1608.462803] __ga10b__ NV_PGRAPH_PRI_GPCS_TPCS_SMS_HWW_GLOBAL_ESR: 0x0 [ 1608.469454] __ga10b__ NV_PGRAPH_PRI_GPCS_TPCS_SMS_DBGR_CONTROL0: 0x1c00 [ 1608.476295] __ga10b__ NV_PGRAPH_PRI_GPCS_TPCS_SMS_DBGR_STATUS0: 0x0 [ 1608.482766] __ga10b__ NV_PGRAPH_PRI_GPCS_TPCS_SMS_DBGR_BPT_PAUSE_MASK_0: 0x0 [ 1608.490019] __ga10b__ NV_PGRAPH_PRI_GPCS_TPCS_SMS_DBGR_BPT_PAUSE_MASK_1: 0x0 [ 1608.497285] __ga10b__ NV_PGRAPH_PRI_GPC0_TPC0_SM0_HWW_WARP_ESR: 0x0 [ 1608.503756] __ga10b__ NV_PGRAPH_PRI_GPC0_TPC0_SM0_HWW_WARP_ESR_REPORT_MASK: 0xe812b60 [ 1608.511834] __ga10b__ NV_PGRAPH_PRI_GPC0_TPC0_SM0_HWW_GLOBAL_ESR: 0x0 [ 1608.518481] __ga10b__ NV_PGRAPH_PRI_GPC0_TPC0_SM0_HWW_GLOBAL_ESR_REPORT_MASK: 0x0 [ 1608.526215] __ga10b__ NV_PGRAPH_PRI_GPC0_TPC0_SM0_DBGR_CONTROL0: 0x1c00 [ 1608.533053] __ga10b__ NV_PGRAPH_PRI_GPC0_TPC0_SM0_DBGR_STATUS0: 0x0 [ 1608.539521] __ga10b__ NV_PGRAPH_PRI_GPC0_TPC0_SM1_HWW_WARP_ESR: 0x0 [ 1608.545987] __ga10b__ NV_PGRAPH_PRI_GPC0_TPC0_SM1_HWW_WARP_ESR_REPORT_MASK: 0x0 [ 1608.553528] __ga10b__ NV_PGRAPH_PRI_GPC0_TPC0_SM1_HWW_GLOBAL_ESR: 0x0 [ 1608.560177] __ga10b__ NV_PGRAPH_PRI_GPC0_TPC0_SM1_HWW_GLOBAL_ESR_REPORT_MASK: 0x1174 [ 1608.568165] __ga10b__ NV_PGRAPH_PRI_GPC0_TPC0_SM1_DBGR_CONTROL0: 0x1c00 [ 1608.574992] __ga10b__ NV_PGRAPH_PRI_GPC0_TPC0_SM1_DBGR_STATUS0: 0x0 [ 1608.581462] __ga10b__ NV_PGRAPH_PRI_GPC0_TPC1_SM0_HWW_WARP_ESR: 0x0 [ 1608.587917] __ga10b__ NV_PGRAPH_PRI_GPC0_TPC1_SM0_HWW_WARP_ESR_REPORT_MASK: 0xe812b60 [ 1608.595991] __ga10b__ NV_PGRAPH_PRI_GPC0_TPC1_SM0_HWW_GLOBAL_ESR: 0x0 [ 1608.602637] __ga10b__ NV_PGRAPH_PRI_GPC0_TPC1_SM0_HWW_GLOBAL_ESR_REPORT_MASK: 0x1174 [ 1608.610625] __ga10b__ NV_PGRAPH_PRI_GPC0_TPC1_SM0_DBGR_CONTROL0: 0x1c00 [ 1608.617459] __ga10b__ NV_PGRAPH_PRI_GPC0_TPC1_SM0_DBGR_STATUS0: 0x0 [ 1608.623917] __ga10b__ NV_PGRAPH_PRI_GPC0_TPC1_SM1_HWW_WARP_ESR: 0x0 [ 1608.630386] __ga10b__ NV_PGRAPH_PRI_GPC0_TPC1_SM1_HWW_WARP_ESR_REPORT_MASK: 0x0 [ 1608.637914] __ga10b__ NV_PGRAPH_PRI_GPC0_TPC1_SM1_HWW_GLOBAL_ESR: 0x0 [ 1608.644565] __ga10b__ NV_PGRAPH_PRI_GPC0_TPC1_SM1_HWW_GLOBAL_ESR_REPORT_MASK: 0x0 [ 1608.652289] __ga10b__ NV_PGRAPH_PRI_GPC0_TPC1_SM1_DBGR_CONTROL0: 0x1c00 [ 1608.659117] __ga10b__ NV_PGRAPH_PRI_GPC0_TPC1_SM1_DBGR_STATUS0: 0x0 [ 1608.665570] __ga10b__ NV_PGRAPH_PRI_GPC0_TPC2_SM0_HWW_WARP_ESR: 0x0 [ 1608.672022] __ga10b__ NV_PGRAPH_PRI_GPC0_TPC2_SM0_HWW_WARP_ESR_REPORT_MASK: 0xe812b60 [ 1608.680091] __ga10b__ NV_PGRAPH_PRI_GPC0_TPC2_SM0_HWW_GLOBAL_ESR: 0x0 [ 1608.686743] __ga10b__ NV_PGRAPH_PRI_GPC0_TPC2_SM0_HWW_GLOBAL_ESR_REPORT_MASK: 0x1174 [ 1608.694730] __ga10b__ NV_PGRAPH_PRI_GPC0_TPC2_SM0_DBGR_CONTROL0: 0x1c00 [ 1608.701561] __ga10b__ NV_PGRAPH_PRI_GPC0_TPC2_SM0_DBGR_STATUS0: 0x0 [ 1608.708045] __ga10b__ NV_PGRAPH_PRI_GPC0_TPC2_SM1_HWW_WARP_ESR: 0x0 [ 1608.714518] __ga10b__ NV_PGRAPH_PRI_GPC0_TPC2_SM1_HWW_WARP_ESR_REPORT_MASK: 0xe812b60 [ 1608.722598] __ga10b__ NV_PGRAPH_PRI_GPC0_TPC2_SM1_HWW_GLOBAL_ESR: 0x0 [ 1608.729246] __ga10b__ NV_PGRAPH_PRI_GPC0_TPC2_SM1_HWW_GLOBAL_ESR_REPORT_MASK: 0x0 [ 1608.736965] __ga10b__ NV_PGRAPH_PRI_GPC0_TPC2_SM1_DBGR_CONTROL0: 0x1c00 [ 1608.743801] __ga10b__ NV_PGRAPH_PRI_GPC0_TPC2_SM1_DBGR_STATUS0: 0x0 [ 1608.750273] __ga10b__ NV_PGRAPH_PRI_GPC1_TPC0_SM0_HWW_WARP_ESR: 0x0 [ 1608.756729] __ga10b__ NV_PGRAPH_PRI_GPC1_TPC0_SM0_HWW_WARP_ESR_REPORT_MASK: 0x781eb60 [ 1608.764805] __ga10b__ NV_PGRAPH_PRI_GPC1_TPC0_SM0_HWW_GLOBAL_ESR: 0x0 [ 1608.771454] __ga10b__ NV_PGRAPH_PRI_GPC1_TPC0_SM0_HWW_GLOBAL_ESR_REPORT_MASK: 0x1174 [ 1608.779439] __ga10b__ NV_PGRAPH_PRI_GPC1_TPC0_SM0_DBGR_CONTROL0: 0x1c00 [ 1608.786264] __ga10b__ NV_PGRAPH_PRI_GPC1_TPC0_SM0_DBGR_STATUS0: 0x0 [ 1608.792714] __ga10b__ NV_PGRAPH_PRI_GPC1_TPC0_SM1_HWW_WARP_ESR: 0x0 [ 1608.799170] __ga10b__ NV_PGRAPH_PRI_GPC1_TPC0_SM1_HWW_WARP_ESR_REPORT_MASK: 0x0 [ 1608.806704] __ga10b__ NV_PGRAPH_PRI_GPC1_TPC0_SM1_HWW_GLOBAL_ESR: 0x0 [ 1608.813360] __ga10b__ NV_PGRAPH_PRI_GPC1_TPC0_SM1_HWW_GLOBAL_ESR_REPORT_MASK: 0x1174 [ 1608.821345] __ga10b__ NV_PGRAPH_PRI_GPC1_TPC0_SM1_DBGR_CONTROL0: 0x1c00 [ 1608.828177] __ga10b__ NV_PGRAPH_PRI_GPC1_TPC0_SM1_DBGR_STATUS0: 0x0 [ 1608.834630] __ga10b__ NV_PGRAPH_PRI_GPC1_TPC1_SM0_HWW_WARP_ESR: 0x0 [ 1608.841098] __ga10b__ NV_PGRAPH_PRI_GPC1_TPC1_SM0_HWW_WARP_ESR_REPORT_MASK: 0x0 [ 1608.848637] __ga10b__ NV_PGRAPH_PRI_GPC1_TPC1_SM0_HWW_GLOBAL_ESR: 0x0 [ 1608.855287] __ga10b__ NV_PGRAPH_PRI_GPC1_TPC1_SM0_HWW_GLOBAL_ESR_REPORT_MASK: 0x1174 [ 1608.863273] __ga10b__ NV_PGRAPH_PRI_GPC1_TPC1_SM0_DBGR_CONTROL0: 0x1c00 [ 1608.870099] __ga10b__ NV_PGRAPH_PRI_GPC1_TPC1_SM0_DBGR_STATUS0: 0x0 [ 1608.876570] __ga10b__ NV_PGRAPH_PRI_GPC1_TPC1_SM1_HWW_WARP_ESR: 0x0 [ 1608.883038] __ga10b__ NV_PGRAPH_PRI_GPC1_TPC1_SM1_HWW_WARP_ESR_REPORT_MASK: 0x0 [ 1608.890580] __ga10b__ NV_PGRAPH_PRI_GPC1_TPC1_SM1_HWW_GLOBAL_ESR: 0x0 [ 1608.897241] __ga10b__ NV_PGRAPH_PRI_GPC1_TPC1_SM1_HWW_GLOBAL_ESR_REPORT_MASK: 0x1174 [ 1608.905227] __ga10b__ NV_PGRAPH_PRI_GPC1_TPC1_SM1_DBGR_CONTROL0: 0x1c00 [ 1608.912054] __ga10b__ NV_PGRAPH_PRI_GPC1_TPC1_SM1_DBGR_STATUS0: 0x0 [ 1608.918521] __ga10b__ NV_PGRAPH_PRI_GPC1_TPC2_SM0_HWW_WARP_ESR: 0x0 [ 1608.924975] __ga10b__ NV_PGRAPH_PRI_GPC1_TPC2_SM0_HWW_WARP_ESR_REPORT_MASK: 0xe812b60 [ 1608.933058] __ga10b__ NV_PGRAPH_PRI_GPC1_TPC2_SM0_HWW_GLOBAL_ESR: 0x0 [ 1608.939706] __ga10b__ NV_PGRAPH_PRI_GPC1_TPC2_SM0_HWW_GLOBAL_ESR_REPORT_MASK: 0x1174 [ 1608.947696] __ga10b__ NV_PGRAPH_PRI_GPC1_TPC2_SM0_DBGR_CONTROL0: 0x1c00 [ 1608.954521] __ga10b__ NV_PGRAPH_PRI_GPC1_TPC2_SM0_DBGR_STATUS0: 0x0 [ 1608.960990] __ga10b__ NV_PGRAPH_PRI_GPC1_TPC2_SM1_HWW_WARP_ESR: 0x0 [ 1608.967442] __ga10b__ NV_PGRAPH_PRI_GPC1_TPC2_SM1_HWW_WARP_ESR_REPORT_MASK: 0x0 [ 1608.974982] __ga10b__ NV_PGRAPH_PRI_GPC1_TPC2_SM1_HWW_GLOBAL_ESR: 0x0 [ 1608.981628] __ga10b__ NV_PGRAPH_PRI_GPC1_TPC2_SM1_HWW_GLOBAL_ESR_REPORT_MASK: 0x1174 [ 1608.989613] __ga10b__ NV_PGRAPH_PRI_GPC1_TPC2_SM1_DBGR_CONTROL0: 0x1c00 [ 1608.996450] __ga10b__ NV_PGRAPH_PRI_GPC1_TPC2_SM1_DBGR_STATUS0: 0x0 [ 1609.002918] __ga10b__ NV_PGRAPH_PRI_GPC1_TPC3_SM0_HWW_WARP_ESR: 0x0 [ 1609.009389] __ga10b__ NV_PGRAPH_PRI_GPC1_TPC3_SM0_HWW_WARP_ESR_REPORT_MASK: 0x0 [ 1609.016929] __ga10b__ NV_PGRAPH_PRI_GPC1_TPC3_SM0_HWW_GLOBAL_ESR: 0x0 [ 1609.023571] __ga10b__ NV_PGRAPH_PRI_GPC1_TPC3_SM0_HWW_GLOBAL_ESR_REPORT_MASK: 0x1174 [ 1609.031558] __ga10b__ NV_PGRAPH_PRI_GPC1_TPC3_SM0_DBGR_CONTROL0: 0x1c00 [ 1609.038383] __ga10b__ NV_PGRAPH_PRI_GPC1_TPC3_SM0_DBGR_STATUS0: 0x0 [ 1609.044855] __ga10b__ NV_PGRAPH_PRI_GPC1_TPC3_SM1_HWW_WARP_ESR: 0x0 [ 1609.051310] __ga10b__ NV_PGRAPH_PRI_GPC1_TPC3_SM1_HWW_WARP_ESR_REPORT_MASK: 0xe812b60 [ 1609.059391] __ga10b__ NV_PGRAPH_PRI_GPC1_TPC3_SM1_HWW_GLOBAL_ESR: 0x0 [ 1609.066027] __ga10b__ NV_PGRAPH_PRI_GPC1_TPC3_SM1_HWW_GLOBAL_ESR_REPORT_MASK: 0x1174 [ 1609.074020] __ga10b__ NV_PGRAPH_PRI_GPC1_TPC3_SM1_DBGR_CONTROL0: 0x1c00 [ 1609.080851] __ga10b__ NV_PGRAPH_PRI_GPC1_TPC3_SM1_DBGR_STATUS0: 0x0 [ 1609.087767] nvgpu: 17000000.ga10b nvgpu_tsg_force_reset_ch:469 [ERR] chid: 509 is not bound to tsg [ 1612.308631] cpufreq: cpu4,cur:2315000,set:2188800,set ndiv:171 [ 1616.807975] nvgpu: 17000000.ga10b ga10b_pbdma_handle_intr_0_legacy:437 [ERR] semaphore acquire timeout! [ 1616.818083] __ga10b__ Channel Status - chip ga10b [ 1616.818085] __ga10b__ --------------------------- [ 1616.822967] __ga10b__ 492-ga10b, TSG: 19, pid 3008, refs: 2, deterministic: no, domain name: (default) [ 1616.827835] __ga10b__ channel status: in use idle not busy [ 1616.837441] __ga10b__ RAMFC: TOP: 80000020068b7580 PUT: 0020068b7580 GET: 0020068b7580 FETCH: 000000000000 HEADER: 2140006c COUNT: 00000000 SEMAPHORE: addr 0020068a0000 payload 0000000000000000 execute 00100001 [ 1616.843199] __ga10b__ [ 1616.862475] __ga10b__ 493-ga10b, TSG: 18, pid 3008, refs: 2, deterministic: no, domain name: (default) [ 1616.864998] __ga10b__ channel status: in use idle not busy [ 1616.874591] __ga10b__ RAMFC: TOP: 8000002006078960 PUT: 002006078960 GET: 002006078960 FETCH: 000000000000 HEADER: 2140006c COUNT: 00000000 SEMAPHORE: addr 0020063c0000 payload 0000000000000000 execute 00100001 [ 1616.880330] __ga10b__ [ 1616.899579] __ga10b__ 494-ga10b, TSG: 17, pid 3008, refs: 2, deterministic: no, domain name: (default) [ 1616.902095] __ga10b__ channel status: in use idle not busy [ 1616.911681] __ga10b__ RAMFC: TOP: 8000002006064ee8 PUT: 002006064ee8 GET: 002006064ee8 FETCH: 000000000000 HEADER: 2140006c COUNT: 00000000 SEMAPHORE: addr 002006010000 payload 0000000000000000 execute 00100001 [ 1616.917422] __ga10b__ [ 1616.936666] __ga10b__ 495-ga10b, TSG: 16, pid 3008, refs: 2, deterministic: no, domain name: (default) [ 1616.939189] __ga10b__ channel status: in use idle not busy [ 1616.948791] __ga10b__ RAMFC: TOP: 8000002006004d30 PUT: 002006004d30 GET: 002006004d30 FETCH: 000000000000 HEADER: 2140006c COUNT: 00000000 SEMAPHORE: addr 002005d30000 payload 0000000000000000 execute 00100001 [ 1616.954528] __ga10b__ [ 1616.973767] __ga10b__ 496-ga10b, TSG: 15, pid 3090, refs: 2, deterministic: no, domain name: (default) [ 1616.976288] __ga10b__ channel status: in use idle not busy [ 1616.985871] __ga10b__ RAMFC: TOP: 80000020040080a0 PUT: 0020040080a0 GET: 0020040080a0 FETCH: 000000000000 HEADER: 2140006c COUNT: 00000000 SEMAPHORE: addr 002004010000 payload 0000000000000000 execute 00100001 [ 1616.991610] __ga10b__ [ 1617.010845] __ga10b__ 497-ga10b, TSG: 14, pid 3008, refs: 5, deterministic: no, domain name: (default) [ 1617.013361] __ga10b__ channel status: in use on_eng, pending, acquire_fail busy [ 1617.022948] __ga10b__ RAMFC: TOP: 8000001ffac60b64 PUT: 001ffac60b78 GET: 001ffac60b64 FETCH: 000000000000 HEADER: 2140006c COUNT: 11110000 SEMAPHORE: addr 002000040000 payload 00000000000195e9 execute 00081003 [ 1617.030570] __ga10b__ [ 1617.049794] __ga10b__ 498-ga10b, TSG: 13, pid 3008, refs: 2, deterministic: no, domain name: (default) [ 1617.052336] __ga10b__ channel status: in use idle not busy [ 1617.061914] __ga10b__ RAMFC: TOP: 8000002005a30168 PUT: 002005a30168 GET: 002005a30168 FETCH: 000000000000 HEADER: 2140006c COUNT: 00000000 SEMAPHORE: addr 002005a20000 payload 0000000000000000 execute 00100001 [ 1617.067654] __ga10b__ [ 1617.086909] __ga10b__ 499-ga10b, TSG: 12, pid 3008, refs: 2, deterministic: no, domain name: (default) [ 1617.089430] __ga10b__ channel status: in use idle not busy [ 1617.099019] __ga10b__ RAMFC: TOP: 80000020051f8208 PUT: 0020051f8208 GET: 0020051f8208 FETCH: 000000000000 HEADER: 2140006c COUNT: 00000000 SEMAPHORE: addr 002005640000 payload 0000000000000000 execute 00100001 [ 1617.104759] __ga10b__ [ 1617.123988] __ga10b__ 500-ga10b, TSG: 11, pid 3008, refs: 2, deterministic: no, domain name: (default) [ 1617.126515] __ga10b__ channel status: in use idle not busy [ 1617.136103] __ga10b__ RAMFC: TOP: 8000002005198230 PUT: 002005198230 GET: 002005198230 FETCH: 000000000000 HEADER: 2140006c COUNT: 00000000 SEMAPHORE: addr 0020051e0000 payload 0000000000000000 execute 00100001 [ 1617.141847] __ga10b__ [ 1617.161095] __ga10b__ 501-ga10b, TSG: 10, pid 3008, refs: 2, deterministic: no, domain name: (default) [ 1617.163628] __ga10b__ channel status: in use idle not busy [ 1617.173212] __ga10b__ RAMFC: TOP: 8000002005180208 PUT: 002005180208 GET: 002005180208 FETCH: 000000000000 HEADER: 2140006c COUNT: 00000000 SEMAPHORE: addr 002004eb0000 payload 0000000000000000 execute 00100001 [ 1617.178957] __ga10b__ [ 1617.198201] __ga10b__ 502-ga10b, TSG: 9, pid 3008, refs: 2, deterministic: no, domain name: (default) [ 1617.200728] __ga10b__ channel status: in use idle not busy [ 1617.210229] __ga10b__ RAMFC: TOP: 8000002004e88230 PUT: 002004e88230 GET: 002004e88230 FETCH: 000000000000 HEADER: 2140006c COUNT: 00000000 SEMAPHORE: addr 002004e90000 payload 0000000000000000 execute 00100001 [ 1617.215974] __ga10b__ [ 1617.235226] __ga10b__ 503-ga10b, TSG: 8, pid 3008, refs: 2, deterministic: no, domain name: (default) [ 1617.237764] __ga10b__ channel status: in use idle not busy [ 1617.247259] __ga10b__ RAMFC: TOP: 8000002004bb0208 PUT: 002004bb0208 GET: 002004bb0208 FETCH: 000000000000 HEADER: 2140006c COUNT: 00000000 SEMAPHORE: addr 002004ba0000 payload 0000000000000000 execute 00100001 [ 1617.253005] __ga10b__ [ 1617.272247] __ga10b__ 504-ga10b, TSG: 7, pid 3008, refs: 2, deterministic: no, domain name: (default) [ 1617.274773] __ga10b__ channel status: in use idle not busy [ 1617.284277] __ga10b__ RAMFC: TOP: 80000020046d0230 PUT: 0020046d0230 GET: 0020046d0230 FETCH: 000000000000 HEADER: 2140006c COUNT: 00000000 SEMAPHORE: addr 0020046c0000 payload 0000000000000000 execute 00100001 [ 1617.290021] __ga10b__ [ 1617.309274] __ga10b__ 505-ga10b, TSG: 6, pid 3008, refs: 2, deterministic: no, domain name: (default) [ 1617.311797] __ga10b__ channel status: in use idle not busy [ 1617.321295] __ga10b__ RAMFC: TOP: 8000002004318050 PUT: 002004318050 GET: 002004318050 FETCH: 000000000000 HEADER: 2140006c COUNT: 00000000 SEMAPHORE: addr 002004360000 payload 0000000000000000 execute 00100001 [ 1617.327037] __ga10b__ [ 1617.346279] __ga10b__ 506-ga10b, TSG: 5, pid 3008, refs: 2, deterministic: no, domain name: (default) [ 1617.348799] __ga10b__ channel status: in use idle not busy [ 1617.358297] __ga10b__ RAMFC: TOP: 8000002004305230 PUT: 002004305230 GET: 002004305230 FETCH: 000000000000 HEADER: 2140006c COUNT: 00000000 SEMAPHORE: addr 002004030000 payload 0000000000000000 execute 00100001 [ 1617.364041] __ga10b__ [ 1617.383281] __ga10b__ 507-ga10b, TSG: 4, pid 3008, refs: 2, deterministic: no, domain name: (default) [ 1617.385799] __ga10b__ channel status: in use idle not busy [ 1617.395304] __ga10b__ RAMFC: TOP: 80000020040080a0 PUT: 0020040080a0 GET: 0020040080a0 FETCH: 000000000000 HEADER: 2140006c COUNT: 00000000 SEMAPHORE: addr 002004010000 payload 0000000000000000 execute 00100001 [ 1617.401045] __ga10b__ [ 1617.420287] __ga10b__ 508-ga10b, TSG: 3, pid 2755, refs: 2, deterministic: no, domain name: (default) [ 1617.422810] __ga10b__ channel status: in use idle not busy [ 1617.432311] __ga10b__ RAMFC: TOP: 80000020040080a0 PUT: 0020040080a0 GET: 0020040080a0 FETCH: 000000000000 HEADER: 2140006c COUNT: 00000000 SEMAPHORE: addr 002004010000 payload 0000000000000000 execute 00100001 [ 1617.438056] __ga10b__ [ 1617.457311] __ga10b__ 509-ga10b, TSG: 2, pid 2334, refs: 6, deterministic: no, domain name: (default) [ 1617.459841] __ga10b__ channel status: in use on_pbdma, pbdma_busy busy [ 1617.469350] __ga10b__ RAMFC: TOP: 8000001fff3a065c PUT: 001fff3a0670 GET: 001fff3a065c FETCH: 000000000000 HEADER: 2140006c COUNT: 11110000 SEMAPHORE: addr 002000110000 payload 000000000001316e execute 00081003 [ 1617.476174] __ga10b__ [ 1617.495409] __ga10b__ 510-ga10b, TSG: 1, pid 2080, refs: 2, deterministic: no, domain name: (default) [ 1617.497947] __ga10b__ channel status: in use idle not busy [ 1617.507447] __ga10b__ RAMFC: TOP: 8000002004033e60 PUT: 002004033e60 GET: 002004033e60 FETCH: 000000000000 HEADER: 2140006c COUNT: 00000000 SEMAPHORE: addr 002004320000 payload 0000000000000000 execute 00000001 [ 1617.513183] __ga10b__ [ 1617.532431] __ga10b__ 511-ga10b, TSG: 0, pid 2080, refs: 3, deterministic: no, domain name: (default) [ 1617.534953] __ga10b__ channel status: in use pending busy [ 1617.544465] __ga10b__ RAMFC: TOP: 8000002004049448 PUT: 002004049448 GET: 002004049448 FETCH: 000000000000 HEADER: 2140006c COUNT: 00000000 SEMAPHORE: addr 002004020000 payload 0000000000000000 execute 00100001 [ 1617.550109] __ga10b__ [ 1617.569357] __ga10b__ PBDMA Status - chip ga10b [ 1617.571878] __ga10b__ ------------------------- [ 1617.576556] __ga10b__ pbdma 0: [ 1617.581229] __ga10b__ id: 2 - [tsg] next_id: - -1 [channel] | status: valid [ 1617.584389] __ga10b__ PBDMA_PUT 0000001fff3a0670 PBDMA_GET 0000001fff3a065c [ 1617.592101] __ga10b__ GP_PUT 000006e1 GP_GET 000006c7 FETCH 000006c7 HEADER 2140006c [ 1617.599459] __ga10b__ HDR 2001001b SHADOW0 fac60b54 SHADOW1 0000241f [ 1617.608331] __ga10b__ pbdma 1: [ 1617.615775] __ga10b__ id: -1 - [channel] next_id: - -1 [channel] | status: invalid [ 1617.618934] __ga10b__ PBDMA_PUT 00000020046d00a0 PBDMA_GET 00000020046d00a0 [ 1617.626916] __ga10b__ GP_PUT 0000000c GP_GET 0000000c FETCH 0000000c HEADER 2140006c [ 1617.634274] __ga10b__ HDR 2001001b SHADOW0 046d0078 SHADOW1 00002820 [ 1617.643150] __ga10b__ pbdma 2: [ 1617.650593] __ga10b__ id: -1 - [channel] next_id: - -1 [channel] | status: invalid [ 1617.653739] __ga10b__ PBDMA_PUT 0000002004318118 PBDMA_GET 0000002004318118 [ 1617.661716] __ga10b__ GP_PUT 0000001c GP_GET 0000001c FETCH 0000001c HEADER 2140006c [ 1617.669074] __ga10b__ HDR 2001001b SHADOW0 043180f0 SHADOW1 00002820 [ 1617.677941] __ga10b__ pbdma 3: [ 1617.685382] __ga10b__ id: -1 - [channel] next_id: - -1 [channel] | status: invalid [ 1617.688541] __ga10b__ PBDMA_PUT 00000067ef7f6fec PBDMA_GET 000000cd20b86794 [ 1617.696529] __ga10b__ GP_PUT 00000000 GP_GET 7e1826eb FETCH 00000000 HEADER e1955910 [ 1617.703895] __ga10b__ HDR 5f129d84 SHADOW0 61804464 SHADOW1 be122349 [ 1617.712762] __ga10b__ pbdma 4: [ 1617.720210] __ga10b__ id: -1 - [channel] next_id: - -1 [channel] | status: invalid [ 1617.723359] __ga10b__ PBDMA_PUT 0000000db7213608 PBDMA_GET 0000000db7213608 [ 1617.731336] __ga10b__ GP_PUT 00000000 GP_GET 6b8eb69b FETCH 00000000 HEADER 8086a79c [ 1617.738703] __ga10b__ HDR 145ec4b0 SHADOW0 bdc529ee SHADOW1 f4b979a2 [ 1617.747560] __ga10b__ pbdma 5: [ 1617.754991] __ga10b__ id: -1 - [channel] next_id: - -1 [channel] | status: invalid [ 1617.758150] __ga10b__ PBDMA_PUT 0000002004ea8118 PBDMA_GET 0000002004ea8118 [ 1617.766136] __ga10b__ GP_PUT 0000001c GP_GET 0000001c FETCH 0000001c HEADER 2140006c [ 1617.773497] __ga10b__ HDR 2001001b SHADOW0 04ea80f0 SHADOW1 00002820 [ 1617.782362] __ga10b__ [ 1617.789807] __ga10b__ ga10b eng 0: [ 1617.792325] __ga10b__ id: 14 (tsg), next_id: -1 (channel), ctx status: valid [ 1617.795926] __ga10b__ [ 1617.803285] __ga10b__ ga10b eng 1: [ 1617.805811] __ga10b__ id: -1 (channel), next_id: -1 (channel), ctx status: invalid [ 1617.809407] __ga10b__ [ 1617.817299] __ga10b__ ga10b eng 2: [ 1617.819820] __ga10b__ id: -1 (channel), next_id: -1 (channel), ctx status: invalid [ 1617.823420] __ga10b__ [ 1617.831310] __ga10b__ ga10b eng 3: [ 1617.833835] __ga10b__ id: -1 (channel), next_id: -1 (channel), ctx status: invalid [ 1617.837430] __ga10b__ [ 1617.845329] __ga10b__ ga10b eng 4: [ 1617.847851] __ga10b__ id: -1 (channel), next_id: -1 (channel), ctx status: invalid [ 1617.851457] __ga10b__ [ 1617.859353] __ga10b__ ga10b eng 5: [ 1617.861880] __ga10b__ id: -1 (channel), next_id: -1 (channel), ctx status: invalid [ 1617.865479] __ga10b__ [ 1617.873368] __ga10b__ [ 1617.875892] nvgpu: 17000000.ga10b report_pbdma_error:330 [ERR] pbdma_intr_0(0)= 0x04000000 [ 1617.888532] nvgpu: 17000000.ga10b nvgpu_cic_mon_report_err_safety_services:55 [ERR] Error reporting is not supported in this platform [ 1617.889527] nvgpu: 17000000.ga10b nvgpu_channel_recover_from_wdt:107 [ERR] Job on channel 509 timed out [ 1617.901168] nvgpu: 17000000.ga10b nvgpu_set_err_notifier_locked:149 [ERR] error notifier set to 24 for ch 509 [ 1617.912597] __ga10b__ NV_PGRAPH_STATUS: 0x0 [ 1617.927286] cpufreq: cpu4,cur:2020000,set:2188800,set ndiv:171 [ 1617.927914] __ga10b__ NV_PGRAPH_STATUS1: 0x0 [ 1617.933454] __ga10b__ NV_PGRAPH_ENGINE_STATUS: 0x1 [ 1617.938429] __ga10b__ NV_PGRAPH_GRFIFO_STATUS : 0x1010001 [ 1617.944003] __ga10b__ NV_PGRAPH_GRFIFO_CONTROL : 0x10001 [ 1617.949496] __ga10b__ NV_PGRAPH_PRI_FECS_HOST_INT_STATUS : 0x0 [ 1617.955729] __ga10b__ NV_PGRAPH_EXCEPTION : 0x0 [ 1617.960526] __ga10b__ NV_PGRAPH_FECS_INTR : 0x0 [ 1617.965303] __ga10b__ NV_PFIFO_ENGINE_STATUS(GR) : 0x80102010 [ 1617.971280] __ga10b__ NV_PGRAPH_ACTIVITY0: 0x140 [ 1617.976082] __ga10b__ NV_PGRAPH_ACTIVITY1: 0x0 [ 1617.980675] __ga10b__ NV_PGRAPH_ACTIVITY4: 0x0 [ 1617.985265] __ga10b__ NV_PGRAPH_PRI_SKED_ACTIVITY: 0x0 [ 1617.990563] __ga10b__ NV_PGRAPH_PRI_GPC0_GPCCS_GPC_ACTIVITY0: 0x0 [ 1617.996867] __ga10b__ NV_PGRAPH_PRI_GPC0_GPCCS_GPC_ACTIVITY1: 0x0 [ 1618.003166] __ga10b__ NV_PGRAPH_PRI_GPC0_GPCCS_GPC_ACTIVITY2: 0x0 [ 1618.009460] __ga10b__ NV_PGRAPH_PRI_GPC0_GPCCS_GPC_ACTIVITY3: 0x0 [ 1618.015755] __ga10b__ NV_PGRAPH_PRI_GPC0_GPCCS_GPC_ACTIVITY4: 0x0 [ 1618.022042] __ga10b__ NV_PGRAPH_PRI_GPC0_TPC0_TPCCS_TPC_ACTIVITY0: 0x0 [ 1618.028780] __ga10b__ NV_PGRAPH_PRI_GPC0_TPC1_TPCCS_TPC_ACTIVITY0: 0x0 [ 1618.035523] __ga10b__ NV_PGRAPH_PRI_GPC0_TPC2_TPCCS_TPC_ACTIVITY0: 0x0 [ 1618.042257] __ga10b__ NV_PGRAPH_PRI_GPCS_GPCCS_GPC_ACTIVITY0: 0x0 [ 1618.048563] __ga10b__ NV_PGRAPH_PRI_GPCS_GPCCS_GPC_ACTIVITY1: 0x0 [ 1618.054852] __ga10b__ NV_PGRAPH_PRI_GPCS_GPCCS_GPC_ACTIVITY2: 0x0 [ 1618.061154] __ga10b__ NV_PGRAPH_PRI_GPCS_GPCCS_GPC_ACTIVITY3: 0x0 [ 1618.067460] __ga10b__ NV_PGRAPH_PRI_GPCS_GPCCS_GPC_ACTIVITY4: 0x0 [ 1618.073745] __ga10b__ NV_PGRAPH_PRI_GPCS_TPCS_TPCCS_TPC_ACTIVITY0: 0x0 [ 1618.080484] __ga10b__ NV_PGRAPH_PRI_DS_MPIPE_STATUS: 0x0 [ 1618.085968] __ga10b__ NV_PGRAPH_PRI_FE_GO_IDLE_TIMEOUT : 0x1800 [ 1618.092084] __ga10b__ NV_PGRAPH_PRI_FE_GO_IDLE_INFO : 0x1000700 [ 1618.098206] __ga10b__ NV_PGRAPH_PRI_GPC0_TPC0_TEX_M_TEX_SUBUNITS_STATUS: 0x0 [ 1618.105491] __ga10b__ NV_PGRAPH_PRI_CWD_FS: 0x702 [ 1618.110357] __ga10b__ NV_PGRAPH_PRI_FE_TPC_FS(0): 0xf7 [ 1618.115678] __ga10b__ NV_PGRAPH_PRI_CWD_GPC_TPC_ID: 0x1120213 [ 1618.121611] __ga10b__ NV_PGRAPH_PRI_CWD_SM_ID(0): 0x10306 [ 1618.127176] __ga10b__ NV_PGRAPH_PRI_FECS_CTXSW_STATUS_FE_0: 0x3 [ 1618.133281] __ga10b__ NV_PGRAPH_PRI_FECS_CTXSW_STATUS_1: 0x190 [ 1618.139303] __ga10b__ NV_PGRAPH_PRI_GPC0_GPCCS_CTXSW_STATUS_GPC_0: 0x0 [ 1618.146058] __ga10b__ NV_PGRAPH_PRI_GPC0_GPCCS_CTXSW_STATUS_1: 0x390 [ 1618.152625] __ga10b__ NV_PGRAPH_PRI_FECS_CTXSW_IDLESTATE : 0xf [ 1618.158661] __ga10b__ NV_PGRAPH_PRI_GPC0_GPCCS_CTXSW_IDLESTATE : 0xf [ 1618.165222] __ga10b__ NV_PGRAPH_PRI_FECS_CURRENT_CTX : 0xb01bdaef [ 1618.171511] __ga10b__ NV_PGRAPH_PRI_FECS_NEW_CTX : 0xb01b4fc1 [ 1618.177439] __ga10b__ NV_PGRAPH_PRI_FECS_HOST_INT_ENABLE : 0x7f0003 [ 1618.183912] __ga10b__ NV_PGRAPH_PRI_FECS_HOST_INT_STATUS : 0x0 [ 1618.189925] __ga10b__ NV_PGRAPH_PRI_GPCS_ROP0_CROP_STATUS1 : 0x5f00000 [ 1618.196671] __ga10b__ NV_PGRAPH_PRI_GPCS_ROPS_CROP_STATUS1 : 0x5f00000 [ 1618.203412] __ga10b__ NV_PGRAPH_PRI_GPCS_ROP0_ZROP_STATUS : 0x0 [ 1618.209518] __ga10b__ NV_PGRAPH_PRI_GPCS_ROP0_ZROP_STATUS2 : 0x0 [ 1618.215710] __ga10b__ NV_PGRAPH_PRI_GPCS_ROP1_ZROP_STATUS: 0x0 [ 1618.221738] __ga10b__ NV_PGRAPH_PRI_GPCS_ROP1_ZROP_STATUS2: 0x0 [ 1618.227843] __ga10b__ NV_PGRAPH_PRI_GPCS_ROPS_ZROP_STATUS : 0x0 [ 1618.233960] __ga10b__ NV_PGRAPH_PRI_GPCS_ROPS_ZROP_STATUS2 : 0x0 [ 1618.240177] __ga10b__ NV_PGRAPH_PRI_GPC0_GPCCS_GPC_EXCEPTION: 0x0 [ 1618.246486] __ga10b__ NV_PGRAPH_PRI_GPC0_GPCCS_GPC_EXCEPTION_EN: 0xfc0f6004 [ 1618.253666] __ga10b__ NV_PGRAPH_PRI_GPC0_TPC0_TPCCS_TPC_EXCEPTION: 0x0 [ 1618.260411] __ga10b__ NV_PGRAPH_PRI_GPC0_TPC0_TPCCS_TPC_EXCEPTION_EN: 0x16 [ 1618.267513] __ga10b__ NV_PGRAPH_PRI_GPCS_TPCS_SMS_HWW_GLOBAL_ESR_REPORT_MASK: 0x1174 [ 1618.275501] __ga10b__ NV_PGRAPH_PRI_GPCS_TPCS_SMS_HWW_WARP_ESR_REPORT_MASK: 0x0 [ 1618.283049] __ga10b__ NV_PGRAPH_PRI_GPCS_TPCS_SMS_HWW_GLOBAL_ESR: 0x0 [ 1618.289694] __ga10b__ NV_PGRAPH_PRI_GPCS_TPCS_SMS_DBGR_CONTROL0: 0x1c00 [ 1618.296523] __ga10b__ NV_PGRAPH_PRI_GPCS_TPCS_SMS_DBGR_STATUS0: 0x0 [ 1618.302989] __ga10b__ NV_PGRAPH_PRI_GPCS_TPCS_SMS_DBGR_BPT_PAUSE_MASK_0: 0x0 [ 1618.310270] __ga10b__ NV_PGRAPH_PRI_GPCS_TPCS_SMS_DBGR_BPT_PAUSE_MASK_1: 0x0 [ 1618.317538] __ga10b__ NV_PGRAPH_PRI_GPC0_TPC0_SM0_HWW_WARP_ESR: 0x0 [ 1618.324010] __ga10b__ NV_PGRAPH_PRI_GPC0_TPC0_SM0_HWW_WARP_ESR_REPORT_MASK: 0x781eb60 [ 1618.332089] __ga10b__ NV_PGRAPH_PRI_GPC0_TPC0_SM0_HWW_GLOBAL_ESR: 0x0 [ 1618.338744] __ga10b__ NV_PGRAPH_PRI_GPC0_TPC0_SM0_HWW_GLOBAL_ESR_REPORT_MASK: 0x1174 [ 1618.346738] __ga10b__ NV_PGRAPH_PRI_GPC0_TPC0_SM0_DBGR_CONTROL0: 0x1c00 [ 1618.353572] __ga10b__ NV_PGRAPH_PRI_GPC0_TPC0_SM0_DBGR_STATUS0: 0x0 [ 1618.360059] __ga10b__ NV_PGRAPH_PRI_GPC0_TPC0_SM1_HWW_WARP_ESR: 0x0 [ 1618.366536] __ga10b__ NV_PGRAPH_PRI_GPC0_TPC0_SM1_HWW_WARP_ESR_REPORT_MASK: 0xe812b60 [ 1618.374620] __ga10b__ NV_PGRAPH_PRI_GPC0_TPC0_SM1_HWW_GLOBAL_ESR: 0x0 [ 1618.381265] __ga10b__ NV_PGRAPH_PRI_GPC0_TPC0_SM1_HWW_GLOBAL_ESR_REPORT_MASK: 0x0 [ 1618.388985] __ga10b__ NV_PGRAPH_PRI_GPC0_TPC0_SM1_DBGR_CONTROL0: 0x1c00 [ 1618.395819] __ga10b__ NV_PGRAPH_PRI_GPC0_TPC0_SM1_DBGR_STATUS0: 0x0 [ 1618.402292] __ga10b__ NV_PGRAPH_PRI_GPC0_TPC1_SM0_HWW_WARP_ESR: 0x0 [ 1618.408764] __ga10b__ NV_PGRAPH_PRI_GPC0_TPC1_SM0_HWW_WARP_ESR_REPORT_MASK: 0xe812b60 [ 1618.416838] __ga10b__ NV_PGRAPH_PRI_GPC0_TPC1_SM0_HWW_GLOBAL_ESR: 0x0 [ 1618.423491] __ga10b__ NV_PGRAPH_PRI_GPC0_TPC1_SM0_HWW_GLOBAL_ESR_REPORT_MASK: 0x1174 [ 1618.431481] __ga10b__ NV_PGRAPH_PRI_GPC0_TPC1_SM0_DBGR_CONTROL0: 0x1c00 [ 1618.438306] __ga10b__ NV_PGRAPH_PRI_GPC0_TPC1_SM0_DBGR_STATUS0: 0x0 [ 1618.444783] __ga10b__ NV_PGRAPH_PRI_GPC0_TPC1_SM1_HWW_WARP_ESR: 0x0 [ 1618.449166] cpufreq: cpu0,cur:897000,set:729600,set ndiv:57 [ 1618.451255] __ga10b__ NV_PGRAPH_PRI_GPC0_TPC1_SM1_HWW_WARP_ESR_REPORT_MASK: 0xe812b60 [ 1618.451930] cpufreq: cpu0,cur:1509000,set:2188800,set ndiv:171 [ 1618.459363] __ga10b__ NV_PGRAPH_PRI_GPC0_TPC1_SM1_HWW_GLOBAL_ESR: 0x0 [ 1618.466019] __ga10b__ NV_PGRAPH_PRI_GPC0_TPC1_SM1_HWW_GLOBAL_ESR_REPORT_MASK: 0x1174 [ 1618.474525] __ga10b__ NV_PGRAPH_PRI_GPC0_TPC1_SM1_DBGR_CONTROL0: 0x1c00 [ 1618.481865] __ga10b__ NV_PGRAPH_PRI_GPC0_TPC1_SM1_DBGR_STATUS0: 0x0 [ 1618.488341] __ga10b__ NV_PGRAPH_PRI_GPC0_TPC2_SM0_HWW_WARP_ESR: 0x0 [ 1618.495348] __ga10b__ NV_PGRAPH_PRI_GPC0_TPC2_SM0_HWW_WARP_ESR_REPORT_MASK: 0x0 [ 1618.503391] __ga10b__ NV_PGRAPH_PRI_GPC0_TPC2_SM0_HWW_GLOBAL_ESR: 0x0 [ 1618.510059] __ga10b__ NV_PGRAPH_PRI_GPC0_TPC2_SM0_HWW_GLOBAL_ESR_REPORT_MASK: 0x0 [ 1618.517783] __ga10b__ NV_PGRAPH_PRI_GPC0_TPC2_SM0_DBGR_CONTROL0: 0x1c00 [ 1618.524621] __ga10b__ NV_PGRAPH_PRI_GPC0_TPC2_SM0_DBGR_STATUS0: 0x0 [ 1618.531091] __ga10b__ NV_PGRAPH_PRI_GPC0_TPC2_SM1_HWW_WARP_ESR: 0x0 [ 1618.537562] __ga10b__ NV_PGRAPH_PRI_GPC0_TPC2_SM1_HWW_WARP_ESR_REPORT_MASK: 0xe812b60 [ 1618.545640] __ga10b__ NV_PGRAPH_PRI_GPC0_TPC2_SM1_HWW_GLOBAL_ESR: 0x0 [ 1618.552286] __ga10b__ NV_PGRAPH_PRI_GPC0_TPC2_SM1_HWW_GLOBAL_ESR_REPORT_MASK: 0x0 [ 1618.560001] __ga10b__ NV_PGRAPH_PRI_GPC0_TPC2_SM1_DBGR_CONTROL0: 0x1c00 [ 1618.566840] __ga10b__ NV_PGRAPH_PRI_GPC0_TPC2_SM1_DBGR_STATUS0: 0x0 [ 1618.573320] __ga10b__ NV_PGRAPH_PRI_GPC1_TPC0_SM0_HWW_WARP_ESR: 0x0 [ 1618.579799] __ga10b__ NV_PGRAPH_PRI_GPC1_TPC0_SM0_HWW_WARP_ESR_REPORT_MASK: 0xe812b60 [ 1618.587869] __ga10b__ NV_PGRAPH_PRI_GPC1_TPC0_SM0_HWW_GLOBAL_ESR: 0x0 [ 1618.594507] __ga10b__ NV_PGRAPH_PRI_GPC1_TPC0_SM0_HWW_GLOBAL_ESR_REPORT_MASK: 0x0 [ 1618.602227] __ga10b__ NV_PGRAPH_PRI_GPC1_TPC0_SM0_DBGR_CONTROL0: 0x1c00 [ 1618.609065] __ga10b__ NV_PGRAPH_PRI_GPC1_TPC0_SM0_DBGR_STATUS0: 0x0 [ 1618.615525] __ga10b__ NV_PGRAPH_PRI_GPC1_TPC0_SM1_HWW_WARP_ESR: 0x0 [ 1618.622004] __ga10b__ NV_PGRAPH_PRI_GPC1_TPC0_SM1_HWW_WARP_ESR_REPORT_MASK: 0xe812b60 [ 1618.630095] __ga10b__ NV_PGRAPH_PRI_GPC1_TPC0_SM1_HWW_GLOBAL_ESR: 0x0 [ 1618.636746] __ga10b__ NV_PGRAPH_PRI_GPC1_TPC0_SM1_HWW_GLOBAL_ESR_REPORT_MASK: 0x1174 [ 1618.644739] __ga10b__ NV_PGRAPH_PRI_GPC1_TPC0_SM1_DBGR_CONTROL0: 0x1c00 [ 1618.651566] __ga10b__ NV_PGRAPH_PRI_GPC1_TPC0_SM1_DBGR_STATUS0: 0x0 [ 1618.658037] __ga10b__ NV_PGRAPH_PRI_GPC1_TPC1_SM0_HWW_WARP_ESR: 0x0 [ 1618.664520] __ga10b__ NV_PGRAPH_PRI_GPC1_TPC1_SM0_HWW_WARP_ESR_REPORT_MASK: 0xe812b60 [ 1618.672597] __ga10b__ NV_PGRAPH_PRI_GPC1_TPC1_SM0_HWW_GLOBAL_ESR: 0x0 [ 1618.679232] __ga10b__ NV_PGRAPH_PRI_GPC1_TPC1_SM0_HWW_GLOBAL_ESR_REPORT_MASK: 0x0 [ 1618.686959] __ga10b__ NV_PGRAPH_PRI_GPC1_TPC1_SM0_DBGR_CONTROL0: 0x1c00 [ 1618.693792] __ga10b__ NV_PGRAPH_PRI_GPC1_TPC1_SM0_DBGR_STATUS0: 0x0 [ 1618.700269] __ga10b__ NV_PGRAPH_PRI_GPC1_TPC1_SM1_HWW_WARP_ESR: 0x0 [ 1618.706723] __ga10b__ NV_PGRAPH_PRI_GPC1_TPC1_SM1_HWW_WARP_ESR_REPORT_MASK: 0xe812b60 [ 1618.714795] __ga10b__ NV_PGRAPH_PRI_GPC1_TPC1_SM1_HWW_GLOBAL_ESR: 0x0 [ 1618.721445] __ga10b__ NV_PGRAPH_PRI_GPC1_TPC1_SM1_HWW_GLOBAL_ESR_REPORT_MASK: 0x1174 [ 1618.729440] __ga10b__ NV_PGRAPH_PRI_GPC1_TPC1_SM1_DBGR_CONTROL0: 0x1c00 [ 1618.736252] __ga10b__ NV_PGRAPH_PRI_GPC1_TPC1_SM1_DBGR_STATUS0: 0x0 [ 1618.742717] __ga10b__ NV_PGRAPH_PRI_GPC1_TPC2_SM0_HWW_WARP_ESR: 0x0 [ 1618.749189] __ga10b__ NV_PGRAPH_PRI_GPC1_TPC2_SM0_HWW_WARP_ESR_REPORT_MASK: 0xe812b60 [ 1618.757266] __ga10b__ NV_PGRAPH_PRI_GPC1_TPC2_SM0_HWW_GLOBAL_ESR: 0x0 [ 1618.763922] __ga10b__ NV_PGRAPH_PRI_GPC1_TPC2_SM0_HWW_GLOBAL_ESR_REPORT_MASK: 0x1174 [ 1618.771909] __ga10b__ NV_PGRAPH_PRI_GPC1_TPC2_SM0_DBGR_CONTROL0: 0x1c00 [ 1618.778736] __ga10b__ NV_PGRAPH_PRI_GPC1_TPC2_SM0_DBGR_STATUS0: 0x0 [ 1618.785211] __ga10b__ NV_PGRAPH_PRI_GPC1_TPC2_SM1_HWW_WARP_ESR: 0x0 [ 1618.791684] __ga10b__ NV_PGRAPH_PRI_GPC1_TPC2_SM1_HWW_WARP_ESR_REPORT_MASK: 0xe812b60 [ 1618.799763] __ga10b__ NV_PGRAPH_PRI_GPC1_TPC2_SM1_HWW_GLOBAL_ESR: 0x0 [ 1618.806412] __ga10b__ NV_PGRAPH_PRI_GPC1_TPC2_SM1_HWW_GLOBAL_ESR_REPORT_MASK: 0x1174 [ 1618.814399] __ga10b__ NV_PGRAPH_PRI_GPC1_TPC2_SM1_DBGR_CONTROL0: 0x1c00 [ 1618.821231] __ga10b__ NV_PGRAPH_PRI_GPC1_TPC2_SM1_DBGR_STATUS0: 0x0 [ 1618.827709] __ga10b__ NV_PGRAPH_PRI_GPC1_TPC3_SM0_HWW_WARP_ESR: 0x0 [ 1618.834178] __ga10b__ NV_PGRAPH_PRI_GPC1_TPC3_SM0_HWW_WARP_ESR_REPORT_MASK: 0xe812b60 [ 1618.842260] __ga10b__ NV_PGRAPH_PRI_GPC1_TPC3_SM0_HWW_GLOBAL_ESR: 0x0 [ 1618.848922] __ga10b__ NV_PGRAPH_PRI_GPC1_TPC3_SM0_HWW_GLOBAL_ESR_REPORT_MASK: 0x1174 [ 1618.856912] __ga10b__ NV_PGRAPH_PRI_GPC1_TPC3_SM0_DBGR_CONTROL0: 0x1c00 [ 1618.863729] __ga10b__ NV_PGRAPH_PRI_GPC1_TPC3_SM0_DBGR_STATUS0: 0x0 [ 1618.870199] __ga10b__ NV_PGRAPH_PRI_GPC1_TPC3_SM1_HWW_WARP_ESR: 0x0 [ 1618.876673] __ga10b__ NV_PGRAPH_PRI_GPC1_TPC3_SM1_HWW_WARP_ESR_REPORT_MASK: 0xe812b60 [ 1618.884774] __ga10b__ NV_PGRAPH_PRI_GPC1_TPC3_SM1_HWW_GLOBAL_ESR: 0x0 [ 1618.891432] __ga10b__ NV_PGRAPH_PRI_GPC1_TPC3_SM1_HWW_GLOBAL_ESR_REPORT_MASK: 0x0 [ 1618.899157] __ga10b__ NV_PGRAPH_PRI_GPC1_TPC3_SM1_DBGR_CONTROL0: 0x1c00 [ 1618.905980] __ga10b__ NV_PGRAPH_PRI_GPC1_TPC3_SM1_DBGR_STATUS0: 0x0 [ 1618.912812] nvgpu: 17000000.ga10b nvgpu_tsg_force_reset_ch:469 [ERR] chid: 509 is not bound to tsg [ 1622.735670] cpufreq: cpu0,cur:2315000,set:2188800,set ndiv:171 [ 1622.736280] cpufreq: cpu4,cur:1392000,set:1574400,set ndiv:123 [ 1626.711057] nvgpu: 17000000.ga10b ga10b_pbdma_handle_intr_0_legacy:437 [ERR] semaphore acquire timeout! [ 1626.721119] __ga10b__ Channel Status - chip ga10b [ 1626.721120] __ga10b__ --------------------------- [ 1626.725991] __ga10b__ 492-ga10b, TSG: 19, pid 3008, refs: 2, deterministic: no, domain name: (default) [ 1626.730870] __ga10b__ channel status: in use idle not busy [ 1626.740544] __ga10b__ RAMFC: TOP: 80000020068b3ed0 PUT: 0020068b3ed0 GET: 0020068b3ed0 FETCH: 000000000000 HEADER: 2140006c COUNT: 00000000 SEMAPHORE: addr 0020068a0000 payload 0000000000000000 execute 00100001 [ 1626.746319] __ga10b__ [ 1626.765607] __ga10b__ 493-ga10b, TSG: 18, pid 3008, refs: 2, deterministic: no, domain name: (default) [ 1626.768132] __ga10b__ channel status: in use idle not busy [ 1626.777732] __ga10b__ RAMFC: TOP: 800000200607e338 PUT: 00200607e338 GET: 00200607e338 FETCH: 000000000000 HEADER: 2140006c COUNT: 00000000 SEMAPHORE: addr 0020063c0000 payload 0000000000000000 execute 00100001 [ 1626.783538] __ga10b__ [ 1626.802845] __ga10b__ 494-ga10b, TSG: 17, pid 3008, refs: 2, deterministic: no, domain name: (default) [ 1626.805396] __ga10b__ channel status: in use idle not busy [ 1626.815053] __ga10b__ RAMFC: TOP: 8000002006061bf8 PUT: 002006061bf8 GET: 002006061bf8 FETCH: 000000000000 HEADER: 2140006c COUNT: 00000000 SEMAPHORE: addr 002006010000 payload 0000000000000000 execute 00100001 [ 1626.820831] __ga10b__ [ 1626.840117] __ga10b__ 495-ga10b, TSG: 16, pid 3008, refs: 2, deterministic: no, domain name: (default) [ 1626.842670] __ga10b__ channel status: in use idle not busy [ 1626.852288] __ga10b__ RAMFC: TOP: 8000002006003250 PUT: 002006003250 GET: 002006003250 FETCH: 000000000000 HEADER: 2140006c COUNT: 00000000 SEMAPHORE: addr 002005d30000 payload 0000000000000000 execute 00100001 [ 1626.858035] __ga10b__ [ 1626.877258] __ga10b__ 496-ga10b, TSG: 15, pid 3090, refs: 2, deterministic: no, domain name: (default) [ 1626.879782] __ga10b__ channel status: in use idle not busy [ 1626.889369] __ga10b__ RAMFC: TOP: 80000020040080a0 PUT: 0020040080a0 GET: 0020040080a0 FETCH: 000000000000 HEADER: 2140006c COUNT: 00000000 SEMAPHORE: addr 002004010000 payload 0000000000000000 execute 00100001 [ 1626.895106] __ga10b__ [ 1626.914347] __ga10b__ 497-ga10b, TSG: 14, pid 3008, refs: 5, deterministic: no, domain name: (default) [ 1626.916884] __ga10b__ channel status: in use pending, acquire_fail busy [ 1626.926463] __ga10b__ RAMFC: TOP: 8000001ffac6a2b4 PUT: 001ffac6a2c8 GET: 001ffac6a2b4 FETCH: 000000000000 HEADER: 2140006c COUNT: 11110000 SEMAPHORE: addr 002000040000 payload 000000000001983b execute 00081003 [ 1626.933378] __ga10b__ [ 1626.952617] __ga10b__ 498-ga10b, TSG: 13, pid 3008, refs: 2, deterministic: no, domain name: (default) [ 1626.955145] __ga10b__ channel status: in use idle not busy [ 1626.964735] __ga10b__ RAMFC: TOP: 8000002005a30168 PUT: 002005a30168 GET: 002005a30168 FETCH: 000000000000 HEADER: 2140006c COUNT: 00000000 SEMAPHORE: addr 002005a20000 payload 0000000000000000 execute 00100001 [ 1626.970479] __ga10b__ [ 1626.989715] __ga10b__ 499-ga10b, TSG: 12, pid 3008, refs: 2, deterministic: no, domain name: (default) [ 1626.992240] __ga10b__ channel status: in use idle not busy [ 1627.001835] __ga10b__ RAMFC: TOP: 80000020051f8208 PUT: 0020051f8208 GET: 0020051f8208 FETCH: 000000000000 HEADER: 2140006c COUNT: 00000000 SEMAPHORE: addr 002005640000 payload 0000000000000000 execute 00100001 [ 1627.007577] __ga10b__ [ 1627.026805] __ga10b__ 500-ga10b, TSG: 11, pid 3008, refs: 2, deterministic: no, domain name: (default) [ 1627.029321] __ga10b__ channel status: in use idle not busy [ 1627.038904] __ga10b__ RAMFC: TOP: 8000002005198230 PUT: 002005198230 GET: 002005198230 FETCH: 000000000000 HEADER: 2140006c COUNT: 00000000 SEMAPHORE: addr 0020051e0000 payload 0000000000000000 execute 00100001 [ 1627.044649] __ga10b__ [ 1627.063885] __ga10b__ 501-ga10b, TSG: 10, pid 3008, refs: 2, deterministic: no, domain name: (default) [ 1627.066429] __ga10b__ channel status: in use idle not busy [ 1627.076012] __ga10b__ RAMFC: TOP: 8000002005180208 PUT: 002005180208 GET: 002005180208 FETCH: 000000000000 HEADER: 2140006c COUNT: 00000000 SEMAPHORE: addr 002004eb0000 payload 0000000000000000 execute 00100001 [ 1627.081754] __ga10b__ [ 1627.101009] __ga10b__ 502-ga10b, TSG: 9, pid 3008, refs: 2, deterministic: no, domain name: (default) [ 1627.103538] __ga10b__ channel status: in use idle not busy [ 1627.113036] __ga10b__ RAMFC: TOP: 8000002004e88230 PUT: 002004e88230 GET: 002004e88230 FETCH: 000000000000 HEADER: 2140006c COUNT: 00000000 SEMAPHORE: addr 002004e90000 payload 0000000000000000 execute 00100001 [ 1627.118780] __ga10b__ [ 1627.138015] __ga10b__ 503-ga10b, TSG: 8, pid 3008, refs: 2, deterministic: no, domain name: (default) [ 1627.140538] __ga10b__ channel status: in use idle not busy [ 1627.150045] __ga10b__ RAMFC: TOP: 8000002004bb0208 PUT: 002004bb0208 GET: 002004bb0208 FETCH: 000000000000 HEADER: 2140006c COUNT: 00000000 SEMAPHORE: addr 002004ba0000 payload 0000000000000000 execute 00100001 [ 1627.155789] __ga10b__ [ 1627.175035] __ga10b__ 504-ga10b, TSG: 7, pid 3008, refs: 2, deterministic: no, domain name: (default) [ 1627.177562] __ga10b__ channel status: in use idle not busy [ 1627.187066] __ga10b__ RAMFC: TOP: 80000020046d0230 PUT: 0020046d0230 GET: 0020046d0230 FETCH: 000000000000 HEADER: 2140006c COUNT: 00000000 SEMAPHORE: addr 0020046c0000 payload 0000000000000000 execute 00100001 [ 1627.192821] __ga10b__ [ 1627.212062] __ga10b__ 505-ga10b, TSG: 6, pid 3008, refs: 2, deterministic: no, domain name: (default) [ 1627.214589] __ga10b__ channel status: in use idle not busy [ 1627.224088] __ga10b__ RAMFC: TOP: 8000002004318050 PUT: 002004318050 GET: 002004318050 FETCH: 000000000000 HEADER: 2140006c COUNT: 00000000 SEMAPHORE: addr 002004360000 payload 0000000000000000 execute 00100001 [ 1627.229826] __ga10b__ [ 1627.249068] __ga10b__ 506-ga10b, TSG: 5, pid 3008, refs: 2, deterministic: no, domain name: (default) [ 1627.251589] __ga10b__ channel status: in use idle not busy [ 1627.261089] __ga10b__ RAMFC: TOP: 8000002004305230 PUT: 002004305230 GET: 002004305230 FETCH: 000000000000 HEADER: 2140006c COUNT: 00000000 SEMAPHORE: addr 002004030000 payload 0000000000000000 execute 00100001 [ 1627.266835] __ga10b__ [ 1627.286072] __ga10b__ 507-ga10b, TSG: 4, pid 3008, refs: 2, deterministic: no, domain name: (default) [ 1627.288596] __ga10b__ channel status: in use idle not busy [ 1627.298095] __ga10b__ RAMFC: TOP: 80000020040080a0 PUT: 0020040080a0 GET: 0020040080a0 FETCH: 000000000000 HEADER: 2140006c COUNT: 00000000 SEMAPHORE: addr 002004010000 payload 0000000000000000 execute 00100001 [ 1627.303835] __ga10b__ [ 1627.323070] __ga10b__ 508-ga10b, TSG: 3, pid 2755, refs: 2, deterministic: no, domain name: (default) [ 1627.325593] __ga10b__ channel status: in use idle not busy [ 1627.335087] __ga10b__ RAMFC: TOP: 80000020040080a0 PUT: 0020040080a0 GET: 0020040080a0 FETCH: 000000000000 HEADER: 2140006c COUNT: 00000000 SEMAPHORE: addr 002004010000 payload 0000000000000000 execute 00100001 [ 1627.340835] __ga10b__ [ 1627.360078] __ga10b__ 509-ga10b, TSG: 2, pid 2334, refs: 6, deterministic: no, domain name: (default) [ 1627.362596] __ga10b__ channel status: in use on_pbdma, pbdma_busy busy [ 1627.372096] __ga10b__ RAMFC: TOP: 8000001fff3a065c PUT: 001fff3a0670 GET: 001fff3a065c FETCH: 000000000000 HEADER: 2140006c COUNT: 11110000 SEMAPHORE: addr 002000110000 payload 0000000000013170 execute 00081003 [ 1627.378921] __ga10b__ [ 1627.398169] __ga10b__ 510-ga10b, TSG: 1, pid 2080, refs: 2, deterministic: no, domain name: (default) [ 1627.400692] __ga10b__ channel status: in use idle not busy [ 1627.410186] __ga10b__ RAMFC: TOP: 80000020040330f0 PUT: 0020040330f0 GET: 0020040330f0 FETCH: 000000000000 HEADER: 2140006c COUNT: 00000000 SEMAPHORE: addr 002004320000 payload 0000000000000000 execute 00000001 [ 1627.415929] __ga10b__ [ 1627.435163] __ga10b__ 511-ga10b, TSG: 0, pid 2080, refs: 2, deterministic: no, domain name: (default) [ 1627.437691] __ga10b__ channel status: in use on_eng not busy [ 1627.447187] __ga10b__ RAMFC: TOP: 800000200404c378 PUT: 00200404c378 GET: 00200404c378 FETCH: 000000000000 HEADER: 2140006c COUNT: 00000000 SEMAPHORE: addr 002004020000 payload 0000000000000000 execute 00100001 [ 1627.453099] __ga10b__ [ 1627.472353] __ga10b__ PBDMA Status - chip ga10b [ 1627.474869] __ga10b__ ------------------------- [ 1627.479542] __ga10b__ pbdma 0: [ 1627.484217] __ga10b__ id: 2 - [tsg] next_id: - -1 [channel] | status: valid [ 1627.487370] __ga10b__ PBDMA_PUT 0000001fff3a0670 PBDMA_GET 0000001fff3a065c [ 1627.495097] __ga10b__ GP_PUT 000006d5 GP_GET 000006bb FETCH 000006bb HEADER 2140006c [ 1627.502456] __ga10b__ HDR 2001001b SHADOW0 0404c350 SHADOW1 00002820 [ 1627.511329] __ga10b__ pbdma 1: [ 1627.518771] __ga10b__ id: -1 - [channel] next_id: - -1 [channel] | status: invalid [ 1627.521933] __ga10b__ PBDMA_PUT 00000020046d00a0 PBDMA_GET 00000020046d00a0 [ 1627.529912] __ga10b__ GP_PUT 0000000c GP_GET 0000000c FETCH 0000000c HEADER 2140006c [ 1627.537257] __ga10b__ HDR 2001001b SHADOW0 046d0078 SHADOW1 00002820 [ 1627.546125] __ga10b__ pbdma 2: [ 1627.553578] __ga10b__ id: -1 - [channel] next_id: - -1 [channel] | status: invalid [ 1627.556738] __ga10b__ PBDMA_PUT 0000002004318118 PBDMA_GET 0000002004318118 [ 1627.564725] __ga10b__ GP_PUT 0000001c GP_GET 0000001c FETCH 0000001c HEADER 2140006c [ 1627.572086] __ga10b__ HDR 2001001b SHADOW0 043180f0 SHADOW1 00002820 [ 1627.580956] __ga10b__ pbdma 3: [ 1627.588397] __ga10b__ id: -1 - [channel] next_id: - -1 [channel] | status: invalid [ 1627.591558] __ga10b__ PBDMA_PUT 00000067ef7f6fec PBDMA_GET 000000cd20b86794 [ 1627.599545] __ga10b__ GP_PUT 00000000 GP_GET 7e1826eb FETCH 00000000 HEADER e1955910 [ 1627.606905] __ga10b__ HDR 5f129d84 SHADOW0 61804464 SHADOW1 be122349 [ 1627.615771] __ga10b__ pbdma 4: [ 1627.623227] __ga10b__ id: -1 - [channel] next_id: - -1 [channel] | status: invalid [ 1627.626386] __ga10b__ PBDMA_PUT 0000000db7213608 PBDMA_GET 0000000db7213608 [ 1627.634380] __ga10b__ GP_PUT 00000000 GP_GET 6b8eb69b FETCH 00000000 HEADER 8086a79c [ 1627.641742] __ga10b__ HDR 145ec4b0 SHADOW0 bdc529ee SHADOW1 f4b979a2 [ 1627.650615] __ga10b__ pbdma 5: [ 1627.658065] __ga10b__ id: -1 - [channel] next_id: - -1 [channel] | status: invalid [ 1627.661230] __ga10b__ PBDMA_PUT 0000002004ea8118 PBDMA_GET 0000002004ea8118 [ 1627.669218] __ga10b__ GP_PUT 0000001c GP_GET 0000001c FETCH 0000001c HEADER 2140006c [ 1627.676573] __ga10b__ HDR 2001001b SHADOW0 04ea80f0 SHADOW1 00002820 [ 1627.685445] __ga10b__ [ 1627.686184] nvgpu: 17000000.ga10b nvgpu_channel_recover_from_wdt:107 [ERR] Job on channel 509 timed out [ 1627.692899] __ga10b__ ga10b eng 0: [ 1627.696175] __ga10b__ NV_PGRAPH_STATUS: 0x0 [ 1627.705543] __ga10b__ id: 0 (tsg), next_id: -1 (channel), ctx status: valid [ 1627.705545] __ga10b__ [ 1627.709242] __ga10b__ NV_PGRAPH_STATUS1: 0x0 [ 1627.713456] __ga10b__ ga10b eng 1: [ 1627.713458] __ga10b__ id: 0 (tsg), next_id: -1 (channel), ctx status: valid [ 1627.713458] __ga10b__ [ 1627.713462] __ga10b__ ga10b eng 2: [ 1627.713464] __ga10b__ id: -1 (channel), next_id: -1 (channel), ctx status: invalid [ 1627.713465] __ga10b__ [ 1627.713469] __ga10b__ ga10b eng 3: [ 1627.720801] __ga10b__ NV_PGRAPH_ENGINE_STATUS: 0x0 [ 1627.723249] __ga10b__ id: -1 (channel), next_id: -1 (channel), ctx status: invalid [ 1627.723249] __ga10b__ [ 1627.723253] __ga10b__ ga10b eng 4: [ 1627.723254] __ga10b__ id: -1 (channel), next_id: -1 (channel), ctx status: invalid [ 1627.723254] __ga10b__ [ 1627.723260] __ga10b__ ga10b eng 5: [ 1627.723261] __ga10b__ id: -1 (channel), next_id: -1 (channel), ctx status: invalid [ 1627.727656] __ga10b__ NV_PGRAPH_GRFIFO_STATUS : 0x8080001 [ 1627.727658] __ga10b__ NV_PGRAPH_GRFIFO_CONTROL : 0x10001 [ 1627.727661] __ga10b__ NV_PGRAPH_PRI_FECS_HOST_INT_STATUS : 0x0 [ 1627.731257] __ga10b__ [ 1627.738514] __ga10b__ NV_PGRAPH_EXCEPTION : 0x0 [ 1627.738516] __ga10b__ NV_PGRAPH_FECS_INTR : 0x0 [ 1627.738520] __ga10b__ NV_PFIFO_ENGINE_STATUS(GR) : 0x2000 [ 1627.741051] __ga10b__ [ 1627.744646] __ga10b__ NV_PGRAPH_ACTIVITY0: 0x0 [ 1627.752532] nvgpu: 17000000.ga10b report_pbdma_error:330 [ERR] pbdma_intr_0(0)= 0x04000000 [ 1627.755060] __ga10b__ NV_PGRAPH_ACTIVITY1: 0x0 [ 1627.755062] __ga10b__ NV_PGRAPH_ACTIVITY4: 0x0 [ 1627.755064] __ga10b__ NV_PGRAPH_PRI_SKED_ACTIVITY: 0x0 [ 1627.755068] __ga10b__ NV_PGRAPH_PRI_GPC0_GPCCS_GPC_ACTIVITY0: 0x0 [ 1627.758666] nvgpu: 17000000.ga10b nvgpu_cic_mon_report_err_safety_services:55 [ERR] Error reporting is not supported in this platform [ 1627.763593] __ga10b__ NV_PGRAPH_PRI_GPC0_GPCCS_GPC_ACTIVITY1: 0x0 [ 1627.763595] __ga10b__ NV_PGRAPH_PRI_GPC0_GPCCS_GPC_ACTIVITY2: 0x0 [ 1627.763597] __ga10b__ NV_PGRAPH_PRI_GPC0_GPCCS_GPC_ACTIVITY3: 0x0 [ 1627.763600] __ga10b__ NV_PGRAPH_PRI_GPC0_GPCCS_GPC_ACTIVITY4: 0x0 [ 1627.771490] nvgpu: 17000000.ga10b nvgpu_set_err_notifier_locked:149 [ERR] error notifier set to 24 for ch 509 [ 1627.774021] __ga10b__ NV_PGRAPH_PRI_GPC0_TPC0_TPCCS_TPC_ACTIVITY0: 0x0 [ 1627.927218] __ga10b__ NV_PGRAPH_PRI_GPC0_TPC1_TPCCS_TPC_ACTIVITY0: 0x0 [ 1627.934047] __ga10b__ NV_PGRAPH_PRI_GPC0_TPC2_TPCCS_TPC_ACTIVITY0: 0x0 [ 1627.940993] __ga10b__ NV_PGRAPH_PRI_GPCS_GPCCS_GPC_ACTIVITY0: 0x1000000 [ 1627.947972] __ga10b__ NV_PGRAPH_PRI_GPCS_GPCCS_GPC_ACTIVITY1: 0x0 [ 1627.954438] __ga10b__ NV_PGRAPH_PRI_GPCS_GPCCS_GPC_ACTIVITY2: 0x0 [ 1627.961132] __ga10b__ NV_PGRAPH_PRI_GPCS_GPCCS_GPC_ACTIVITY3: 0x0 [ 1627.967471] __ga10b__ NV_PGRAPH_PRI_GPCS_GPCCS_GPC_ACTIVITY4: 0x9 [ 1627.973840] __ga10b__ NV_PGRAPH_PRI_GPCS_TPCS_TPCCS_TPC_ACTIVITY0: 0x0 [ 1627.980702] __ga10b__ NV_PGRAPH_PRI_DS_MPIPE_STATUS: 0x0 [ 1627.986337] __ga10b__ NV_PGRAPH_PRI_FE_GO_IDLE_TIMEOUT : 0x1800 [ 1627.992587] __ga10b__ NV_PGRAPH_PRI_FE_GO_IDLE_INFO : 0x1000700 [ 1627.998796] __ga10b__ NV_PGRAPH_PRI_GPC0_TPC0_TEX_M_TEX_SUBUNITS_STATUS: 0x0 [ 1628.006208] __ga10b__ NV_PGRAPH_PRI_CWD_FS: 0x702 [ 1628.011183] __ga10b__ NV_PGRAPH_PRI_FE_TPC_FS(0): 0xf7 [ 1628.016574] __ga10b__ NV_PGRAPH_PRI_CWD_GPC_TPC_ID: 0x1120213 [ 1628.022741] __ga10b__ NV_PGRAPH_PRI_CWD_SM_ID(0): 0x10306 [ 1628.028443] __ga10b__ NV_PGRAPH_PRI_FECS_CTXSW_STATUS_FE_0: 0x1 [ 1628.034641] __ga10b__ NV_PGRAPH_PRI_FECS_CTXSW_STATUS_1: 0x190 [ 1628.040789] __ga10b__ NV_PGRAPH_PRI_GPC0_GPCCS_CTXSW_STATUS_GPC_0: 0x0 [ 1628.047607] __ga10b__ NV_PGRAPH_PRI_GPC0_GPCCS_CTXSW_STATUS_1: 0x13a0 [ 1628.054534] __ga10b__ NV_PGRAPH_PRI_FECS_CTXSW_IDLESTATE : 0xf [ 1628.060636] __ga10b__ NV_PGRAPH_PRI_GPC0_GPCCS_CTXSW_IDLESTATE : 0xf [ 1628.067383] __ga10b__ NV_PGRAPH_PRI_FECS_CURRENT_CTX : 0xb013cdb1 [ 1628.073897] __ga10b__ NV_PGRAPH_PRI_FECS_NEW_CTX : 0xb01b4fc1 [ 1628.080068] __ga10b__ NV_PGRAPH_PRI_FECS_HOST_INT_ENABLE : 0x7f0003 [ 1628.086633] __ga10b__ NV_PGRAPH_PRI_FECS_HOST_INT_STATUS : 0x0 [ 1628.092783] __ga10b__ NV_PGRAPH_PRI_GPCS_ROP0_CROP_STATUS1 : 0x5f00000 [ 1628.099597] __ga10b__ NV_PGRAPH_PRI_GPCS_ROPS_CROP_STATUS1 : 0x700000 [ 1628.106363] __ga10b__ NV_PGRAPH_PRI_GPCS_ROP0_ZROP_STATUS : 0x0 [ 1628.112700] __ga10b__ NV_PGRAPH_PRI_GPCS_ROP0_ZROP_STATUS2 : 0x0 [ 1628.118983] __ga10b__ NV_PGRAPH_PRI_GPCS_ROP1_ZROP_STATUS: 0x0 [ 1628.125151] __ga10b__ NV_PGRAPH_PRI_GPCS_ROP1_ZROP_STATUS2: 0x0 [ 1628.131407] __ga10b__ NV_PGRAPH_PRI_GPCS_ROPS_ZROP_STATUS : 0x0 [ 1628.137748] __ga10b__ NV_PGRAPH_PRI_GPCS_ROPS_ZROP_STATUS2 : 0x0 [ 1628.144100] __ga10b__ NV_PGRAPH_PRI_GPC0_GPCCS_GPC_EXCEPTION: 0x0 [ 1628.150584] __ga10b__ NV_PGRAPH_PRI_GPC0_GPCCS_GPC_EXCEPTION_EN: 0xfc0f6004 [ 1628.157947] __ga10b__ NV_PGRAPH_PRI_GPC0_TPC0_TPCCS_TPC_EXCEPTION: 0x0 [ 1628.164829] __ga10b__ NV_PGRAPH_PRI_GPC0_TPC0_TPCCS_TPC_EXCEPTION_EN: 0x16 [ 1628.172042] __ga10b__ NV_PGRAPH_PRI_GPCS_TPCS_SMS_HWW_GLOBAL_ESR_REPORT_MASK: 0x1174 [ 1628.180143] __ga10b__ NV_PGRAPH_PRI_GPCS_TPCS_SMS_HWW_WARP_ESR_REPORT_MASK: 0x0 [ 1628.187830] __ga10b__ NV_PGRAPH_PRI_GPCS_TPCS_SMS_HWW_GLOBAL_ESR: 0x0 [ 1628.194680] __ga10b__ NV_PGRAPH_PRI_GPCS_TPCS_SMS_DBGR_CONTROL0: 0x1c00 [ 1628.201622] __ga10b__ NV_PGRAPH_PRI_GPCS_TPCS_SMS_DBGR_STATUS0: 0x0 [ 1628.208191] __ga10b__ NV_PGRAPH_PRI_GPCS_TPCS_SMS_DBGR_BPT_PAUSE_MASK_0: 0x0 [ 1628.215541] __ga10b__ NV_PGRAPH_PRI_GPCS_TPCS_SMS_DBGR_BPT_PAUSE_MASK_1: 0x0 [ 1628.222931] __ga10b__ NV_PGRAPH_PRI_GPC0_TPC0_SM0_HWW_WARP_ESR: 0x0 [ 1628.229448] __ga10b__ NV_PGRAPH_PRI_GPC0_TPC0_SM0_HWW_WARP_ESR_REPORT_MASK: 0xe812b60 [ 1628.237677] __ga10b__ NV_PGRAPH_PRI_GPC0_TPC0_SM0_HWW_GLOBAL_ESR: 0x0 [ 1628.244414] __ga10b__ NV_PGRAPH_PRI_GPC0_TPC0_SM0_HWW_GLOBAL_ESR_REPORT_MASK: 0x1174 [ 1628.252513] __ga10b__ NV_PGRAPH_PRI_GPC0_TPC0_SM0_DBGR_CONTROL0: 0x1c00 [ 1628.259460] __ga10b__ NV_PGRAPH_PRI_GPC0_TPC0_SM0_DBGR_STATUS0: 0x0 [ 1628.266027] __ga10b__ NV_PGRAPH_PRI_GPC0_TPC0_SM1_HWW_WARP_ESR: 0x0 [ 1628.272611] __ga10b__ NV_PGRAPH_PRI_GPC0_TPC0_SM1_HWW_WARP_ESR_REPORT_MASK: 0xe812b60 [ 1628.280722] __ga10b__ NV_PGRAPH_PRI_GPC0_TPC0_SM1_HWW_GLOBAL_ESR: 0x0 [ 1628.287387] __ga10b__ NV_PGRAPH_PRI_GPC0_TPC0_SM1_HWW_GLOBAL_ESR_REPORT_MASK: 0x1174 [ 1628.295374] __ga10b__ NV_PGRAPH_PRI_GPC0_TPC0_SM1_DBGR_CONTROL0: 0x1c00 [ 1628.302202] __ga10b__ NV_PGRAPH_PRI_GPC0_TPC0_SM1_DBGR_STATUS0: 0x0 [ 1628.308674] __ga10b__ NV_PGRAPH_PRI_GPC0_TPC1_SM0_HWW_WARP_ESR: 0x0 [ 1628.315126] __ga10b__ NV_PGRAPH_PRI_GPC0_TPC1_SM0_HWW_WARP_ESR_REPORT_MASK: 0xe812b60 [ 1628.323193] __ga10b__ NV_PGRAPH_PRI_GPC0_TPC1_SM0_HWW_GLOBAL_ESR: 0x0 [ 1628.329839] __ga10b__ NV_PGRAPH_PRI_GPC0_TPC1_SM0_HWW_GLOBAL_ESR_REPORT_MASK: 0x1174 [ 1628.337836] __ga10b__ NV_PGRAPH_PRI_GPC0_TPC1_SM0_DBGR_CONTROL0: 0x1c00 [ 1628.344663] __ga10b__ NV_PGRAPH_PRI_GPC0_TPC1_SM0_DBGR_STATUS0: 0x0 [ 1628.351130] __ga10b__ NV_PGRAPH_PRI_GPC0_TPC1_SM1_HWW_WARP_ESR: 0x0 [ 1628.357597] __ga10b__ NV_PGRAPH_PRI_GPC0_TPC1_SM1_HWW_WARP_ESR_REPORT_MASK: 0xe812b60 [ 1628.365675] __ga10b__ NV_PGRAPH_PRI_GPC0_TPC1_SM1_HWW_GLOBAL_ESR: 0x0 [ 1628.372331] __ga10b__ NV_PGRAPH_PRI_GPC0_TPC1_SM1_HWW_GLOBAL_ESR_REPORT_MASK: 0x1174 [ 1628.380320] __ga10b__ NV_PGRAPH_PRI_GPC0_TPC1_SM1_DBGR_CONTROL0: 0x1c00 [ 1628.387147] __ga10b__ NV_PGRAPH_PRI_GPC0_TPC1_SM1_DBGR_STATUS0: 0x0 [ 1628.393607] __ga10b__ NV_PGRAPH_PRI_GPC0_TPC2_SM0_HWW_WARP_ESR: 0x0 [ 1628.400074] __ga10b__ NV_PGRAPH_PRI_GPC0_TPC2_SM0_HWW_WARP_ESR_REPORT_MASK: 0xe812b60 [ 1628.408156] __ga10b__ NV_PGRAPH_PRI_GPC0_TPC2_SM0_HWW_GLOBAL_ESR: 0x0 [ 1628.414818] __ga10b__ NV_PGRAPH_PRI_GPC0_TPC2_SM0_HWW_GLOBAL_ESR_REPORT_MASK: 0x1174 [ 1628.422803] __ga10b__ NV_PGRAPH_PRI_GPC0_TPC2_SM0_DBGR_CONTROL0: 0x1c00 [ 1628.429631] __ga10b__ NV_PGRAPH_PRI_GPC0_TPC2_SM0_DBGR_STATUS0: 0x0 [ 1628.436103] __ga10b__ NV_PGRAPH_PRI_GPC0_TPC2_SM1_HWW_WARP_ESR: 0x0 [ 1628.442570] __ga10b__ NV_PGRAPH_PRI_GPC0_TPC2_SM1_HWW_WARP_ESR_REPORT_MASK: 0x0 [ 1628.450108] __ga10b__ NV_PGRAPH_PRI_GPC0_TPC2_SM1_HWW_GLOBAL_ESR: 0x0 [ 1628.456758] __ga10b__ NV_PGRAPH_PRI_GPC0_TPC2_SM1_HWW_GLOBAL_ESR_REPORT_MASK: 0x1174 [ 1628.464733] __ga10b__ NV_PGRAPH_PRI_GPC0_TPC2_SM1_DBGR_CONTROL0: 0x1c00 [ 1628.471560] __ga10b__ NV_PGRAPH_PRI_GPC0_TPC2_SM1_DBGR_STATUS0: 0x0 [ 1628.478018] __ga10b__ NV_PGRAPH_PRI_GPC1_TPC0_SM0_HWW_WARP_ESR: 0x0 [ 1628.484472] __ga10b__ NV_PGRAPH_PRI_GPC1_TPC0_SM0_HWW_WARP_ESR_REPORT_MASK: 0xe812b60 [ 1628.492545] __ga10b__ NV_PGRAPH_PRI_GPC1_TPC0_SM0_HWW_GLOBAL_ESR: 0x0 [ 1628.499194] __ga10b__ NV_PGRAPH_PRI_GPC1_TPC0_SM0_HWW_GLOBAL_ESR_REPORT_MASK: 0x0 [ 1628.506916] __ga10b__ NV_PGRAPH_PRI_GPC1_TPC0_SM0_DBGR_CONTROL0: 0x1c00 [ 1628.513743] __ga10b__ NV_PGRAPH_PRI_GPC1_TPC0_SM0_DBGR_STATUS0: 0x0 [ 1628.520208] __ga10b__ NV_PGRAPH_PRI_GPC1_TPC0_SM1_HWW_WARP_ESR: 0x0 [ 1628.526681] __ga10b__ NV_PGRAPH_PRI_GPC1_TPC0_SM1_HWW_WARP_ESR_REPORT_MASK: 0x0 [ 1628.534230] __ga10b__ NV_PGRAPH_PRI_GPC1_TPC0_SM1_HWW_GLOBAL_ESR: 0x0 [ 1628.540882] __ga10b__ NV_PGRAPH_PRI_GPC1_TPC0_SM1_HWW_GLOBAL_ESR_REPORT_MASK: 0x0 [ 1628.548601] __ga10b__ NV_PGRAPH_PRI_GPC1_TPC0_SM1_DBGR_CONTROL0: 0x1c00 [ 1628.555427] __ga10b__ NV_PGRAPH_PRI_GPC1_TPC0_SM1_DBGR_STATUS0: 0x0 [ 1628.561879] __ga10b__ NV_PGRAPH_PRI_GPC1_TPC1_SM0_HWW_WARP_ESR: 0x0 [ 1628.568327] __ga10b__ NV_PGRAPH_PRI_GPC1_TPC1_SM0_HWW_WARP_ESR_REPORT_MASK: 0xe812b60 [ 1628.576404] __ga10b__ NV_PGRAPH_PRI_GPC1_TPC1_SM0_HWW_GLOBAL_ESR: 0x0 [ 1628.583052] __ga10b__ NV_PGRAPH_PRI_GPC1_TPC1_SM0_HWW_GLOBAL_ESR_REPORT_MASK: 0x0 [ 1628.590766] __ga10b__ NV_PGRAPH_PRI_GPC1_TPC1_SM0_DBGR_CONTROL0: 0x1c00 [ 1628.597587] __ga10b__ NV_PGRAPH_PRI_GPC1_TPC1_SM0_DBGR_STATUS0: 0x0 [ 1628.604057] __ga10b__ NV_PGRAPH_PRI_GPC1_TPC1_SM1_HWW_WARP_ESR: 0x0 [ 1628.610515] __ga10b__ NV_PGRAPH_PRI_GPC1_TPC1_SM1_HWW_WARP_ESR_REPORT_MASK: 0xe812b60 [ 1628.618595] __ga10b__ NV_PGRAPH_PRI_GPC1_TPC1_SM1_HWW_GLOBAL_ESR: 0x0 [ 1628.625240] __ga10b__ NV_PGRAPH_PRI_GPC1_TPC1_SM1_HWW_GLOBAL_ESR_REPORT_MASK: 0x0 [ 1628.632962] __ga10b__ NV_PGRAPH_PRI_GPC1_TPC1_SM1_DBGR_CONTROL0: 0x1c00 [ 1628.639802] __ga10b__ NV_PGRAPH_PRI_GPC1_TPC1_SM1_DBGR_STATUS0: 0x0 [ 1628.646274] __ga10b__ NV_PGRAPH_PRI_GPC1_TPC2_SM0_HWW_WARP_ESR: 0x0 [ 1628.652739] __ga10b__ NV_PGRAPH_PRI_GPC1_TPC2_SM0_HWW_WARP_ESR_REPORT_MASK: 0xe812b60 [ 1628.660817] __ga10b__ NV_PGRAPH_PRI_GPC1_TPC2_SM0_HWW_GLOBAL_ESR: 0x0 [ 1628.667451] __ga10b__ NV_PGRAPH_PRI_GPC1_TPC2_SM0_HWW_GLOBAL_ESR_REPORT_MASK: 0x0 [ 1628.675171] __ga10b__ NV_PGRAPH_PRI_GPC1_TPC2_SM0_DBGR_CONTROL0: 0x1c00 [ 1628.681996] __ga10b__ NV_PGRAPH_PRI_GPC1_TPC2_SM0_DBGR_STATUS0: 0x0 [ 1628.688450] __ga10b__ NV_PGRAPH_PRI_GPC1_TPC2_SM1_HWW_WARP_ESR: 0x0 [ 1628.694929] __ga10b__ NV_PGRAPH_PRI_GPC1_TPC2_SM1_HWW_WARP_ESR_REPORT_MASK: 0xe812b60 [ 1628.702998] __ga10b__ NV_PGRAPH_PRI_GPC1_TPC2_SM1_HWW_GLOBAL_ESR: 0x0 [ 1628.709658] __ga10b__ NV_PGRAPH_PRI_GPC1_TPC2_SM1_HWW_GLOBAL_ESR_REPORT_MASK: 0x1174 [ 1628.717662] __ga10b__ NV_PGRAPH_PRI_GPC1_TPC2_SM1_DBGR_CONTROL0: 0x1c00 [ 1628.724495] __ga10b__ NV_PGRAPH_PRI_GPC1_TPC2_SM1_DBGR_STATUS0: 0x0 [ 1628.730963] __ga10b__ NV_PGRAPH_PRI_GPC1_TPC3_SM0_HWW_WARP_ESR: 0x0 [ 1628.737432] __ga10b__ NV_PGRAPH_PRI_GPC1_TPC3_SM0_HWW_WARP_ESR_REPORT_MASK: 0xe812b60 [ 1628.745508] __ga10b__ NV_PGRAPH_PRI_GPC1_TPC3_SM0_HWW_GLOBAL_ESR: 0x0 [ 1628.752154] __ga10b__ NV_PGRAPH_PRI_GPC1_TPC3_SM0_HWW_GLOBAL_ESR_REPORT_MASK: 0x0 [ 1628.759870] __ga10b__ NV_PGRAPH_PRI_GPC1_TPC3_SM0_DBGR_CONTROL0: 0x1c00 [ 1628.766705] __ga10b__ NV_PGRAPH_PRI_GPC1_TPC3_SM0_DBGR_STATUS0: 0x0 [ 1628.773176] __ga10b__ NV_PGRAPH_PRI_GPC1_TPC3_SM1_HWW_WARP_ESR: 0x0 [ 1628.779650] __ga10b__ NV_PGRAPH_PRI_GPC1_TPC3_SM1_HWW_WARP_ESR_REPORT_MASK: 0x0 [ 1628.787202] __ga10b__ NV_PGRAPH_PRI_GPC1_TPC3_SM1_HWW_GLOBAL_ESR: 0x0 [ 1628.793848] __ga10b__ NV_PGRAPH_PRI_GPC1_TPC3_SM1_HWW_GLOBAL_ESR_REPORT_MASK: 0x0 [ 1628.801573] __ga10b__ NV_PGRAPH_PRI_GPC1_TPC3_SM1_DBGR_CONTROL0: 0x1c00 [ 1628.808411] __ga10b__ NV_PGRAPH_PRI_GPC1_TPC3_SM1_DBGR_STATUS0: 0x0 [ 1628.815236] nvgpu: 17000000.ga10b nvgpu_tsg_force_reset_ch:469 [ERR] chid: 509 is not bound to tsg [ 1636.566810] nvgpu: 17000000.ga10b ga10b_pbdma_handle_intr_0_legacy:437 [ERR] semaphore acquire timeout! [ 1636.576954] __ga10b__ Channel Status - chip ga10b [ 1636.576955] __ga10b__ --------------------------- [ 1636.581843] __ga10b__ 492-ga10b, TSG: 19, pid 3008, refs: 2, deterministic: no, domain name: (default) [ 1636.586706] __ga10b__ channel status: in use on_eng not busy [ 1636.596309] __ga10b__ RAMFC: TOP: 80000020068b0b18 PUT: 0020068b0b18 GET: 0020068b0b18 FETCH: 000000000000 HEADER: 2140006c COUNT: 00000000 SEMAPHORE: addr 0020068a0000 payload 0000000000000000 execute 00100001 [ 1636.602221] __ga10b__ [ 1636.621515] __ga10b__ 493-ga10b, TSG: 18, pid 3008, refs: 2, deterministic: no, domain name: (default) [ 1636.624056] __ga10b__ channel status: in use idle not busy [ 1636.633671] __ga10b__ RAMFC: TOP: 800000200607c308 PUT: 00200607c308 GET: 00200607c308 FETCH: 000000000000 HEADER: 2140006c COUNT: 00000000 SEMAPHORE: addr 0020063c0000 payload 0000000000000000 execute 00100001 [ 1636.639459] __ga10b__ [ 1636.658751] __ga10b__ 494-ga10b, TSG: 17, pid 3008, refs: 2, deterministic: no, domain name: (default) [ 1636.661288] __ga10b__ channel status: in use idle not busy [ 1636.670893] __ga10b__ RAMFC: TOP: 8000002006066a40 PUT: 002006066a40 GET: 002006066a40 FETCH: 000000000000 HEADER: 2140006c COUNT: 00000000 SEMAPHORE: addr 002006010000 payload 0000000000000000 execute 00100001 [ 1636.676652] __ga10b__ [ 1636.695911] __ga10b__ 495-ga10b, TSG: 16, pid 3008, refs: 2, deterministic: no, domain name: (default) [ 1636.698435] __ga10b__ channel status: in use idle not busy [ 1636.708028] __ga10b__ RAMFC: TOP: 80000020060019a0 PUT: 0020060019a0 GET: 0020060019a0 FETCH: 000000000000 HEADER: 2140006c COUNT: 00000000 SEMAPHORE: addr 002005d30000 payload 0000000000000000 execute 00100001 [ 1636.713765] __ga10b__ [ 1636.733000] __ga10b__ 496-ga10b, TSG: 15, pid 3090, refs: 2, deterministic: no, domain name: (default) [ 1636.735540] __ga10b__ channel status: in use idle not busy [ 1636.745124] __ga10b__ RAMFC: TOP: 80000020040080a0 PUT: 0020040080a0 GET: 0020040080a0 FETCH: 000000000000 HEADER: 2140006c COUNT: 00000000 SEMAPHORE: addr 002004010000 payload 0000000000000000 execute 00100001 [ 1636.750866] __ga10b__ [ 1636.770110] __ga10b__ 497-ga10b, TSG: 14, pid 3008, refs: 5, deterministic: no, domain name: (default) [ 1636.772631] __ga10b__ channel status: in use pending, acquire_fail busy [ 1636.782214] __ga10b__ RAMFC: TOP: 8000001ffad0e360 PUT: 001ffad0e374 GET: 001ffad0e360 FETCH: 000000000000 HEADER: 2140006c COUNT: 11110000 SEMAPHORE: addr 002000040000 payload 0000000000019a8a execute 00081003 [ 1636.789115] __ga10b__ [ 1636.808344] __ga10b__ 498-ga10b, TSG: 13, pid 3008, refs: 2, deterministic: no, domain name: (default) [ 1636.810870] __ga10b__ channel status: in use idle not busy [ 1636.820454] __ga10b__ RAMFC: TOP: 8000002005a30168 PUT: 002005a30168 GET: 002005a30168 FETCH: 000000000000 HEADER: 2140006c COUNT: 00000000 SEMAPHORE: addr 002005a20000 payload 0000000000000000 execute 00100001 [ 1636.826198] __ga10b__ [ 1636.845444] __ga10b__ 499-ga10b, TSG: 12, pid 3008, refs: 2, deterministic: no, domain name: (default) [ 1636.847980] __ga10b__ channel status: in use idle not busy [ 1636.857568] __ga10b__ RAMFC: TOP: 80000020051f8208 PUT: 0020051f8208 GET: 0020051f8208 FETCH: 000000000000 HEADER: 2140006c COUNT: 00000000 SEMAPHORE: addr 002005640000 payload 0000000000000000 execute 00100001 [ 1636.863310] __ga10b__ [ 1636.882555] __ga10b__ 500-ga10b, TSG: 11, pid 3008, refs: 2, deterministic: no, domain name: (default) [ 1636.885078] __ga10b__ channel status: in use idle not busy [ 1636.894663] __ga10b__ RAMFC: TOP: 8000002005198230 PUT: 002005198230 GET: 002005198230 FETCH: 000000000000 HEADER: 2140006c COUNT: 00000000 SEMAPHORE: addr 0020051e0000 payload 0000000000000000 execute 00100001 [ 1636.900404] __ga10b__ [ 1636.919647] __ga10b__ 501-ga10b, TSG: 10, pid 3008, refs: 2, deterministic: no, domain name: (default) [ 1636.922172] __ga10b__ channel status: in use idle not busy [ 1636.931762] __ga10b__ RAMFC: TOP: 8000002005180208 PUT: 002005180208 GET: 002005180208 FETCH: 000000000000 HEADER: 2140006c COUNT: 00000000 SEMAPHORE: addr 002004eb0000 payload 0000000000000000 execute 00100001 [ 1636.937502] __ga10b__ [ 1636.956740] __ga10b__ 502-ga10b, TSG: 9, pid 3008, refs: 2, deterministic: no, domain name: (default) [ 1636.959282] __ga10b__ channel status: in use idle not busy [ 1636.968783] __ga10b__ RAMFC: TOP: 8000002004e88230 PUT: 002004e88230 GET: 002004e88230 FETCH: 000000000000 HEADER: 2140006c COUNT: 00000000 SEMAPHORE: addr 002004e90000 payload 0000000000000000 execute 00100001 [ 1636.974535] __ga10b__ [ 1636.993785] __ga10b__ 503-ga10b, TSG: 8, pid 3008, refs: 2, deterministic: no, domain name: (default) [ 1636.996305] __ga10b__ channel status: in use idle not busy [ 1637.005810] __ga10b__ RAMFC: TOP: 8000002004bb0208 PUT: 002004bb0208 GET: 002004bb0208 FETCH: 000000000000 HEADER: 2140006c COUNT: 00000000 SEMAPHORE: addr 002004ba0000 payload 0000000000000000 execute 00100001 [ 1637.011551] __ga10b__ [ 1637.030793] __ga10b__ 504-ga10b, TSG: 7, pid 3008, refs: 2, deterministic: no, domain name: (default) [ 1637.033331] __ga10b__ channel status: in use idle not busy [ 1637.042828] __ga10b__ RAMFC: TOP: 80000020046d0230 PUT: 0020046d0230 GET: 0020046d0230 FETCH: 000000000000 HEADER: 2140006c COUNT: 00000000 SEMAPHORE: addr 0020046c0000 payload 0000000000000000 execute 00100001 [ 1637.048576] __ga10b__ [ 1637.067820] __ga10b__ 505-ga10b, TSG: 6, pid 3008, refs: 2, deterministic: no, domain name: (default) [ 1637.070352] __ga10b__ channel status: in use idle not busy [ 1637.079852] __ga10b__ RAMFC: TOP: 8000002004318050 PUT: 002004318050 GET: 002004318050 FETCH: 000000000000 HEADER: 2140006c COUNT: 00000000 SEMAPHORE: addr 002004360000 payload 0000000000000000 execute 00100001 [ 1637.085589] __ga10b__ [ 1637.104834] __ga10b__ 506-ga10b, TSG: 5, pid 3008, refs: 2, deterministic: no, domain name: (default) [ 1637.107358] __ga10b__ channel status: in use idle not busy [ 1637.116853] __ga10b__ RAMFC: TOP: 8000002004305230 PUT: 002004305230 GET: 002004305230 FETCH: 000000000000 HEADER: 2140006c COUNT: 00000000 SEMAPHORE: addr 002004030000 payload 0000000000000000 execute 00100001 [ 1637.122589] __ga10b__ [ 1637.141837] __ga10b__ 507-ga10b, TSG: 4, pid 3008, refs: 2, deterministic: no, domain name: (default) [ 1637.144364] __ga10b__ channel status: in use idle not busy [ 1637.153859] __ga10b__ RAMFC: TOP: 80000020040080a0 PUT: 0020040080a0 GET: 0020040080a0 FETCH: 000000000000 HEADER: 2140006c COUNT: 00000000 SEMAPHORE: addr 002004010000 payload 0000000000000000 execute 00100001 [ 1637.159596] __ga10b__ [ 1637.178850] __ga10b__ 508-ga10b, TSG: 3, pid 2755, refs: 2, deterministic: no, domain name: (default) [ 1637.181370] __ga10b__ channel status: in use idle not busy [ 1637.190861] __ga10b__ RAMFC: TOP: 80000020040080a0 PUT: 0020040080a0 GET: 0020040080a0 FETCH: 000000000000 HEADER: 2140006c COUNT: 00000000 SEMAPHORE: addr 002004010000 payload 0000000000000000 execute 00100001 [ 1637.196599] __ga10b__ [ 1637.215841] __ga10b__ 509-ga10b, TSG: 2, pid 2334, refs: 6, deterministic: no, domain name: (default) [ 1637.218369] __ga10b__ channel status: in use on_pbdma, pbdma_busy busy [ 1637.227868] __ga10b__ RAMFC: TOP: 8000001fff3a065c PUT: 001fff3a0670 GET: 001fff3a065c FETCH: 000000000000 HEADER: 2140006c COUNT: 11110000 SEMAPHORE: addr 002000110000 payload 0000000000013172 execute 00081003 [ 1637.234695] __ga10b__ [ 1637.253930] __ga10b__ 510-ga10b, TSG: 1, pid 2080, refs: 2, deterministic: no, domain name: (default) [ 1637.256446] __ga10b__ channel status: in use idle not busy [ 1637.265944] __ga10b__ RAMFC: TOP: 8000002004033370 PUT: 002004033370 GET: 002004033370 FETCH: 000000000000 HEADER: 2140006c COUNT: 00000000 SEMAPHORE: addr 002004320000 payload 0000000000000000 execute 00000001 [ 1637.271681] __ga10b__ [ 1637.290913] __ga10b__ 511-ga10b, TSG: 0, pid 2080, refs: 2, deterministic: no, domain name: (default) [ 1637.293454] __ga10b__ channel status: in use on_eng not busy [ 1637.302958] __ga10b__ RAMFC: TOP: 8000002004052480 PUT: 002004052480 GET: 002004052480 FETCH: 000000000000 HEADER: 2140006c COUNT: 00000000 SEMAPHORE: addr 002004020000 payload 0000000000000000 execute 00100001 [ 1637.308868] __ga10b__ [ 1637.328116] __ga10b__ PBDMA Status - chip ga10b [ 1637.330637] __ga10b__ ------------------------- [ 1637.335316] __ga10b__ pbdma 0: [ 1637.339991] __ga10b__ id: 2 - [tsg] next_id: - -1 [channel] | status: valid [ 1637.343150] __ga10b__ PBDMA_PUT 0000001fff3a0670 PBDMA_GET 0000001fff3a065c [ 1637.350861] __ga10b__ GP_PUT 000006f3 GP_GET 000006e7 FETCH 000006e7 HEADER 2140006c [ 1637.358223] __ga10b__ HDR 2001001b SHADOW0 068b0af0 SHADOW1 00002820 [ 1637.367090] __ga10b__ pbdma 1: [ 1637.374532] __ga10b__ id: -1 - [channel] next_id: - -1 [channel] | status: invalid [ 1637.377691] __ga10b__ PBDMA_PUT 00000020046d00a0 PBDMA_GET 00000020046d00a0 [ 1637.385677] __ga10b__ GP_PUT 0000000c GP_GET 0000000c FETCH 0000000c HEADER 2140006c [ 1637.393034] __ga10b__ HDR 2001001b SHADOW0 046d0078 SHADOW1 00002820 [ 1637.401900] __ga10b__ pbdma 2: [ 1637.409346] __ga10b__ id: -1 - [channel] next_id: - -1 [channel] | status: invalid [ 1637.412504] __ga10b__ PBDMA_PUT 0000002004318118 PBDMA_GET 0000002004318118 [ 1637.420495] __ga10b__ GP_PUT 0000001c GP_GET 0000001c FETCH 0000001c HEADER 2140006c [ 1637.427856] __ga10b__ HDR 2001001b SHADOW0 043180f0 SHADOW1 00002820 [ 1637.436718] __ga10b__ pbdma 3: [ 1637.444163] __ga10b__ id: -1 - [channel] next_id: - -1 [channel] | status: invalid [ 1637.447329] __ga10b__ PBDMA_PUT 00000067ef7f6fec PBDMA_GET 000000cd20b86794 [ 1637.455316] __ga10b__ GP_PUT 00000000 GP_GET 7e1826eb FETCH 00000000 HEADER e1955910 [ 1637.462680] __ga10b__ HDR 5f129d84 SHADOW0 61804464 SHADOW1 be122349 [ 1637.471557] __ga10b__ pbdma 4: [ 1637.479017] __ga10b__ id: -1 - [channel] next_id: - -1 [channel] | status: invalid [ 1637.482181] __ga10b__ PBDMA_PUT 0000000db7213608 PBDMA_GET 0000000db7213608 [ 1637.490169] __ga10b__ GP_PUT 00000000 GP_GET 6b8eb69b FETCH 00000000 HEADER 8086a79c [ 1637.497527] __ga10b__ HDR 145ec4b0 SHADOW0 bdc529ee SHADOW1 f4b979a2 [ 1637.506407] __ga10b__ pbdma 5: [ 1637.513851] __ga10b__ id: -1 - [channel] next_id: - -1 [channel] | status: invalid [ 1637.517014] __ga10b__ PBDMA_PUT 0000002004ea8118 PBDMA_GET 0000002004ea8118 [ 1637.524992] __ga10b__ GP_PUT 0000001c GP_GET 0000001c FETCH 0000001c HEADER 2140006c [ 1637.532363] __ga10b__ HDR 2001001b SHADOW0 04ea80f0 SHADOW1 00002820 [ 1637.541233] __ga10b__ [ 1637.548682] __ga10b__ ga10b eng 0: [ 1637.551207] __ga10b__ id: 19 (tsg), next_id: -1 (channel), ctx status: valid [ 1637.554806] __ga10b__ [ 1637.562165] __ga10b__ ga10b eng 1: [ 1637.564689] __ga10b__ id: 0 (tsg), next_id: -1 (channel), ctx status: valid [ 1637.568289] __ga10b__ [ 1637.575545] __ga10b__ ga10b eng 2: [ 1637.578069] __ga10b__ id: -1 (channel), next_id: -1 (channel), ctx status: invalid [ 1637.581671] __ga10b__ [ 1637.589561] __ga10b__ ga10b eng 3: [ 1637.592095] __ga10b__ id: -1 (channel), next_id: -1 (channel), ctx status: invalid [ 1637.595698] __ga10b__ [ 1637.603589] __ga10b__ ga10b eng 4: [ 1637.606113] __ga10b__ id: -1 (channel), next_id: -1 (channel), ctx status: invalid [ 1637.609718] __ga10b__ [ 1637.617618] __ga10b__ ga10b eng 5: [ 1637.620159] __ga10b__ id: -1 (channel), next_id: -1 (channel), ctx status: invalid [ 1637.623761] __ga10b__ [ 1637.631653] __ga10b__ [ 1637.632227] nvgpu: 17000000.ga10b nvgpu_channel_recover_from_wdt:107 [ERR] Job on channel 509 timed out [ 1637.634183] nvgpu: 17000000.ga10b report_pbdma_error:330 [ERR] pbdma_intr_0(0)= 0x04000000 [ 1637.637476] __ga10b__ NV_PGRAPH_STATUS: 0x0 [ 1637.646823] nvgpu: 17000000.ga10b nvgpu_cic_mon_report_err_safety_services:55 [ERR] Error reporting is not supported in this platform [ 1637.646830] nvgpu: 17000000.ga10b nvgpu_set_err_notifier_locked:149 [ERR] error notifier set to 24 for ch 509 [ 1637.656953] __ga10b__ NV_PGRAPH_STATUS1: 0x0 [ 1637.656955] __ga10b__ NV_PGRAPH_ENGINE_STATUS: 0x0 [ 1637.656960] __ga10b__ NV_PGRAPH_GRFIFO_STATUS : 0x10100001 [ 1637.656965] __ga10b__ NV_PGRAPH_GRFIFO_CONTROL : 0x10001 [ 1637.705168] __ga10b__ NV_PGRAPH_PRI_FECS_HOST_INT_STATUS : 0x0 [ 1637.711222] __ga10b__ NV_PGRAPH_EXCEPTION : 0x0 [ 1637.715990] __ga10b__ NV_PGRAPH_FECS_INTR : 0x0 [ 1637.720756] __ga10b__ NV_PFIFO_ENGINE_STATUS(GR) : 0xe200e [ 1637.726427] __ga10b__ NV_PGRAPH_ACTIVITY0: 0x0 [ 1637.731034] __ga10b__ NV_PGRAPH_ACTIVITY1: 0x0 [ 1637.735628] __ga10b__ NV_PGRAPH_ACTIVITY4: 0x0 [ 1637.742417] __ga10b__ NV_PGRAPH_PRI_SKED_ACTIVITY: 0x0 [ 1637.747788] __ga10b__ NV_PGRAPH_PRI_GPC0_GPCCS_GPC_ACTIVITY0: 0x0 [ 1637.754096] __ga10b__ NV_PGRAPH_PRI_GPC0_GPCCS_GPC_ACTIVITY1: 0x0 [ 1637.760397] __ga10b__ NV_PGRAPH_PRI_GPC0_GPCCS_GPC_ACTIVITY2: 0x0 [ 1637.766695] __ga10b__ NV_PGRAPH_PRI_GPC0_GPCCS_GPC_ACTIVITY3: 0x0 [ 1637.772992] __ga10b__ NV_PGRAPH_PRI_GPC0_GPCCS_GPC_ACTIVITY4: 0x0 [ 1637.779280] __ga10b__ NV_PGRAPH_PRI_GPC0_TPC0_TPCCS_TPC_ACTIVITY0: 0x0 [ 1637.786020] __ga10b__ NV_PGRAPH_PRI_GPC0_TPC1_TPCCS_TPC_ACTIVITY0: 0x0 [ 1637.792755] __ga10b__ NV_PGRAPH_PRI_GPC0_TPC2_TPCCS_TPC_ACTIVITY0: 0x0 [ 1637.799496] __ga10b__ NV_PGRAPH_PRI_GPCS_GPCCS_GPC_ACTIVITY0: 0x0 [ 1637.805793] __ga10b__ NV_PGRAPH_PRI_GPCS_GPCCS_GPC_ACTIVITY1: 0x0 [ 1637.812095] __ga10b__ NV_PGRAPH_PRI_GPCS_GPCCS_GPC_ACTIVITY2: 0x0 [ 1637.818396] __ga10b__ NV_PGRAPH_PRI_GPCS_GPCCS_GPC_ACTIVITY3: 0x0 [ 1637.824699] __ga10b__ NV_PGRAPH_PRI_GPCS_GPCCS_GPC_ACTIVITY4: 0x0 [ 1637.831011] __ga10b__ NV_PGRAPH_PRI_GPCS_TPCS_TPCCS_TPC_ACTIVITY0: 0x0 [ 1637.837752] __ga10b__ NV_PGRAPH_PRI_DS_MPIPE_STATUS: 0x0 [ 1637.843238] __ga10b__ NV_PGRAPH_PRI_FE_GO_IDLE_TIMEOUT : 0x1800 [ 1637.849355] __ga10b__ NV_PGRAPH_PRI_FE_GO_IDLE_INFO : 0x1000700 [ 1637.855469] __ga10b__ NV_PGRAPH_PRI_GPC0_TPC0_TEX_M_TEX_SUBUNITS_STATUS: 0x0 [ 1637.862739] __ga10b__ NV_PGRAPH_PRI_CWD_FS: 0x702 [ 1637.867601] __ga10b__ NV_PGRAPH_PRI_FE_TPC_FS(0): 0xf7 [ 1637.872917] __ga10b__ NV_PGRAPH_PRI_CWD_GPC_TPC_ID: 0x1120213 [ 1637.878842] __ga10b__ NV_PGRAPH_PRI_CWD_SM_ID(0): 0x10306 [ 1637.884413] __ga10b__ NV_PGRAPH_PRI_FECS_CTXSW_STATUS_FE_0: 0x1 [ 1637.890528] __ga10b__ NV_PGRAPH_PRI_FECS_CTXSW_STATUS_1: 0x190 [ 1637.896574] __ga10b__ NV_PGRAPH_PRI_GPC0_GPCCS_CTXSW_STATUS_GPC_0: 0x0 [ 1637.903321] __ga10b__ NV_PGRAPH_PRI_GPC0_GPCCS_CTXSW_STATUS_1: 0x390 [ 1637.909866] __ga10b__ NV_PGRAPH_PRI_FECS_CTXSW_IDLESTATE : 0xf [ 1637.915895] __ga10b__ NV_PGRAPH_PRI_GPC0_GPCCS_CTXSW_IDLESTATE : 0xf [ 1637.922462] __ga10b__ NV_PGRAPH_PRI_FECS_CURRENT_CTX : 0xb01b354a [ 1637.928767] __ga10b__ NV_PGRAPH_PRI_FECS_NEW_CTX : 0xb01b4fc1 [ 1637.934695] __ga10b__ NV_PGRAPH_PRI_FECS_HOST_INT_ENABLE : 0x7f0003 [ 1637.941170] __ga10b__ NV_PGRAPH_PRI_FECS_HOST_INT_STATUS : 0x0 [ 1637.947206] __ga10b__ NV_PGRAPH_PRI_GPCS_ROP0_CROP_STATUS1 : 0x5f00000 [ 1637.953953] __ga10b__ NV_PGRAPH_PRI_GPCS_ROPS_CROP_STATUS1 : 0x5f00000 [ 1637.960700] __ga10b__ NV_PGRAPH_PRI_GPCS_ROP0_ZROP_STATUS : 0x0 [ 1637.966802] __ga10b__ NV_PGRAPH_PRI_GPCS_ROP0_ZROP_STATUS2 : 0x0 [ 1637.973018] __ga10b__ NV_PGRAPH_PRI_GPCS_ROP1_ZROP_STATUS: 0x0 [ 1637.979057] __ga10b__ NV_PGRAPH_PRI_GPCS_ROP1_ZROP_STATUS2: 0x0 [ 1637.985284] __ga10b__ NV_PGRAPH_PRI_GPCS_ROPS_ZROP_STATUS : 0x0 [ 1637.991398] __ga10b__ NV_PGRAPH_PRI_GPCS_ROPS_ZROP_STATUS2 : 0x0 [ 1637.997591] __ga10b__ NV_PGRAPH_PRI_GPC0_GPCCS_GPC_EXCEPTION: 0x0 [ 1638.003895] __ga10b__ NV_PGRAPH_PRI_GPC0_GPCCS_GPC_EXCEPTION_EN: 0xfc0f6004 [ 1638.011086] __ga10b__ NV_PGRAPH_PRI_GPC0_TPC0_TPCCS_TPC_EXCEPTION: 0x0 [ 1638.017823] __ga10b__ NV_PGRAPH_PRI_GPC0_TPC0_TPCCS_TPC_EXCEPTION_EN: 0x16 [ 1638.024931] __ga10b__ NV_PGRAPH_PRI_GPCS_TPCS_SMS_HWW_GLOBAL_ESR_REPORT_MASK: 0x1174 [ 1638.032918] __ga10b__ NV_PGRAPH_PRI_GPCS_TPCS_SMS_HWW_WARP_ESR_REPORT_MASK: 0xe812b60 [ 1638.041004] __ga10b__ NV_PGRAPH_PRI_GPCS_TPCS_SMS_HWW_GLOBAL_ESR: 0x0 [ 1638.047653] __ga10b__ NV_PGRAPH_PRI_GPCS_TPCS_SMS_DBGR_CONTROL0: 0x1c00 [ 1638.054480] __ga10b__ NV_PGRAPH_PRI_GPCS_TPCS_SMS_DBGR_STATUS0: 0x0 [ 1638.060950] __ga10b__ NV_PGRAPH_PRI_GPCS_TPCS_SMS_DBGR_BPT_PAUSE_MASK_0: 0x0 [ 1638.068236] __ga10b__ NV_PGRAPH_PRI_GPCS_TPCS_SMS_DBGR_BPT_PAUSE_MASK_1: 0x0 [ 1638.075518] __ga10b__ NV_PGRAPH_PRI_GPC0_TPC0_SM0_HWW_WARP_ESR: 0x0 [ 1638.081992] __ga10b__ NV_PGRAPH_PRI_GPC0_TPC0_SM0_HWW_WARP_ESR_REPORT_MASK: 0x0 [ 1638.089528] __ga10b__ NV_PGRAPH_PRI_GPC0_TPC0_SM0_HWW_GLOBAL_ESR: 0x0 [ 1638.096183] __ga10b__ NV_PGRAPH_PRI_GPC0_TPC0_SM0_HWW_GLOBAL_ESR_REPORT_MASK: 0x0 [ 1638.103907] __ga10b__ NV_PGRAPH_PRI_GPC0_TPC0_SM0_DBGR_CONTROL0: 0x1c00 [ 1638.110747] __ga10b__ NV_PGRAPH_PRI_GPC0_TPC0_SM0_DBGR_STATUS0: 0x0 [ 1638.117202] __ga10b__ NV_PGRAPH_PRI_GPC0_TPC0_SM1_HWW_WARP_ESR: 0x0 [ 1638.123661] __ga10b__ NV_PGRAPH_PRI_GPC0_TPC0_SM1_HWW_WARP_ESR_REPORT_MASK: 0x781eb60 [ 1638.131734] __ga10b__ NV_PGRAPH_PRI_GPC0_TPC0_SM1_HWW_GLOBAL_ESR: 0x0 [ 1638.138382] __ga10b__ NV_PGRAPH_PRI_GPC0_TPC0_SM1_HWW_GLOBAL_ESR_REPORT_MASK: 0x1174 [ 1638.146369] __ga10b__ NV_PGRAPH_PRI_GPC0_TPC0_SM1_DBGR_CONTROL0: 0x1c00 [ 1638.153197] __ga10b__ NV_PGRAPH_PRI_GPC0_TPC0_SM1_DBGR_STATUS0: 0x0 [ 1638.159656] __ga10b__ NV_PGRAPH_PRI_GPC0_TPC1_SM0_HWW_WARP_ESR: 0x0 [ 1638.166123] __ga10b__ NV_PGRAPH_PRI_GPC0_TPC1_SM0_HWW_WARP_ESR_REPORT_MASK: 0xe812b60 [ 1638.174196] __ga10b__ NV_PGRAPH_PRI_GPC0_TPC1_SM0_HWW_GLOBAL_ESR: 0x0 [ 1638.180860] __ga10b__ NV_PGRAPH_PRI_GPC0_TPC1_SM0_HWW_GLOBAL_ESR_REPORT_MASK: 0x0 [ 1638.188585] __ga10b__ NV_PGRAPH_PRI_GPC0_TPC1_SM0_DBGR_CONTROL0: 0x1c00 [ 1638.195413] __ga10b__ NV_PGRAPH_PRI_GPC0_TPC1_SM0_DBGR_STATUS0: 0x0 [ 1638.201869] __ga10b__ NV_PGRAPH_PRI_GPC0_TPC1_SM1_HWW_WARP_ESR: 0x0 [ 1638.208337] __ga10b__ NV_PGRAPH_PRI_GPC0_TPC1_SM1_HWW_WARP_ESR_REPORT_MASK: 0xe812b60 [ 1638.216417] __ga10b__ NV_PGRAPH_PRI_GPC0_TPC1_SM1_HWW_GLOBAL_ESR: 0x0 [ 1638.223071] __ga10b__ NV_PGRAPH_PRI_GPC0_TPC1_SM1_HWW_GLOBAL_ESR_REPORT_MASK: 0x1174 [ 1638.227715] cpufreq: cpu0,cur:2330000,set:2188800,set ndiv:171 [ 1638.231062] __ga10b__ NV_PGRAPH_PRI_GPC0_TPC1_SM1_DBGR_CONTROL0: 0x1c00 [ 1638.234532] cpufreq: cpu4,cur:2019000,set:2188800,set ndiv:171 [ 1638.237903] __ga10b__ NV_PGRAPH_PRI_GPC0_TPC1_SM1_DBGR_STATUS0: 0x0 [ 1638.237906] __ga10b__ NV_PGRAPH_PRI_GPC0_TPC2_SM0_HWW_WARP_ESR: 0x0 [ 1638.237908] __ga10b__ NV_PGRAPH_PRI_GPC0_TPC2_SM0_HWW_WARP_ESR_REPORT_MASK: 0xe812b60 [ 1638.237911] __ga10b__ NV_PGRAPH_PRI_GPC0_TPC2_SM0_HWW_GLOBAL_ESR: 0x0 [ 1638.237913] __ga10b__ NV_PGRAPH_PRI_GPC0_TPC2_SM0_HWW_GLOBAL_ESR_REPORT_MASK: 0x1174 [ 1638.237919] __ga10b__ NV_PGRAPH_PRI_GPC0_TPC2_SM0_DBGR_CONTROL0: 0x1c00 [ 1638.280358] __ga10b__ NV_PGRAPH_PRI_GPC0_TPC2_SM0_DBGR_STATUS0: 0x0 [ 1638.286819] __ga10b__ NV_PGRAPH_PRI_GPC0_TPC2_SM1_HWW_WARP_ESR: 0x0 [ 1638.293276] __ga10b__ NV_PGRAPH_PRI_GPC0_TPC2_SM1_HWW_WARP_ESR_REPORT_MASK: 0xe812b60 [ 1638.301353] __ga10b__ NV_PGRAPH_PRI_GPC0_TPC2_SM1_HWW_GLOBAL_ESR: 0x0 [ 1638.308000] __ga10b__ NV_PGRAPH_PRI_GPC0_TPC2_SM1_HWW_GLOBAL_ESR_REPORT_MASK: 0x0 [ 1638.315719] __ga10b__ NV_PGRAPH_PRI_GPC0_TPC2_SM1_DBGR_CONTROL0: 0x1c00 [ 1638.322559] __ga10b__ NV_PGRAPH_PRI_GPC0_TPC2_SM1_DBGR_STATUS0: 0x0 [ 1638.329034] __ga10b__ NV_PGRAPH_PRI_GPC1_TPC0_SM0_HWW_WARP_ESR: 0x0 [ 1638.335503] __ga10b__ NV_PGRAPH_PRI_GPC1_TPC0_SM0_HWW_WARP_ESR_REPORT_MASK: 0xe812b60 [ 1638.343582] __ga10b__ NV_PGRAPH_PRI_GPC1_TPC0_SM0_HWW_GLOBAL_ESR: 0x0 [ 1638.350227] __ga10b__ NV_PGRAPH_PRI_GPC1_TPC0_SM0_HWW_GLOBAL_ESR_REPORT_MASK: 0x0 [ 1638.357944] __ga10b__ NV_PGRAPH_PRI_GPC1_TPC0_SM0_DBGR_CONTROL0: 0x1c00 [ 1638.364776] __ga10b__ NV_PGRAPH_PRI_GPC1_TPC0_SM0_DBGR_STATUS0: 0x0 [ 1638.371252] __ga10b__ NV_PGRAPH_PRI_GPC1_TPC0_SM1_HWW_WARP_ESR: 0x0 [ 1638.377725] __ga10b__ NV_PGRAPH_PRI_GPC1_TPC0_SM1_HWW_WARP_ESR_REPORT_MASK: 0xe812b60 [ 1638.385802] __ga10b__ NV_PGRAPH_PRI_GPC1_TPC0_SM1_HWW_GLOBAL_ESR: 0x0 [ 1638.392457] __ga10b__ NV_PGRAPH_PRI_GPC1_TPC0_SM1_HWW_GLOBAL_ESR_REPORT_MASK: 0x0 [ 1638.400182] __ga10b__ NV_PGRAPH_PRI_GPC1_TPC0_SM1_DBGR_CONTROL0: 0x1c00 [ 1638.406996] __ga10b__ NV_PGRAPH_PRI_GPC1_TPC0_SM1_DBGR_STATUS0: 0x0 [ 1638.413468] __ga10b__ NV_PGRAPH_PRI_GPC1_TPC1_SM0_HWW_WARP_ESR: 0x0 [ 1638.419941] __ga10b__ NV_PGRAPH_PRI_GPC1_TPC1_SM0_HWW_WARP_ESR_REPORT_MASK: 0xe812b60 [ 1638.428018] __ga10b__ NV_PGRAPH_PRI_GPC1_TPC1_SM0_HWW_GLOBAL_ESR: 0x0 [ 1638.434664] __ga10b__ NV_PGRAPH_PRI_GPC1_TPC1_SM0_HWW_GLOBAL_ESR_REPORT_MASK: 0x0 [ 1638.442389] __ga10b__ NV_PGRAPH_PRI_GPC1_TPC1_SM0_DBGR_CONTROL0: 0x1c00 [ 1638.449219] __ga10b__ NV_PGRAPH_PRI_GPC1_TPC1_SM0_DBGR_STATUS0: 0x0 [ 1638.455692] __ga10b__ NV_PGRAPH_PRI_GPC1_TPC1_SM1_HWW_WARP_ESR: 0x0 [ 1638.462165] __ga10b__ NV_PGRAPH_PRI_GPC1_TPC1_SM1_HWW_WARP_ESR_REPORT_MASK: 0xe812b60 [ 1638.470242] __ga10b__ NV_PGRAPH_PRI_GPC1_TPC1_SM1_HWW_GLOBAL_ESR: 0x0 [ 1638.476900] __ga10b__ NV_PGRAPH_PRI_GPC1_TPC1_SM1_HWW_GLOBAL_ESR_REPORT_MASK: 0x0 [ 1638.484620] __ga10b__ NV_PGRAPH_PRI_GPC1_TPC1_SM1_DBGR_CONTROL0: 0x1c00 [ 1638.491453] __ga10b__ NV_PGRAPH_PRI_GPC1_TPC1_SM1_DBGR_STATUS0: 0x0 [ 1638.497936] __ga10b__ NV_PGRAPH_PRI_GPC1_TPC2_SM0_HWW_WARP_ESR: 0x0 [ 1638.504406] __ga10b__ NV_PGRAPH_PRI_GPC1_TPC2_SM0_HWW_WARP_ESR_REPORT_MASK: 0xe812b60 [ 1638.512474] __ga10b__ NV_PGRAPH_PRI_GPC1_TPC2_SM0_HWW_GLOBAL_ESR: 0x0 [ 1638.519122] __ga10b__ NV_PGRAPH_PRI_GPC1_TPC2_SM0_HWW_GLOBAL_ESR_REPORT_MASK: 0x0 [ 1638.526838] __ga10b__ NV_PGRAPH_PRI_GPC1_TPC2_SM0_DBGR_CONTROL0: 0x1c00 [ 1638.533662] __ga10b__ NV_PGRAPH_PRI_GPC1_TPC2_SM0_DBGR_STATUS0: 0x0 [ 1638.540119] __ga10b__ NV_PGRAPH_PRI_GPC1_TPC2_SM1_HWW_WARP_ESR: 0x0 [ 1638.546585] __ga10b__ NV_PGRAPH_PRI_GPC1_TPC2_SM1_HWW_WARP_ESR_REPORT_MASK: 0x0 [ 1638.554126] __ga10b__ NV_PGRAPH_PRI_GPC1_TPC2_SM1_HWW_GLOBAL_ESR: 0x0 [ 1638.560775] __ga10b__ NV_PGRAPH_PRI_GPC1_TPC2_SM1_HWW_GLOBAL_ESR_REPORT_MASK: 0x1174 [ 1638.568763] __ga10b__ NV_PGRAPH_PRI_GPC1_TPC2_SM1_DBGR_CONTROL0: 0x1c00 [ 1638.575591] __ga10b__ NV_PGRAPH_PRI_GPC1_TPC2_SM1_DBGR_STATUS0: 0x0 [ 1638.582063] __ga10b__ NV_PGRAPH_PRI_GPC1_TPC3_SM0_HWW_WARP_ESR: 0x0 [ 1638.588533] __ga10b__ NV_PGRAPH_PRI_GPC1_TPC3_SM0_HWW_WARP_ESR_REPORT_MASK: 0xe812b60 [ 1638.596618] __ga10b__ NV_PGRAPH_PRI_GPC1_TPC3_SM0_HWW_GLOBAL_ESR: 0x0 [ 1638.603270] __ga10b__ NV_PGRAPH_PRI_GPC1_TPC3_SM0_HWW_GLOBAL_ESR_REPORT_MASK: 0x1174 [ 1638.611260] __ga10b__ NV_PGRAPH_PRI_GPC1_TPC3_SM0_DBGR_CONTROL0: 0x1c00 [ 1638.618087] __ga10b__ NV_PGRAPH_PRI_GPC1_TPC3_SM0_DBGR_STATUS0: 0x0 [ 1638.624557] __ga10b__ NV_PGRAPH_PRI_GPC1_TPC3_SM1_HWW_WARP_ESR: 0x0 [ 1638.631031] __ga10b__ NV_PGRAPH_PRI_GPC1_TPC3_SM1_HWW_WARP_ESR_REPORT_MASK: 0xe812b60 [ 1638.639098] __ga10b__ NV_PGRAPH_PRI_GPC1_TPC3_SM1_HWW_GLOBAL_ESR: 0x0 [ 1638.645748] __ga10b__ NV_PGRAPH_PRI_GPC1_TPC3_SM1_HWW_GLOBAL_ESR_REPORT_MASK: 0x1174 [ 1638.653744] __ga10b__ NV_PGRAPH_PRI_GPC1_TPC3_SM1_DBGR_CONTROL0: 0x1c00 [ 1638.660583] __ga10b__ NV_PGRAPH_PRI_GPC1_TPC3_SM1_DBGR_STATUS0: 0x0 [ 1638.667389] nvgpu: 17000000.ga10b nvgpu_tsg_force_reset_ch:469 [ERR] chid: 509 is not bound to tsg [ 1640.337196] cpufreq: cpu4,cur:2389000,set:2188800,set ndiv:171 [ 1640.859122] cpufreq: cpu4,cur:1154000,set:1036800,set ndiv:81