Loading experiment spec at /workspace/fire_detection/fire/specs/unet_train_resnet_unet_isbi.txt. Loading experiment spec at /workspace/fire_detection/fire/specs/unet_train_resnet_unet_isbi.txt. Loading experiment spec at /workspace/fire_detection/fire/specs/unet_train_resnet_unet_isbi.txt. Running for 500 Epochs Epoch: 0/500:, Cur-Step: 0, loss(cross_dice_sum): 1.05657, Running average loss:1.05657, Time taken: 0:00:00 ETA: 0:00:00 Epoch: 0/500:, Cur-Step: 20, loss(cross_dice_sum): 1.02958, Running average loss:1.04630, Time taken: 0:00:00 ETA: 0:00:00 Epoch: 0/500:, Cur-Step: 40, loss(cross_dice_sum): 0.98391, Running average loss:1.02549, Time taken: 0:00:00 ETA: 0:00:00 Epoch: 0/500:, Cur-Step: 60, loss(cross_dice_sum): 0.30164, Running average loss:0.94165, Time taken: 0:00:00 ETA: 0:00:00 Epoch: 0/500:, Cur-Step: 80, loss(cross_dice_sum): 0.06499, Running average loss:0.74509, Time taken: 0:00:00 ETA: 0:00:00 Epoch: 0/500:, Cur-Step: 100, loss(cross_dice_sum): 0.16813, Running average loss:0.62823, Time taken: 0:00:00 ETA: 0:00:00 Epoch: 0/500:, Cur-Step: 120, loss(cross_dice_sum): 0.32043, Running average loss:0.55022, Time taken: 0:00:00 ETA: 0:00:00 Epoch: 0/500:, Cur-Step: 140, loss(cross_dice_sum): 0.14016, Running average loss:0.49091, Time taken: 0:00:00 ETA: 0:00:00 Epoch: 0/500:, Cur-Step: 160, loss(cross_dice_sum): 0.08215, Running average loss:0.44645, Time taken: 0:00:00 ETA: 0:00:00 Epoch: 0/500:, Cur-Step: 180, loss(cross_dice_sum): 0.12781, Running average loss:0.41188, Time taken: 0:00:00 ETA: 0:00:00 Epoch: 0/500:, Cur-Step: 200, loss(cross_dice_sum): 0.09447, Running average loss:0.38336, Time taken: 0:00:00 ETA: 0:00:00 Epoch: 0/500:, Cur-Step: 220, loss(cross_dice_sum): 0.06719, Running average loss:0.35967, Time taken: 0:00:00 ETA: 0:00:00 Epoch: 0/500:, Cur-Step: 240, loss(cross_dice_sum): 0.08263, Running average loss:0.34049, Time taken: 0:00:00 ETA: 0:00:00 Epoch: 0/500:, Cur-Step: 260, loss(cross_dice_sum): 0.08694, Running average loss:0.32330, Time taken: 0:00:00 ETA: 0:00:00 Epoch: 0/500:, Cur-Step: 280, loss(cross_dice_sum): 0.13394, Running average loss:0.30921, Time taken: 0:00:00 ETA: 0:00:00 Epoch: 0/500:, Cur-Step: 300, loss(cross_dice_sum): 0.10410, Running average loss:0.29697, Time taken: 0:00:00 ETA: 0:00:00 Epoch: 0/500:, Cur-Step: 320, loss(cross_dice_sum): 0.10743, Running average loss:0.28627, Time taken: 0:00:00 ETA: 0:00:00 Epoch: 0/500:, Cur-Step: 340, loss(cross_dice_sum): 0.14588, Running average loss:0.27626, Time taken: 0:00:00 ETA: 0:00:00 Epoch: 0/500:, Cur-Step: 360, loss(cross_dice_sum): 0.07170, Running average loss:0.26697, Time taken: 0:00:00 ETA: 0:00:00 Epoch: 0/500:, Cur-Step: 380, loss(cross_dice_sum): 0.10115, Running average loss:0.25836, Time taken: 0:00:00 ETA: 0:00:00 Epoch: 0/500:, Cur-Step: 400, loss(cross_dice_sum): 0.07779, Running average loss:0.25108, Time taken: 0:00:00 ETA: 0:00:00 Epoch: 0/500:, Cur-Step: 420, loss(cross_dice_sum): 0.07632, Running average loss:0.24425, Time taken: 0:00:00 ETA: 0:00:00 Epoch: 0/500:, Cur-Step: 440, loss(cross_dice_sum): 0.13788, Running average loss:0.23811, Time taken: 0:00:00 ETA: 0:00:00 Epoch: 0/500:, Cur-Step: 460, loss(cross_dice_sum): 0.12675, Running average loss:0.23260, Time taken: 0:00:00 ETA: 0:00:00 Epoch: 0/500:, Cur-Step: 480, loss(cross_dice_sum): 0.13705, Running average loss:0.22770, Time taken: 0:00:00 ETA: 0:00:00 Epoch: 0/500:, Cur-Step: 500, loss(cross_dice_sum): 0.14140, Running average loss:0.22270, Time taken: 0:00:00 ETA: 0:00:00 Epoch: 0/500:, Cur-Step: 520, loss(cross_dice_sum): 0.13328, Running average loss:0.21819, Time taken: 0:00:00 ETA: 0:00:00 Epoch: 0/500:, Cur-Step: 540, loss(cross_dice_sum): 0.09213, Running average loss:0.21386, Time taken: 0:00:00 ETA: 0:00:00 Epoch: 0/500:, Cur-Step: 560, loss(cross_dice_sum): 0.13346, Running average loss:0.21019, Time taken: 0:00:00 ETA: 0:00:00 Epoch: 0/500:, Cur-Step: 580, loss(cross_dice_sum): 0.11532, Running average loss:0.20658, Time taken: 0:00:00 ETA: 0:00:00 Epoch: 0/500:, Cur-Step: 600, loss(cross_dice_sum): 0.08870, Running average loss:0.20305, Time taken: 0:00:00 ETA: 0:00:00 Epoch: 0/500:, Cur-Step: 620, loss(cross_dice_sum): 0.13362, Running average loss:0.19988, Time taken: 0:00:00 ETA: 0:00:00 Epoch: 0/500:, Cur-Step: 640, loss(cross_dice_sum): 0.06559, Running average loss:0.19654, Time taken: 0:00:00 ETA: 0:00:00 Epoch: 0/500:, Cur-Step: 660, loss(cross_dice_sum): 0.09163, Running average loss:0.19372, Time taken: 0:00:00 ETA: 0:00:00 Epoch: 0/500:, Cur-Step: 680, loss(cross_dice_sum): 0.05889, Running average loss:0.19069, Time taken: 0:00:00 ETA: 0:00:00 Epoch: 1/500:, Cur-Step: 700, loss(cross_dice_sum): 0.12825, Running average loss:0.12825, Time taken: 0:01:51.901783 ETA: 15:30:38.989712 Epoch: 1/500:, Cur-Step: 720, loss(cross_dice_sum): 0.09741, Running average loss:0.10159, Time taken: 0:01:51.901783 ETA: 15:30:38.989712 Epoch: 1/500:, Cur-Step: 740, loss(cross_dice_sum): 0.08173, Running average loss:0.10458, Time taken: 0:01:51.901783 ETA: 15:30:38.989712 Epoch: 1/500:, Cur-Step: 760, loss(cross_dice_sum): 0.13142, Running average loss:0.10438, Time taken: 0:01:51.901783 ETA: 15:30:38.989712 Epoch: 1/500:, Cur-Step: 780, loss(cross_dice_sum): 0.06853, Running average loss:0.10087, Time taken: 0:01:51.901783 ETA: 15:30:38.989712 Epoch: 1/500:, Cur-Step: 800, loss(cross_dice_sum): 0.10105, Running average loss:0.10105, Time taken: 0:01:51.901783 ETA: 15:30:38.989712