I try to load a trained model using the function “keras.model.load_model” but I get an OOM error. My model is of memory 235 MB. This is the error trace:
…
8] 1 Chunks of size 35651584 totalling 34.00MiB
2020-09-28 11:44:30.540303: I tensorflow/core/common_runtime/bfc_allocator.cc:1002] Sum Total of in-use chunks: 283.36MiB
2020-09-28 11:44:30.540323: I tensorflow/core/common_runtime/bfc_allocator.cc:1004] total_region_allocated_bytes_: 298536960 memory_limit_: 298536960 available bytes: 0 curr_region_allocation_bytes_: 597073920
2020-09-28 11:44:30.565096: I tensorflow/core/common_runtime/bfc_allocator.cc:1010] Stats:
Limit: 298536960
InUse: 297128960
MaxInUse: 298171136
NumAllocs: 1582
MaxAllocSize: 35651584
2020-09-28 11:44:30.565202: W tensorflow/core/common_runtime/bfc_allocator.cc:439] xxxx**************************xx
2020-09-28 11:44:30.565399: W tensorflow/core/framework/op_kernel.cc:1753] OP_REQUIRES failed at cwise_ops_common.h:134 : Resource exhausted: OOM when allocating tensor with shape[3,3,128,256] and type float on /job:localhost/replica:0/task:0/device:GPU:0 by allocator GPU_0_bfc
Traceback (most recent call last):
File “”, line 1, in
File “/usr/local/lib/python3.6/dist-packages/tensorflow/python/keras/saving/save.py”, line 184, in load_model
return hdf5_format.load_model_from_hdf5(filepath, custom_objects, compile)
File “/usr/local/lib/python3.6/dist-packages/tensorflow/python/keras/saving/hdf5_format.py”, line 178, in load_model_from_hdf5
custom_objects=custom_objects)
File “/usr/local/lib/python3.6/dist-packages/tensorflow/python/keras/saving/model_config.py”, line 55, in model_from_config
return deserialize(config, custom_objects=custom_objects)
File “/usr/local/lib/python3.6/dist-packages/tensorflow/python/keras/layers/serialization.py”, line 109, in deserialize
printable_module_name=‘layer’)
File “/usr/local/lib/python3.6/dist-packages/tensorflow/python/keras/utils/generic_utils.py”, line 373, in deserialize_keras_object
list(custom_objects.items())))
File “/usr/local/lib/python3.6/dist-packages/tensorflow/python/keras/engine/network.py”, line 987, in from_config
config, custom_objects)
File “/usr/local/lib/python3.6/dist-packages/tensorflow/python/keras/engine/network.py”, line 2029, in reconstruct_from_config
process_node(layer, node_data)
File “/usr/local/lib/python3.6/dist-packages/tensorflow/python/keras/engine/network.py”, line 1977, in process_node
output_tensors = layer(input_tensors, **kwargs)
File “/usr/local/lib/python3.6/dist-packages/tensorflow/python/keras/engine/base_layer.py”, line 897, in call
self._maybe_build(inputs)
File “/usr/local/lib/python3.6/dist-packages/tensorflow/python/keras/engine/base_layer.py”, line 2416, in _maybe_build
self.build(input_shapes) # pylint:disable=not-callable
File “/usr/local/lib/python3.6/dist-packages/tensorflow/python/keras/layers/convolutional.py”, line 166, in build
dtype=self.dtype)
File “/usr/local/lib/python3.6/dist-packages/tensorflow/python/keras/engine/base_layer.py”, line 577, in add_weight
caching_device=caching_device)
File “/usr/local/lib/python3.6/dist-packages/tensorflow/python/training/tracking/base.py”, line 743, in _add_variable_with_custom_getter
**kwargs_for_getter)
File “/usr/local/lib/python3.6/dist-packages/tensorflow/python/keras/engine/base_layer_utils.py”, line 141, in make_variable
shape=variable_shape if variable_shape else None)
File “/usr/local/lib/python3.6/dist-packages/tensorflow/python/ops/variables.py”, line 259, in call
return cls._variable_v1_call(*args, **kwargs)
File “/usr/local/lib/python3.6/dist-packages/tensorflow/python/ops/variables.py”, line 220, in _variable_v1_call
shape=shape)
File “/usr/local/lib/python3.6/dist-packages/tensorflow/python/ops/variables.py”, line 198, in
previous_getter = lambda **kwargs: default_variable_creator(None, **kwargs)
File “/usr/local/lib/python3.6/dist-packages/tensorflow/python/ops/variable_scope.py”, line 2598, in default_variable_creator
shape=shape)
File “/usr/local/lib/python3.6/dist-packages/tensorflow/python/ops/variables.py”, line 263, in call
return super(VariableMetaclass, cls).call(*args, **kwargs)
File “/usr/local/lib/python3.6/dist-packages/tensorflow/python/ops/resource_variable_ops.py”, line 1434, in init
distribute_strategy=distribute_strategy)
File “/usr/local/lib/python3.6/dist-packages/tensorflow/python/ops/resource_variable_ops.py”, line 1567, in _init_from_args
initial_value() if init_from_fn else initial_value,
File “/usr/local/lib/python3.6/dist-packages/tensorflow/python/keras/engine/base_layer_utils.py”, line 121, in
init_val = lambda: initializer(shape, dtype=dtype)
File “/usr/local/lib/python3.6/dist-packages/tensorflow/python/ops/init_ops_v2.py”, line 558, in call
return self._random_generator.random_uniform(shape, -limit, limit, dtype)
File “/usr/local/lib/python3.6/dist-packages/tensorflow/python/ops/init_ops_v2.py”, line 1068, in random_uniform
shape=shape, minval=minval, maxval=maxval, dtype=dtype, seed=self.seed)
File “/usr/local/lib/python3.6/dist-packages/tensorflow/python/ops/random_ops.py”, line 301, in random_uniform
result = math_ops.add(result * (maxval - minval), minval, name=name)
File “/usr/local/lib/python3.6/dist-packages/tensorflow/python/ops/math_ops.py”, line 984, in binary_op_wrapper
return func(x, y, name=name)
File “/usr/local/lib/python3.6/dist-packages/tensorflow/python/ops/math_ops.py”, line 1283, in _mul_dispatch
return gen_math_ops.mul(x, y, name=name)
File “/usr/local/lib/python3.6/dist-packages/tensorflow/python/ops/gen_math_ops.py”, line 6089, in mul
_ops.raise_from_not_ok_status(e, name)
File “/usr/local/lib/python3.6/dist-packages/tensorflow/python/framework/ops.py”, line 6653, in raise_from_not_ok_status
six.raise_from(core._status_to_exception(e.code, message), None)
File “”, line 3, in raise_from
tensorflow.python.framework.errors_impl.ResourceExhaustedError: OOM when allocating tensor with shape[3,3,128,256] and type float on /job:localhost/replica:0/task:0/device:GPU:0 by allocator GPU_0_bfc [Op:Mul]
Any help please?