hello, I meet a problem.I use RTX4080,with docker and cuda10.1, tf1.9.0,and I meet the flowing issue: about dh3d

[bobododosjl]

2023-03-31 07:40:30.825504: E tensorflow/stream_executor/cuda/cuda_blas.cc:647] failed to run cuBLAS routine cublasSgemm_v2: CUBLAS_STATUS_EXECUTION_FAILED
Traceback (most recent call last):
File "/root/anaconda3/envs/DH3D/lib/python3.6/site-packages/tensorflow/python/client/session.py", line 1322, in _do_call
return fn(*args)
File "/root/anaconda3/envs/DH3D/lib/python3.6/site-packages/tensorflow/python/client/session.py", line 1307, in _run_fn
options, feed_dict, fetch_list, target_list, run_metadata)
File "/root/anaconda3/envs/DH3D/lib/python3.6/site-packages/tensorflow/python/client/session.py", line 1409, in _call_tf_sessionrun
run_metadata)
tensorflow.python.framework.errors_impl.InternalError: Blas SGEMM launch failed : m=65536, n=16, k=64
[[Node: stage1/se/f1/tfconv0/Conv2D = Conv2D[T=DT_FLOAT, data_format="NHWC", dilations=[1, 1, 1, 1], padding="VALID", strides=[1, 1, 1, 1], use_cudnn_on_gpu=true, _device="/job:localhost/replica:0/task:0/device:GPU:0"](stage1/se/ExpandDims, stage1/se/f1/tfconv0/W/read)]]

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
File "localdesc_extract.py", line 176, in
pred_local_oxford(args)
File "localdesc_extract.py", line 155, in pred_local_oxford
totalnum = perform_pred(df, totalbatch, predictor, eval_args)
File "localdesc_extract.py", line 125, in perform_pred
result = predictor(pc, knn_ind)[0]
File "/root/anaconda3/envs/DH3D/lib/python3.6/site-packages/tensorpack/predict/base.py", line 41, in call
output = self._do_call(dp)
File "/root/anaconda3/envs/DH3D/lib/python3.6/site-packages/tensorpack/predict/base.py", line 136, in _do_call
return self._callable(*dp)
File "/root/anaconda3/envs/DH3D/lib/python3.6/site-packages/tensorflow/python/client/session.py", line 1187, in _generic_run
return self.run(fetches, feed_dict=feed_dict, **kwargs)
File "/root/anaconda3/envs/DH3D/lib/python3.6/site-packages/tensorflow/python/client/session.py", line 900, in run
run_metadata_ptr)
File "/root/anaconda3/envs/DH3D/lib/python3.6/site-packages/tensorflow/python/client/session.py", line 1135, in _run
feed_dict_tensor, options, run_metadata)
File "/root/anaconda3/envs/DH3D/lib/python3.6/site-packages/tensorflow/python/client/session.py", line 1316, in _do_run
run_metadata)
File "/root/anaconda3/envs/DH3D/lib/python3.6/site-packages/tensorflow/python/client/session.py", line 1335, in _do_call
raise type(e)(node_def, op, message)
tensorflow.python.framework.errors_impl.InternalError: Blas SGEMM launch failed : m=65536, n=16, k=64
[[Node: stage1/se/f1/tfconv0/Conv2D = Conv2D[T=DT_FLOAT, data_format="NHWC", dilations=[1, 1, 1, 1], padding="VALID", strides=[1, 1, 1, 1], use_cudnn_on_gpu=true, _device="/job:localhost/replica:0/task:0/device:GPU:0"](stage1/se/ExpandDims, stage1/se/f1/tfconv0/W/read)]]

Caused by op 'stage1/se/f1/tfconv0/Conv2D', defined at:
File "localdesc_extract.py", line 176, in
pred_local_oxford(args)
File "localdesc_extract.py", line 149, in pred_local_oxford
predictor = get_predictor(model_config, eval_args.ModelPath)
File "localdesc_extract.py", line 81, in get_predictor
predictor = OfflinePredictor(pred_config)
File "/root/anaconda3/envs/DH3D/lib/python3.6/site-packages/tensorpack/predict/base.py", line 166, in init
config.tower_func(*input.get_input_tensors())
File "/root/anaconda3/envs/DH3D/lib/python3.6/site-packages/tensorpack/tfutils/tower.py", line 294, in call
output = self._tower_fn(*args)
File "/home/git/DH3D/core/model.py", line 176, in build_graph
newpoints, localdesc = self.compute_local(points, freeze_local)
File "/home/git/DH3D/core/model.py", line 107, in compute_local
newpoints, localdesc = getattr(backbones, self.local_backbone)(**inputs_dict)
File "/home/git/DH3D/core/backbones.py", line 114, in backbone_local_dilate
knn_indices=nn_8, concat=False, add_se='max_pool')
File "/home/git/DH3D/core/backbones.py", line 79, in flex_conv_dilate
newx = se_res_bottleneck(x, x_pool, outdims[-1], "se") # l: B, 64, N
File "/home/git/DH3D/core/backbones.py", line 50, in se_res_bottleneck
squeeze = feature_conv1d_1(pool_l_T, ch_out // 4, 'f1', ac_func=tf.nn.relu, c_last=True)
File "/home/git/DH3D/core/tf_utils.py", line 105, in feature_conv1d_1
newfeat = Conv2D('tfconv0', feat, dim, kernel_shape=1, padding='VALID', activation=ac_func)
File "/root/anaconda3/envs/DH3D/lib/python3.6/site-packages/tensorpack/models/registry.py", line 173, in wrapped_func
outputs = func(*args, **actual_args)
File "/root/anaconda3/envs/DH3D/lib/python3.6/site-packages/tensorpack/models/tflayer.py", line 66, in decorated_func
return func(inputs, **ret)
File "/root/anaconda3/envs/DH3D/lib/python3.6/site-packages/tensorpack/models/conv2d.py", line 76, in Conv2D
ret = layer.apply(inputs, scope=tf.get_variable_scope())
File "/root/anaconda3/envs/DH3D/lib/python3.6/site-packages/tensorflow/python/keras/engine/base_layer.py", line 774, in apply
return self.call(inputs, *args, **kwargs)
File "/root/anaconda3/envs/DH3D/lib/python3.6/site-packages/tensorflow/python/layers/base.py", line 329, in call
outputs = super(Layer, self).call(inputs, *args, **kwargs)
File "/root/anaconda3/envs/DH3D/lib/python3.6/site-packages/tensorflow/python/keras/engine/base_layer.py", line 703, in call
outputs = self.call(inputs, *args, **kwargs)
File "/root/anaconda3/envs/DH3D/lib/python3.6/site-packages/tensorflow/python/keras/layers/convolutional.py", line 184, in call
outputs = self._convolution_op(inputs, self.kernel)
File "/root/anaconda3/envs/DH3D/lib/python3.6/site-packages/tensorflow/python/ops/nn_ops.py", line 868, in call
return self.conv_op(inp, filter)
File "/root/anaconda3/envs/DH3D/lib/python3.6/site-packages/tensorflow/python/ops/nn_ops.py", line 520, in call
return self.call(inp, filter)
File "/root/anaconda3/envs/DH3D/lib/python3.6/site-packages/tensorflow/python/ops/nn_ops.py", line 204, in call
name=self.name)
File "/root/anaconda3/envs/DH3D/lib/python3.6/site-packages/tensorflow/python/ops/gen_nn_ops.py", line 956, in conv2d
data_format=data_format, dilations=dilations, name=name)
File "/root/anaconda3/envs/DH3D/lib/python3.6/site-packages/tensorflow/python/framework/op_def_library.py", line 787, in _apply_op_helper
op_def=op_def)
File "/root/anaconda3/envs/DH3D/lib/python3.6/site-packages/tensorflow/python/framework/ops.py", line 3414, in create_op
op_def=op_def)
File "/root/anaconda3/envs/DH3D/lib/python3.6/site-packages/tensorflow/python/framework/ops.py", line 1740, in init
self._traceback = self._graph._extract_stack() # pylint: disable=protected-access

InternalError (see above for traceback): Blas SGEMM launch failed : m=65536, n=16, k=64
[[Node: stage1/se/f1/tfconv0/Conv2D = Conv2D[T=DT_FLOAT, data_format="NHWC", dilations=[1, 1, 1, 1], padding="VALID", strides=[1, 1, 1, 1], use_cudnn_on_gpu=true, _device="/job:localhost/replica:0/task:0/device:GPU:0"](stage1/se/ExpandDims, stage1/se/f1/tfconv0/W/read)]]