layog / accurate-binary-convolution-network Goto Github PK

Hi, @layog

I am glad to find your code, it is a great help for my understanding of the paper. But I am confused that how do you update the β and ν in activation layer? I notice that you have set the initial value, 1 for β and 0 for ν, is it the exact implementation of the paper which is described as statistical features?

How to make the network more accuracy?

A validation batch is defined to be X_batch, y_batch but it seems that you feed batch[0] and batch[1]. I tried feeding the X_batch and y_batch, still had good performance but could not get above 90%

tf.version=1.4.1

ValueError Traceback (most recent call last)
/data/xdong/miniconda3/lib/python3.6/site-packages/tensorflow/python/framework/op_def_library.py in _apply_op_helper(self, op_type_name, name, **keywords)
509 as_ref=input_arg.is_ref,
--> 510 preferred_dtype=default_dtype)
511 except TypeError as err:

/data/xdong/miniconda3/lib/python3.6/site-packages/tensorflow/python/framework/ops.py in internal_convert_to_tensor(value, dtype, name, as_ref, preferred_dtype, ctx)
925 if ret is None:
--> 926 ret = conversion_func(value, dtype=dtype, name=name, as_ref=as_ref)
927

/data/xdong/miniconda3/lib/python3.6/site-packages/tensorflow/python/framework/constant_op.py in _constant_tensor_conversion_function(v, dtype, name, as_ref)
228 _ = as_ref
--> 229 return constant(v, dtype=dtype, name=name)
230

/data/xdong/miniconda3/lib/python3.6/site-packages/tensorflow/python/framework/constant_op.py in constant(value, dtype, shape, name, verify_shape)
207 tensor_util.make_tensor_proto(
--> 208 value, dtype=dtype, shape=shape, verify_shape=verify_shape))
209 dtype_value = attr_value_pb2.AttrValue(type=tensor_value.tensor.dtype)

/data/xdong/miniconda3/lib/python3.6/site-packages/tensorflow/python/framework/tensor_util.py in make_tensor_proto(values, dtype, shape, verify_shape)
391 values, list(nparray.shape),
--> 392 _GetDenseDimensions(values)))
393

ValueError: Argument must be a dense tensor: range(0, 4) - got shape [4], but wanted [].

During handling of the above exception, another exception occurred:

ValueError Traceback (most recent call last)
/data/xdong/miniconda3/lib/python3.6/site-packages/tensorflow/python/framework/op_def_library.py in _apply_op_helper(self, op_type_name, name, **keywords)
523 observed = ops.internal_convert_to_tensor(
--> 524 values, as_ref=input_arg.is_ref).dtype.name
525 except ValueError as err:

/data/xdong/miniconda3/lib/python3.6/site-packages/tensorflow/python/framework/ops.py in internal_convert_to_tensor(value, dtype, name, as_ref, preferred_dtype, ctx)
925 if ret is None:
--> 926 ret = conversion_func(value, dtype=dtype, name=name, as_ref=as_ref)
927

/data/xdong/miniconda3/lib/python3.6/site-packages/tensorflow/python/framework/constant_op.py in _constant_tensor_conversion_function(v, dtype, name, as_ref)
228 _ = as_ref
--> 229 return constant(v, dtype=dtype, name=name)
230

/data/xdong/miniconda3/lib/python3.6/site-packages/tensorflow/python/framework/constant_op.py in constant(value, dtype, shape, name, verify_shape)
207 tensor_util.make_tensor_proto(
--> 208 value, dtype=dtype, shape=shape, verify_shape=verify_shape))
209 dtype_value = attr_value_pb2.AttrValue(type=tensor_value.tensor.dtype)

/data/xdong/miniconda3/lib/python3.6/site-packages/tensorflow/python/framework/tensor_util.py in make_tensor_proto(values, dtype, shape, verify_shape)
391 values, list(nparray.shape),
--> 392 _GetDenseDimensions(values)))
393

ValueError: Argument must be a dense tensor: range(0, 4) - got shape [4], but wanted [].

During handling of the above exception, another exception occurred:

ValueError Traceback (most recent call last)
in ()
13 no_binary_filters=5,
14 no_ApproxConvLayers=5,
---> 15 padding="SAME")
16 alphas_training_operations.append(alphas_training_op1)
17 conv1 = ABCLayer1(x_image)

in ABC(convolution_filters, convolution_biases, no_binary_filters, no_ApproxConvLayers, strides, padding, name)
11 alphas_training_op, ApproxConvLayer, alphas_loss = ApproxConv(no_binary_filters,
12 convolution_filters, convolution_biases,
---> 13 strides, padding)
14
15 def ABCLayer(input_tensor, name=None):

in ApproxConv(no_filters, convolution_filters, convolution_biases, strides, padding, name)
10
11 # Creating binary filters
---> 12 binary_filters = get_binary_filters(filters, no_filters)
13
14 # Getting alphas

in get_binary_filters(convolution_filters, no_filters, name)
1 def get_binary_filters(convolution_filters, no_filters, name=None):
2 with tf.name_scope(name, default_name="get_binary_filters"):
----> 3 mean, stddev = get_mean_stddev(convolution_filters)
4 shifted_stddev = get_shifted_stddev(stddev, no_filters)
5

in get_mean_stddev(input_tensor)
1 def get_mean_stddev(input_tensor):
2 with tf.name_scope('mean_stddev_cal'):
----> 3 mean, variance = tf.nn.moments(input_tensor, axes=range(len(input_tensor.get_shape())))
4 stddev = tf.sqrt(variance, name="standard_deviation")
5 return mean, stddev

/data/xdong/miniconda3/lib/python3.6/site-packages/tensorflow/python/ops/nn_impl.py in moments(x, axes, shift, name, keep_dims)
616 y = math_ops.cast(x, dtypes.float32) if x.dtype == dtypes.float16 else x
617 # Compute true mean while keeping the dims for proper broadcasting.
--> 618 mean = math_ops.reduce_mean(y, axes, keep_dims=True, name="mean")
619 # sample variance, not unbiased variance
620 variance = math_ops.reduce_mean(

/data/xdong/miniconda3/lib/python3.6/site-packages/tensorflow/python/ops/math_ops.py in reduce_mean(input_tensor, axis, keep_dims, name, reduction_indices)
1409 _ReductionDims(input_tensor, axis, reduction_indices),
1410 keep_dims,
-> 1411 name=name)
1412
1413

/data/xdong/miniconda3/lib/python3.6/site-packages/tensorflow/python/ops/gen_math_ops.py in _mean(input, reduction_indices, keep_dims, name)
2566 _, _, _op = _op_def_lib._apply_op_helper(
2567 "Mean", input=input, reduction_indices=reduction_indices,
-> 2568 keep_dims=keep_dims, name=name)
2569 _result = _op.outputs[:]
2570 _inputs_flat = _op.inputs

/data/xdong/miniconda3/lib/python3.6/site-packages/tensorflow/python/framework/op_def_library.py in _apply_op_helper(self, op_type_name, name, **keywords)
526 raise ValueError(
527 "Tried to convert '%s' to a tensor and failed. Error: %s" %
--> 528 (input_name, err))
529 prefix = ("Input '%s' of '%s' Op has type %s that does not match" %
530 (input_name, op_type_name, observed))

ValueError: Tried to convert 'reduction_indices' to a tensor and failed. Error: Argument must be a dense tensor: range(0, 4) - got shape [4], but wanted [].

I read your codes carefully and it is great. And I want to know how did you implement the STE in equation 10?

Hi,

I don't think weights get updated because of the tf.sign function cutting off backprop gradients:

binarized_filters = tf.sign(tiled_filters + expanded_stddev, name="binarized_filters")

i believe this line is needed:

with tf.get_default_graph().gradient_override_map({"Sign": "Identity"}):

Hi,thank you for code.

I test the resnet-34 on cifar-10. the result is poor.

When is the imageNet model added?

Thanks.