Can you explain codes in https://github.com/soskek/attention_is_all_you_need/blob/master/net.py#L308-L340, I have to implemented the code of another similar paper which is inspired by "all you need": https://arxiv.org/pdf/1711.11575.pdf , in bottom of page 3. The author said: This 4-d feature is embedded to a high-dimensional representation by method in [49], which computes cosine and sine functions of different wavelengths. The feature dimension after embedding is dg.

How to understand the position sin& cos embedding, I have read the two paper, And I still can not understand this part.

When I try to run your code, I run into a problem like bellow and I don't know how to fix it:

Exception in main training loop: Out of memory allocating 255,688,704 bytes (allocated so far: 3,670,475,776 bytes).
Traceback (most recent call last):
File "C:\Users\phong\AppData\Local\Programs\Python\Python37\lib\site-packages\chainer\training\trainer.py", line 343, in run
update()
File "C:\Users\phong\AppData\Local\Programs\Python\Python37\lib\site-packages\chainer\training\updaters\standard_updater.py", line 240, in update
self.update_core()
File "C:\Users\phong\AppData\Local\Programs\Python\Python37\lib\site-packages\chainer\training\updaters\standard_updater.py", line 253, in update_core
optimizer.update(loss_func, *in_arrays)
File "C:\Users\phong\AppData\Local\Programs\Python\Python37\lib\site-packages\chainer\optimizer.py", line 876, in update
loss.backward(loss_scale=self._loss_scale)
File "C:\Users\phong\AppData\Local\Programs\Python\Python37\lib\site-packages\chainer\variable.py", line 1582, in backward
[(node, grad_var)], retain_grad, loss_scale)
File "C:\Users\phong\AppData\Local\Programs\Python\Python37\lib\site-packages\chainer_backprop.py", line 226, in _backprop_to_all
func, target_input_indexes, out_grad, in_grad, is_debug)
File "C:\Users\phong\AppData\Local\Programs\Python\Python37\lib\site-packages\chainer_backprop_utils.py", line 140, in backprop_step
_reraise_with_stack(func, e)
File "C:\Users\phong\AppData\Local\Programs\Python\Python37\lib\site-packages\chainer_backprop_utils.py", line 138, in backprop_step
target_input_indexes, grad_outputs)
File "C:\Users\phong\AppData\Local\Programs\Python\Python37\lib\site-packages\chainer\functions\math\basic_math.py", line 373, in backward
for i in indexes
File "C:\Users\phong\AppData\Local\Programs\Python\Python37\lib\site-packages\chainer\functions\math\basic_math.py", line 373, in
for i in indexes
File "C:\Users\phong\AppData\Local\Programs\Python\Python37\lib\site-packages\chainer\functions\math\basic_math.py", line 413, in mul
return Mul().apply((self, rhs))[0]
File "C:\Users\phong\AppData\Local\Programs\Python\Python37\lib\site-packages\chainer\function_node.py", line 334, in apply
outputs = self.forward(in_data)
File "C:\Users\phong\AppData\Local\Programs\Python\Python37\lib\site-packages\chainer\functions\math\basic_math.py", line 367, in forward
return utils.force_array(x[0] * x[1]),
File "cupy\core\core.pyx", line 960, in cupy.core.core.ndarray.mul
File "cupy\core_kernel.pyx", line 909, in cupy.core._kernel.ufunc.call
File "cupy\core_kernel.pyx", line 381, in cupy.core._kernel._get_out_args
File "cupy\core\core.pyx", line 134, in cupy.core.core.ndarray.init
File "cupy\cuda\memory.pyx", line 540, in cupy.cuda.memory.alloc
File "cupy\cuda\memory.pyx", line 1234, in cupy.cuda.memory.MemoryPool.malloc
File "cupy\cuda\memory.pyx", line 1255, in cupy.cuda.memory.MemoryPool.malloc
File "cupy\cuda\memory.pyx", line 1033, in cupy.cuda.memory.SingleDeviceMemoryPool.malloc
File "cupy\cuda\memory.pyx", line 1053, in cupy.cuda.memory.SingleDeviceMemoryPool._malloc
File "cupy\cuda\memory.pyx", line 775, in cupy.cuda.memory._try_malloc
Will finalize trainer extensions and updater before reraising the exception.
Traceback (most recent call last):
File "train.py", line 348, in
main()
File "train.py", line 344, in main
trainer.run()
File "C:\Users\phong\AppData\Local\Programs\Python\Python37\lib\site-packages\chainer\training\trainer.py", line 376, in run
six.reraise(*exc_info)
File "C:\Users\phong\AppData\Roaming\Python\Python37\site-packages\six.py", line 693, in reraise
raise value
File "C:\Users\phong\AppData\Local\Programs\Python\Python37\lib\site-packages\chainer\training\trainer.py", line 343, in run
update()
File "C:\Users\phong\AppData\Local\Programs\Python\Python37\lib\site-packages\chainer\training\updaters\standard_updater.py", line 240, in update
self.update_core()
File "C:\Users\phong\AppData\Local\Programs\Python\Python37\lib\site-packages\chainer\training\updaters\standard_updater.py", line 253, in update_core
optimizer.update(loss_func, *in_arrays)
File "C:\Users\phong\AppData\Local\Programs\Python\Python37\lib\site-packages\chainer\optimizer.py", line 876, in update
loss.backward(loss_scale=self._loss_scale)
File "C:\Users\phong\AppData\Local\Programs\Python\Python37\lib\site-packages\chainer\variable.py", line 1582, in backward
[(node, grad_var)], retain_grad, loss_scale)
File "C:\Users\phong\AppData\Local\Programs\Python\Python37\lib\site-packages\chainer_backprop.py", line 226, in _backprop_to_all
func, target_input_indexes, out_grad, in_grad, is_debug)
File "C:\Users\phong\AppData\Local\Programs\Python\Python37\lib\site-packages\chainer_backprop_utils.py", line 140, in backprop_step
_reraise_with_stack(func, e)
File "C:\Users\phong\AppData\Local\Programs\Python\Python37\lib\site-packages\chainer_backprop_utils.py", line 138, in backprop_step
target_input_indexes, grad_outputs)
File "C:\Users\phong\AppData\Local\Programs\Python\Python37\lib\site-packages\chainer\functions\math\basic_math.py", line 373, in backward
for i in indexes
File "C:\Users\phong\AppData\Local\Programs\Python\Python37\lib\site-packages\chainer\functions\math\basic_math.py", line 373, in
for i in indexes
File "C:\Users\phong\AppData\Local\Programs\Python\Python37\lib\site-packages\chainer\functions\math\basic_math.py", line 413, in mul
return Mul().apply((self, rhs))[0]
File "C:\Users\phong\AppData\Local\Programs\Python\Python37\lib\site-packages\chainer\function_node.py", line 334, in apply
outputs = self.forward(in_data)
File "C:\Users\phong\AppData\Local\Programs\Python\Python37\lib\site-packages\chainer\functions\math\basic_math.py", line 367, in forward
return utils.force_array(x[0] * x[1]),
File "cupy\core\core.pyx", line 960, in cupy.core.core.ndarray.mul
File "cupy\core_kernel.pyx", line 909, in cupy.core._kernel.ufunc.call
File "cupy\core_kernel.pyx", line 381, in cupy.core._kernel._get_out_args
File "cupy\core\core.pyx", line 134, in cupy.core.core.ndarray.init
File "cupy\cuda\memory.pyx", line 540, in cupy.cuda.memory.alloc
File "cupy\cuda\memory.pyx", line 1234, in cupy.cuda.memory.MemoryPool.malloc
File "cupy\cuda\memory.pyx", line 1255, in cupy.cuda.memory.MemoryPool.malloc
File "cupy\cuda\memory.pyx", line 1033, in cupy.cuda.memory.SingleDeviceMemoryPool.malloc
File "cupy\cuda\memory.pyx", line 1053, in cupy.cuda.memory.SingleDeviceMemoryPool._malloc
File "cupy\cuda\memory.pyx", line 775, in cupy.cuda.memory._try_malloc
cupy.cuda.memory.OutOfMemoryError: Out of memory allocating 255,688,704 bytes (allocated so far: 3,670,475,776 bytes).

In nets.py there are 2 functions : make_attention_mask and make_history_mask
can you explain about that ? I can't understand it.

one is in https://github.com/soskek/attention_is_all_you_need/blob/master/net.py#L345,
and another in https://github.com/soskek/attention_is_all_you_need/blob/master/net.py#L346-L350,
by the way, is that the calculate of sin&cos position embeded in self.initialize_position_encoding ( return the "all position encoding array"), am I correct?