Source code for espnet.nets.chainer_backend.transformer.embedding

# encoding: utf-8
"""Class Declaration of Transformer's Positional Encoding."""

import chainer
import chainer.functions as F
import numpy as np

[docs]class PositionalEncoding(chainer.Chain): """Positional encoding module. :param int n_units: embedding dim :param float dropout: dropout rate :param int length: maximum input length """ def __init__(self, n_units, dropout=0.1, length=5000): """Initialize Positional Encoding.""" # Implementation described in the paper super(PositionalEncoding, self).__init__() self.dropout = dropout posi_block = np.arange(0, length, dtype=np.float32)[:, None] unit_block = np.exp( np.arange(0, n_units, 2, dtype=np.float32) * -(np.log(10000.0) / n_units) ) = np.zeros((length, n_units), dtype=np.float32)[:, ::2] = np.sin(posi_block * unit_block)[:, 1::2] = np.cos(posi_block * unit_block) self.scale = np.sqrt(n_units)
[docs] def forward(self, e): """Forward Positional Encoding.""" length = e.shape[1] e = e * self.scale + self.xp.array([:length]) return F.dropout(e, self.dropout)