Should I interleave sin and cosine in sinusoidal positional encoding?

I'm trying to implement a sinusoidal positional encoding. I found two solutions that give different encodings. I am wondering if one of them is wrong or both are correct. I showcase visual figures of the resulting encodings for both options. Thank you :) class SinusoidalPosEmb(nn.Module): def __init__(self, dim): super().__init__() self.dim = dim def forward(self, x): device = x.device half_dim = self.dim // 2 emb = math.log(10000) / (half_dim - 1) emb = torch.exp(torch.arange(half_dim, device=device) * -emb) emb = x[:, None] * emb[None, :] emb = torch.cat((emb.sin(), emb.cos()), dim=-1) return emb 2) class TransformerPositionalEmbedding(nn.Module): """ From paper "Attention Is All You Need", section 3.5 """ def __init__(self, dimension, max_timesteps=1000): super(TransformerPositionalEmbedding, self).__init__() assert dimension % 2 == 0, "Embedding dimension must be even" self.dimension = dimension self.pe_matrix = torch.zeros(max_timesteps, dimension) # Gather all the even dimensions across the embedding vector even_indices = torch.arange(0, self.dimension, 2) # Calculate the term using log transforms for faster calculations # (https://stackoverflow.com/questions/17891595/pow-vs-exp-performance) log_term = torch.log(torch.tensor(10000.0)) / self.dimension div_term = torch.exp(even_indices * -log_term) # Precompute positional encoding matrix based on odd/even timesteps timesteps = torch.arange(max_timesteps).unsqueeze(1) self.pe_matrix[:, 0::2] = torch.sin(timesteps * div_t…