@inproceedings{d02d98deabe24cbbadf21bb2ce5742e5,
title = "Position Information Emerges in Causal Transformers Without Positional Encodings via Similarity of Nearby Embeddings",
abstract = "Transformers with causal attention can solve tasks that require positional information without using positional encodings. In this work, we propose and investigate a new hypothesis about how positional information can be stored without using explicit positional encoding. We observe that nearby embeddings are more similar to each other than faraway embeddings, allowing the transformer to potentially reconstruct the positions of tokens. We show that this pattern can occur in both the trained and the randomly initialized Transformer models with causal attention and no positional encodings over a common range of hyperparameters.",
author = "Chunsheng Zuo and Pavel Guerzhoy and Michael Guerzhoy",
note = "Publisher Copyright: {\textcopyright} 2025 Association for Computational Linguistics.; 31st International Conference on Computational Linguistics, COLING 2025 ; Conference date: 19-01-2025 Through 24-01-2025",
year = "2025",
language = "English (US)",
series = "Proceedings - International Conference on Computational Linguistics, COLING",
publisher = "Association for Computational Linguistics (ACL)",
pages = "9418--9430",
editor = "Owen Rambow and Leo Wanner and Marianna Apidianaki and Hend Al-Khalifa and {Di Eugenio}, Barbara and Steven Schockaert",
booktitle = "Main Conference",
}