BibTeX
– Copied!
– Copy failed!
@inProceedings{modica-etal-2026-factual-364265,
title = {Do Factual Recall Mechanisms Carry over from Text to Speech in Multimodal Language Models?},
abstract = {In recent years, several Speech Language Models (SLMs) that represent speech and written text jointly have been presented. The question then emerges about how model-internal mechanisms are similar and different when operating in the two modalities. We focus on how these systems encode, store, and retrieve factual knowledge, which has previously been investigated for text-only models. To investigate mechanisms behind the storage and recall of factual association in SLMs, we leverage Causal Mediation Analysis, a technique previously applied to text-based models. Initial results using SpiritLM, a multimodal model integrating discrete speech tokens reveal discrepancies between text-to-text and speech-to-text results, suggesting that the emergent mechanisms for factual recall are only partially carried over from the text to the speech modality. These results advance our understanding of how internal mechanisms encode factual associations in SLMs while contributing insights for improving speech-enabled AI systems.},
booktitle = {Proceedings of the 15th Joint Conference on Lexical and Computational Semantics (*SEM 2026), San Diego, California, United States},
author = {Modica, Luca and Landin, Filip and Farahani, Mehrdad and Qian, Livia and Skantze, Gabriel and Johansson, Richard},
year = {2026},
publisher = {Association for Computational Linguistics},
ISBN = {979-8-89176-413-2},
pages = {401--409},
}
@inProceedings{hagstrom-etal-2026-benchmarking-364264,
title = {CUB: Benchmarking Context Utilisation Techniques for Language Models},
abstract = {Incorporating external knowledge is crucial for knowledge-intensive tasks, such as question answering and fact checking. However, language models (LMs) may ignore relevant information that contradicts outdated parametric memory or be distracted by irrelevant contexts. While many context utilisation manipulation techniques (CMTs) have recently been proposed to alleviate these issues, few have seen systematic comparison. In this paper, we develop CUB (Context Utilisation Benchmark) - the first comprehensive benchmark designed to help diagnose CMTs under diverse noisy context conditions within retrieval-augmented generation (RAG). With this benchmark, we conduct the most extensive evaluation to date of seven state-of-the-art methods, representative of the main categories of CMTs, across three diverse datasets and tasks, applied to 11 LMs. Our findings expose critical gaps in current CMT evaluation practices, demonstrating the need for holistic testing. We reveal that most existing CMTs struggle to handle the full spectrum of context types encountered in real-world RAG scenarios. We also find that many CMTs display inflated performance on simple synthesised datasets, compared to more realistic datasets with naturally occurring samples.},
booktitle = {Proceedings of the 64th Annual Meeting of the Association for Computational Linguistics (Volume 1: Long Papers), San Diego, California, United States},
author = {Hagström, Lovisa and Kim, Youna and Yu, Haeun and Lee, Sang-goo and Johansson, Richard and Cho, Hyunsoo and Augenstein, Isabelle},
year = {2026},
publisher = {Association for Computational Linguistics},
ISBN = {979-8-89176-390-6},
pages = {25101--25133},
}