@inproceedings{dilip2025_111565,
  title = {A Three-Level Evaluation Protocoll for Acoustic Scene Understanding of Large Language Audio Models},
  author = {Dilip, Harish and Abeßer, Jakob},
  year = {2025},
  booktitle = {Proceedings of the 10th Workshop on Detection and Classification of Acoustic Scenes and Events (DCASE 2025), October 2025},
  pages = {185-189},
  publisher = {zenodo},
  isbn = {978-84-09-77652-8},
  doi = {10.5281/zenodo.17251589},
  url = {https://fis.uni-bamberg.de/handle/uniba/111565},
  language = {eng},
  abstract = {Reaching a semantic understanding of complex acoustic scenes requires computational models to capture the temporal-spatial sound source composition as well as individual sound events. This is a great challenge for computational models due to the large variety of everyday sound events and the extensive temporal-spectral overlap in real-life acoustic scenes. In this work, we aim to evaluate the acoustic scene understanding capabilities of two large audio-language models (LALMs). As a challenging scenario, we use the USM dataset, which features synthetic urban soundscapes with 2-6 overlapping sound sources per mixture. Our main contribution is a novel three-layer evaluation protocol, which includes four analysis tasks for low-level sound event understanding (sound event tagging), mid-level understanding and reasoning (sound polyphony estimation, sound source loudness ranking), as well as high-level scene understanding (audio captioning). We apply standardized metrics to assess the model's performances for each task. The proposed mulit-layer protocol allows for a fine-grained analysis of model behavior across soundscapes of various complexity levels. Our results indicate that despite their rmarkable controllability using textual instructions, the ability of state-of-the-art LALMs to understand acoustic scenes is still limited as the performance on individual analysis tasks desgrades with increasing sound polyphony.},
  keywords = {audio captioning, large audio-language models, sound event tagging, sound polyphony estimation, sound source loudness ranking}
}
