{
 "uuid": "de6038eb-6f14-4b97-b2b8-5c30c2d118d5",
 "handle": "uniba/112676",
 "type": "Publication",
 "name": "Are Humans as Brittle as Large Language Models?",
 "url": "/entities/publication/de6038eb-6f14-4b97-b2b8-5c30c2d118d5/",
 "lastModified": "2026-02-14T03:08:10.630+00:00",
 "metadata": {
  "cris.virtual.author-orcid": [
   {
    "value": null
   },
   {
    "value": null
   },
   {
    "value": "0000-0002-2014-6619"
   }
  ],
  "cris.virtualsource.author-orcid": [
   {
    "value": "810936d6-ad00-4547-8555-8904da457042"
   },
   {
    "value": "d03c938d-61d6-469e-99b1-0afd7186bfaf"
   },
   {
    "value": "a2a36d65-e914-48fb-8e87-a2796ce66cb3"
   }
  ],
  "dc.contributor.author": [
   {
    "value": "Li, Jiahui",
    "authority": "810936d6-ad00-4547-8555-8904da457042"
   },
   {
    "value": "Papay, Sean",
    "authority": "d03c938d-61d6-469e-99b1-0afd7186bfaf"
   },
   {
    "value": "Klinger, Roman",
    "authority": "a2a36d65-e914-48fb-8e87-a2796ce66cb3"
   }
  ],
  "dc.date.accessioned": [
   {
    "value": "2026-01-21T15:08:13Z"
   }
  ],
  "dc.date.available": [
   {
    "value": "2026-01-21T15:08:13Z"
   }
  ],
  "dc.date.issued": [
   {
    "value": "2025"
   }
  ],
  "dc.description.abstract": [
   {
    "value": "The output of large language models (LLMs) is unstable, due both to non-determinism of the decoding process as well as to prompt brittleness. While the intrinsic non-determinism of LLM generation may mimic existing uncertainty in human annotations through distributional shifts in outputs, it is largely assumed, yet unexplored, that the prompt brittleness effect is unique to LLMs. This raises the question: do human annotators show similar sensitivity to prompt changes? If so, should prompt brittleness in LLMs be considered problematic? One may alternatively hypothesize that prompt brittleness correctly reflects human annotation variances. To fill this research gap, we systematically compare the effects of prompt modifications on LLMs and identical instruction modifications for human annotators, focusing on the question of whether humans are similarly sensitive to prompt perturbations. To study this, we prompt both humans and LLMs for a set of text classification tasks conditioned on prompt variations. Our findings indicate that both humans and LLMs exhibit increased brittleness in response to specific types of prompt modifications, particularly those involving the substitution of alternative label sets or label formats. However, the distribution of human judgments is less affected by typographical errors and reversed label order than that of LLMs."
   }
  ],
  "dc.identifier.isbn": [
   {
    "value": "979-8-89176-298-5"
   }
  ],
  "dc.identifier.uri": [
   {
    "value": "https://fis.uni-bamberg.de/handle/uniba/112676"
   }
  ],
  "dc.identifier.url": [
   {
    "value": "https://aclanthology.org/2025.ijcnlp-long.116/"
   }
  ],
  "dc.language.iso": [
   {
    "value": "eng"
   }
  ],
  "dc.publisher": [
   {
    "value": "The Asian Federation of Natural Language Processing and The Association for Computational Linguistics"
   }
  ],
  "dc.publisher.place": [
   {
    "value": "Mumbai, India"
   }
  ],
  "dc.relation.authororgunit": [
   {
    "value": null,
    "authority": "#PLACEHOLDER_PARENT_METADATA_VALUE#"
   },
   {
    "value": "Lehrstuhl für Grundlagen der Sprachverarbeitung",
    "authority": "e42c6d56-e293-45c9-9582-a71b5e4aee22"
   },
   {
    "value": "Lehrstuhl für Grundlagen der Sprachverarbeitung",
    "authority": "e42c6d56-e293-45c9-9582-a71b5e4aee22"
   }
  ],
  "dc.subject": [
   {
    "value": "Large Language Model"
   }
  ],
  "dc.title": [
   {
    "value": "Are Humans as Brittle as Large Language Models?"
   }
  ],
  "dc.type": [
   {
    "value": "conferenceobject"
   }
  ],
  "dspace.entity.type": [
   {
    "value": "Publication"
   }
  ],
  "ubg.conference.name": [
   {
    "value": "14th International Joint Conference on Natural Language Processing and the 4th Conference of the Asia-Pacific Chapter of the Association for Computational Linguistics, December 2025"
   }
  ],
  "ubg.conference.place": [
   {
    "value": "Mumbai, India"
   }
  ],
  "ubg.editor.articlecollection": [
   {
    "value": "Inui, Kentaro"
   },
   {
    "value": "Sakti, Sakriani",
    "authority": "65a7a669-e49c-4a10-9134-c6b7cfa544d0"
   },
   {
    "value": "Wang, Haofen"
   },
   {
    "value": "Wong, Derek F."
   },
   {
    "value": "Bhattacharyya, Pushpak"
   },
   {
    "value": "Banerjee, Biplab"
   },
   {
    "value": "Ekbal, Asif"
   },
   {
    "value": "Chakraborty, Tanmoy"
   },
   {
    "value": "Singh, Dhirendra Pratap"
   }
  ],
  "ubg.fulltexticon": [
   {
    "value": "true"
   }
  ],
  "ubg.generated.csl": [
   {
    "value": "<div class=\"csl-bib-body\">\n  <div class=\"csl-entry\">Li, Jiahui; Papay, Sean; Klinger, Roman (2025): Are Humans as Brittle as Large Language Models?, in: Kentaro Inui, Sakriani Sakti, Haofen Wang, u. a. (Hrsg.), <i>Proceedings of the 14th International Joint Conference on Natural Language Processing and the 4th Conference of the Asia-Pacific Chapter of the Association for Computational Linguistics</i>, Mumbai, India: The Asian Federation of Natural Language Processing and The Association for Computational Linguistics, S. 2130–2155.</div>\n</div>"
   }
  ],
  "ubg.generated.cslContainerTitle": [
   {
    "value": "Proceedings of the 14th International Joint Conference on Natural Language Processing and the 4th Conference of the Asia-Pacific Chapter of the Association for Computational Linguistics"
   }
  ],
  "ubg.generated.cslEditor": [
   {
    "value": "Inui, Kentaro",
    "authority": "ad4d393c-4541-485f-991c-bb05db92831b"
   },
   {
    "value": "Sakti, Sakriani",
    "authority": "65a7a669-e49c-4a10-9134-c6b7cfa544d0"
   },
   {
    "value": "Wang, Haofen",
    "authority": "f9ab53b8-1e6c-4933-b4dc-79ee9ef5fcf9"
   },
   {
    "value": "Wong, Derek F.",
    "authority": "9cf55cb2-79be-4ccc-81fe-498daa08b6f4"
   },
   {
    "value": "Bhattacharyya, Pushpak",
    "authority": "9401bd84-3b82-4791-aac0-feeefa6cda95"
   },
   {
    "value": "Banerjee, Biplab",
    "authority": "e913362c-937c-4047-9786-d869091fcb26"
   },
   {
    "value": "Ekbal, Asif",
    "authority": "d79f0d37-8f94-4870-8e21-2369e3ef1f74"
   },
   {
    "value": "Chakraborty, Tanmoy",
    "authority": "8c5d4cf4-149d-4e88-a50c-8f511d5e75c7"
   },
   {
    "value": "Singh, Dhirendra Pratap",
    "authority": "64d1ce0f-ed32-4d8b-91a7-450f4fcf4358"
   },
   {
    "value": "Inui, Kentaro"
   },
   {
    "value": "Wang, Haofen"
   },
   {
    "value": "Wong, Derek F."
   },
   {
    "value": "Bhattacharyya, Pushpak"
   },
   {
    "value": "Banerjee, Biplab"
   },
   {
    "value": "Ekbal, Asif"
   },
   {
    "value": "Chakraborty, Tanmoy"
   },
   {
    "value": "Singh, Dhirendra Pratap"
   }
  ],
  "ubg.intern.org": [
   {
    "value": "Lehrstuhl für Grundlagen der Sprachverarbeitung",
    "authority": "e42c6d56-e293-45c9-9582-a71b5e4aee22"
   }
  ],
  "ubg.intern.vonfeld": [
   {
    "value": "",
    "language": "de_DE"
   }
  ],
  "ubg.intern.vonfeldeditor": [
   {
    "value": "",
    "language": "de_DE"
   }
  ],
  "ubg.internatorientation": [
   {
    "value": "Ja"
   }
  ],
  "ubg.openaccess": [
   {
    "value": "Ja"
   }
  ],
  "ubg.pages.range": [
   {
    "value": "2130–2155"
   }
  ],
  "ubg.peerreview": [
   {
    "value": "Ja"
   }
  ],
  "ubg.relation.project": [
   {
    "value": "Interactive Prompt Optimization with the Human in the Loop for Natural Language Understanding Model Development and Intervention",
    "authority": "098980c8-a33f-4560-8f62-0ebe368d248f"
   }
  ],
  "ubg.submission.type": [
   {
    "value": "full-text"
   }
  ],
  "ubg.titleparent.articlecollection": [
   {
    "value": "Proceedings of the 14th International Joint Conference on Natural Language Processing and the 4th Conference of the Asia-Pacific Chapter of the Association for Computational Linguistics"
   }
  ],
  "ubg.version.hasversions": [
   {
    "value": "uniba/113162",
    "authority": "55a30924-ddd2-4824-ad6e-d2fbd09f5c2b"
   }
  ]
 },
 "relations": {},
 "files": []
}