AI/ML News & Innovations Hub

@article{li-fowlie-2025-gpt,
    title = "{GPT} makes a poor {AMR} parser",
    author = "Li, Yanming  and
      Fowlie, Meaghan",
    editor = "Tu, Ngoc Duyen Tanja  and
      Brunner, Annelen  and
      Lang, Christian",
    journal = "Journal for Language Technology and Computational Linguistics",
    volume = "38",
    number = "2",
    month = jul,
    year = "2025",
    publisher = "German Society for Computational Linguistics and Language Technology",
    url = "https://aclanthology.org/2025.jlcl-2.4/",
    doi = "10.21248/jlcl.38.2025.285",
    pages = "43--76",
    abstract = "This paper evaluates GPT models as out-of-the-box Abstract Meaning Representation (AMR) parsers using prompt-based strategies, including 0-shot, few-shot, Chain-of-Thought (CoT), and a two-step approach in which core arguments and non-core roles are handled separately. Our results show that GPT-3.5 and GPT-4o fall well short of state-of-the-art parsers, with a maximum Smatch score of 60 using GPT-4o in a 5-shot setting. While CoT prompting provides some interpretability, it does not improve performance. We further conduct fine-grained evaluations, revealing GPT{'}s limited ability to handle AMR-specific linguistic structures and complex semantic roles. Our findings suggest that, despite recent advances, GPT models are not yet suitable as standalone AMR parsers."
}

<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="li-fowlie-2025-gpt">
    <titleInfo>
        <title>GPT makes a poor AMR parser</title>
    </titleInfo>
    <name type="personal">
        <namePart type="given">Yanming</namePart>
        <namePart type="family">Li</namePart>
        <role>
            <roleTerm authority="marcrelator" type="text">author</roleTerm>
        </role>
    </name>
    <name type="personal">
        <namePart type="given">Meaghan</namePart>
        <namePart type="family">Fowlie</namePart>
        <role>
            <roleTerm authority="marcrelator" type="text">author</roleTerm>
        </role>
    </name>
    <originInfo>
        <dateIssued>2025-07</dateIssued>
    </originInfo>
    <typeOfResource>text</typeOfResource>
    <genre authority="bibutilsgt">journal article</genre>
    <relatedItem type="host">
        <titleInfo>
            <title>Journal for Language Technology and Computational Linguistics</title>
        </titleInfo>
        <originInfo>
            <issuance>continuing</issuance>
            <publisher>German Society for Computational Linguistics and Language Technology</publisher>
        </originInfo>
        <genre authority="marcgt">periodical</genre>
        <genre authority="bibutilsgt">academic journal</genre>
    </relatedItem>
    <abstract>This paper evaluates GPT models as out-of-the-box Abstract Meaning Representation (AMR) parsers using prompt-based strategies, including 0-shot, few-shot, Chain-of-Thought (CoT), and a two-step approach in which core arguments and non-core roles are handled separately. Our results show that GPT-3.5 and GPT-4o fall well short of state-of-the-art parsers, with a maximum Smatch score of 60 using GPT-4o in a 5-shot setting. While CoT prompting provides some interpretability, it does not improve performance. We further conduct fine-grained evaluations, revealing GPT’s limited ability to handle AMR-specific linguistic structures and complex semantic roles. Our findings suggest that, despite recent advances, GPT models are not yet suitable as standalone AMR parsers.</abstract>
    <identifier type="citekey">li-fowlie-2025-gpt</identifier>
    <identifier type="doi">10.21248/jlcl.38.2025.285</identifier>
    <location>
        <url>https://aclanthology.org/2025.jlcl-2.4/</url>
    </location>
    <part>
        <date>2025-07</date>
        <detail type="volume"><number>38</number></detail>
        <detail type="issue"><number>2</number></detail>
        <extent unit="page">
            <start>43</start>
            <end>76</end>
        </extent>
    </part>
</mods>
</modsCollection>

%0 Journal Article
%T GPT makes a poor AMR parser
%A Li, Yanming
%A Fowlie, Meaghan
%J Journal for Language Technology and Computational Linguistics
%D 2025
%8 July
%V 38
%N 2
%I German Society for Computational Linguistics and Language Technology
%F li-fowlie-2025-gpt
%X This paper evaluates GPT models as out-of-the-box Abstract Meaning Representation (AMR) parsers using prompt-based strategies, including 0-shot, few-shot, Chain-of-Thought (CoT), and a two-step approach in which core arguments and non-core roles are handled separately. Our results show that GPT-3.5 and GPT-4o fall well short of state-of-the-art parsers, with a maximum Smatch score of 60 using GPT-4o in a 5-shot setting. While CoT prompting provides some interpretability, it does not improve performance. We further conduct fine-grained evaluations, revealing GPT’s limited ability to handle AMR-specific linguistic structures and complex semantic roles. Our findings suggest that, despite recent advances, GPT models are not yet suitable as standalone AMR parsers.
%R 10.21248/jlcl.38.2025.285
%U https://aclanthology.org/2025.jlcl-2.4/
%U https://doi.org/10.21248/jlcl.38.2025.285
%P 43-76

Markdown (Informal)

[GPT makes a poor AMR parser](https://aclanthology.org/2025.jlcl-2.4/) (Li & Fowlie, JLCL 2025)

ACL

Yanming Li and Meaghan Fowlie. 2025. GPT makes a poor AMR parser. Journal for Language Technology and Computational Linguistics, 38(2):43–76.