@inproceedings{guerini-etal-2018-methodology,
title = "A Methodology for Evaluating Interaction Strategies of Task-Oriented Conversational Agents",
author = "Guerini, Marco and
Falcone, Sara and
Magnini, Bernardo",
editor = "Chuklin, Aleksandr and
Dalton, Jeff and
Kiseleva, Julia and
Borisov, Alexey and
Burtsev, Mikhail",
booktitle = "Proceedings of the 2018 {EMNLP} Workshop {SCAI}: The 2nd International Workshop on Search-Oriented Conversational {AI}",
month = oct,
year = "2018",
address = "Brussels, Belgium",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/W18-5704",
doi = "10.18653/v1/W18-5704",
pages = "24--32",
abstract = "In task-oriented conversational agents, more attention has been usually devoted to assessing task effectiveness, rather than to \textit{how} the task is achieved. However, conversational agents are moving towards more complex and human-like interaction capabilities (e.g. the ability to use a formal/informal register, to show an empathetic behavior), for which standard evaluation methodologies may not suffice. In this paper, we provide a novel methodology to assess - in a completely controlled way - the impact on the quality of experience of agent{'}s interaction strategies. The methodology is based on a within subject design, where two slightly different transcripts of the same interaction with a conversational agent are presented to the user. Through a series of pilot experiments we prove that this methodology allows fast and cheap experimentation/evaluation, focusing on aspects that are overlooked by current methods.",
}
<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="guerini-etal-2018-methodology">
<titleInfo>
<title>A Methodology for Evaluating Interaction Strategies of Task-Oriented Conversational Agents</title>
</titleInfo>
<name type="personal">
<namePart type="given">Marco</namePart>
<namePart type="family">Guerini</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Sara</namePart>
<namePart type="family">Falcone</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Bernardo</namePart>
<namePart type="family">Magnini</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2018-10</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Proceedings of the 2018 EMNLP Workshop SCAI: The 2nd International Workshop on Search-Oriented Conversational AI</title>
</titleInfo>
<name type="personal">
<namePart type="given">Aleksandr</namePart>
<namePart type="family">Chuklin</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jeff</namePart>
<namePart type="family">Dalton</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Julia</namePart>
<namePart type="family">Kiseleva</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Alexey</namePart>
<namePart type="family">Borisov</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Mikhail</namePart>
<namePart type="family">Burtsev</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Brussels, Belgium</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
</relatedItem>
<abstract>In task-oriented conversational agents, more attention has been usually devoted to assessing task effectiveness, rather than to how the task is achieved. However, conversational agents are moving towards more complex and human-like interaction capabilities (e.g. the ability to use a formal/informal register, to show an empathetic behavior), for which standard evaluation methodologies may not suffice. In this paper, we provide a novel methodology to assess - in a completely controlled way - the impact on the quality of experience of agent’s interaction strategies. The methodology is based on a within subject design, where two slightly different transcripts of the same interaction with a conversational agent are presented to the user. Through a series of pilot experiments we prove that this methodology allows fast and cheap experimentation/evaluation, focusing on aspects that are overlooked by current methods.</abstract>
<identifier type="citekey">guerini-etal-2018-methodology</identifier>
<identifier type="doi">10.18653/v1/W18-5704</identifier>
<location>
<url>https://aclanthology.org/W18-5704</url>
</location>
<part>
<date>2018-10</date>
<extent unit="page">
<start>24</start>
<end>32</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T A Methodology for Evaluating Interaction Strategies of Task-Oriented Conversational Agents
%A Guerini, Marco
%A Falcone, Sara
%A Magnini, Bernardo
%Y Chuklin, Aleksandr
%Y Dalton, Jeff
%Y Kiseleva, Julia
%Y Borisov, Alexey
%Y Burtsev, Mikhail
%S Proceedings of the 2018 EMNLP Workshop SCAI: The 2nd International Workshop on Search-Oriented Conversational AI
%D 2018
%8 October
%I Association for Computational Linguistics
%C Brussels, Belgium
%F guerini-etal-2018-methodology
%X In task-oriented conversational agents, more attention has been usually devoted to assessing task effectiveness, rather than to how the task is achieved. However, conversational agents are moving towards more complex and human-like interaction capabilities (e.g. the ability to use a formal/informal register, to show an empathetic behavior), for which standard evaluation methodologies may not suffice. In this paper, we provide a novel methodology to assess - in a completely controlled way - the impact on the quality of experience of agent’s interaction strategies. The methodology is based on a within subject design, where two slightly different transcripts of the same interaction with a conversational agent are presented to the user. Through a series of pilot experiments we prove that this methodology allows fast and cheap experimentation/evaluation, focusing on aspects that are overlooked by current methods.
%R 10.18653/v1/W18-5704
%U https://aclanthology.org/W18-5704
%U https://doi.org/10.18653/v1/W18-5704
%P 24-32
Markdown (Informal)
[A Methodology for Evaluating Interaction Strategies of Task-Oriented Conversational Agents](https://aclanthology.org/W18-5704) (Guerini et al., EMNLP 2018)
ACL