@misc{10481/104484,
year = {2025},
month = {3},
url = {https://hdl.handle.net/10481/104484},
abstract = {The popularity of Artificial Intelligence (AI) has risen sharply in recent years, revolutionizing applications in
most sectors with unprecedented functionalities. Milestones and achievements like ChatGPT demonstrate not
only the impressive capabilities of AI, but also how accessible such technologies have become in recent times.
However, the success of AI applications depends heavily on the underlying information integration processes.
Among the most important processes are the training of the AI model at the core of the application and
the collection and pre-processing of training data. In particular, the task of collecting high-quality training
data can be very costly and resource-intensive, as in many cases large amounts of data have to be annotated
manually. Human annotators must have extensive expertise for certain tasks in order to provide high-quality
training data. In this paper, we present a framework to maximize the efficiency of human experts in a Machine
Learning (ML) scenario, with the aim of optimizing the use of human expertise in active learning. This is done
by constantly measuring the quality of human experts’ input, as well as by involving human annotators only
when needed. We showcase the benefits of our proposed framework by applying it to a problem in image
classification, proving its usefulness to reduce the cost of annotating training data. The source code of the
framework is publicly available at https://github.com/human-centered-ai-lab/app-HITL-annotator.},
organization = {Austrian Science Fund
(FWF), Project: P-32554},
organization = {European Union’s Horizon
2020 research and innovation programme under grant agreement No.
826078 (Feature Cloud)},
organization = {European Union’s Horizon
2020 research and innovation programme under grant agreement No. 101079183 (BioMedAI TWINNING)},
organization = {Knowledge Generation
Project PID2023-149128NB-I00, funded by the Ministry of Science,
Innovation and Universities of Spain (Proyectos de generación de
conocimiento n◦ PID2023-149128NB-I00)},
publisher = {Elsevier},
keywords = {Human-Centric},
keywords = {AI Human-in-the-Loop},
keywords = {Human–machine interaction},
title = {On the disagreement problem in Human-in-the-Loop federated machine learning},
doi = {10.1016/j.jii.2025.100827},
author = {Huelser, Matthias and Mueller, Heimo and Díaz Rodríguez, Natalia Ana and Holzinger, Andreas},
}