@misc{17026,
  author = {Cise Midoglu and Andrea Stor{\r a}s and Saeed Sabet and Malek Hammou and Steven Hicks and Inga Str{\"u}mke and Michael Riegler and Carsten Griwodz and P{\r a}l Halvorsen},
  title = {Experiences and Lessons Learned from a Crowdsourced-Remote Hybrid User Survey Framework},
  abstract = {Subjective user studies are important to ensure the fidelity and usability of systems that generate multimedia content. Testing how end-users and domain experts perceive multimedia assets might provide crucial information. In this paper, we present our experiences with the open source hybrid crowdsourced-remote user survey framework called Huldra, which is intended for conducting web-based subjective user studies and aims to integrate the individual benefits associated with traditional, crowdsourced, and remote methods. We disseminate our experiences and insights from two actively deployed use cases and discuss challenges and opportunities associated with using Huldra as a framework for conducting user studies.},
  year = {2022},
  journal = {2022 IEEE International Symposium on Multimedia (ISM)},
  pages = {161-162},
  publisher = {IEEE},
  address = {Italy},
  url = {https://ieeexplore.ieee.org/document/10019678},
  doi = {10.1109/ISM55400.2022.00035},
}