<?xml version="1.0" encoding="UTF-8"?><!DOCTYPE article PUBLIC "-//NLM//DTD JATS (Z39.96) Journal Publishing DTD v1.2 20190208//EN" "http://jats.nlm.nih.gov/publishing/1.2/JATS-journalpublishing1.dtd"><article xmlns:mml="http://www.w3.org/1998/Math/MathML" xmlns:xlink="http://www.w3.org/1999/xlink" article-type="methods-article" dtd-version="1.2" xml:lang="en">
    <front>
        <journal-meta>
            <journal-id journal-id-type="pmc">Gates Open Res</journal-id>
            <journal-title-group>
                <journal-title>Gates Open Research</journal-title>
            </journal-title-group>
            <issn pub-type="epub">2572-4754</issn>
            <publisher>
                <publisher-name>F1000 Research Limited</publisher-name>
                <publisher-loc>London, UK</publisher-loc>
            </publisher>
        </journal-meta>
        <article-meta>
            <article-id pub-id-type="doi">10.12688/gatesopenres.15376.1</article-id>
            <article-categories>
                <subj-group subj-group-type="heading">
                    <subject>Method Article</subject>
                </subj-group>
                <subj-group>
                    <subject>Articles</subject>
                </subj-group>
            </article-categories>
            <title-group>
                <article-title>Supporting contraceptive self-care and reproductive empowerment with a digital health game in Barbados: Development and Pre-implementation study for What&#x2019;s My Method?</article-title>
                <fn-group content-type="pub-status">
                    <fn>
                        <p>[version 1; peer review: 2 approved with reservations, 1 not approved]</p>
                    </fn>
                </fn-group>
            </title-group>
            <contrib-group>
                <contrib contrib-type="author" corresp="yes">
                    <name>
                        <surname>Bertozzi</surname>
                        <given-names>Elena</given-names>
                    </name>
                    <role content-type="http://credit.niso.org/">Conceptualization</role>
                    <role content-type="http://credit.niso.org/">Data Curation</role>
                    <role content-type="http://credit.niso.org/">Formal Analysis</role>
                    <role content-type="http://credit.niso.org/">Investigation</role>
                    <role content-type="http://credit.niso.org/">Methodology</role>
                    <role content-type="http://credit.niso.org/">Project Administration</role>
                    <role content-type="http://credit.niso.org/">Software</role>
                    <role content-type="http://credit.niso.org/">Supervision</role>
                    <role content-type="http://credit.niso.org/">Validation</role>
                    <role content-type="http://credit.niso.org/">Writing &#x2013; Original Draft Preparation</role>
                    <role content-type="http://credit.niso.org/">Writing &#x2013; Review &amp; Editing</role>
                    <uri content-type="orcid">https://orcid.org/0000-0002-5286-2978</uri>
                    <xref ref-type="corresp" rid="c1">a</xref>
                    <xref ref-type="aff" rid="a1">1</xref>
                </contrib>
                <contrib contrib-type="author" corresp="no">
                    <name>
                        <surname>Bertozzi-Villa</surname>
                        <given-names>Clara</given-names>
                    </name>
                    <role content-type="http://credit.niso.org/">Data Curation</role>
                    <role content-type="http://credit.niso.org/">Software</role>
                    <role content-type="http://credit.niso.org/">Writing &#x2013; Original Draft Preparation</role>
                    <role content-type="http://credit.niso.org/">Writing &#x2013; Review &amp; Editing</role>
                    <xref ref-type="aff" rid="a2">2</xref>
                </contrib>
                <contrib contrib-type="author" corresp="no">
                    <name>
                        <surname>Sabato</surname>
                        <given-names>Erin</given-names>
                    </name>
                    <role content-type="http://credit.niso.org/">Funding Acquisition</role>
                    <role content-type="http://credit.niso.org/">Resources</role>
                    <role content-type="http://credit.niso.org/">Writing &#x2013; Review &amp; Editing</role>
                    <uri content-type="orcid">https://orcid.org/0009-0007-3256-3990</uri>
                    <xref ref-type="aff" rid="a3">3</xref>
                </contrib>
                <contrib contrib-type="author" corresp="no">
                    <name>
                        <surname>Alleyne</surname>
                        <given-names>Nicole</given-names>
                    </name>
                    <role content-type="http://credit.niso.org/">Investigation</role>
                    <role content-type="http://credit.niso.org/">Project Administration</role>
                    <role content-type="http://credit.niso.org/">Supervision</role>
                    <role content-type="http://credit.niso.org/">Writing &#x2013; Review &amp; Editing</role>
                    <xref ref-type="aff" rid="a4">4</xref>
                </contrib>
                <contrib contrib-type="author" corresp="no">
                    <name>
                        <surname>Watson-Miller</surname>
                        <given-names>Sonia</given-names>
                    </name>
                    <role content-type="http://credit.niso.org/">Resources</role>
                    <role content-type="http://credit.niso.org/">Supervision</role>
                    <role content-type="http://credit.niso.org/">Writing &#x2013; Review &amp; Editing</role>
                    <xref ref-type="aff" rid="a5">5</xref>
                </contrib>
                <contrib contrib-type="author" corresp="no">
                    <name>
                        <surname>Jordan</surname>
                        <given-names>Tiffany</given-names>
                    </name>
                    <role content-type="http://credit.niso.org/">Investigation</role>
                    <role content-type="http://credit.niso.org/">Resources</role>
                    <role content-type="http://credit.niso.org/">Writing &#x2013; Review &amp; Editing</role>
                    <xref ref-type="aff" rid="a6">6</xref>
                </contrib>
                <contrib contrib-type="author" corresp="no">
                    <name>
                        <surname>Langdon</surname>
                        <given-names>Anderson</given-names>
                    </name>
                    <role content-type="http://credit.niso.org/">Project Administration</role>
                    <role content-type="http://credit.niso.org/">Resources</role>
                    <role content-type="http://credit.niso.org/">Supervision</role>
                    <role content-type="http://credit.niso.org/">Writing &#x2013; Review &amp; Editing</role>
                    <xref ref-type="aff" rid="a6">6</xref>
                </contrib>
                <aff id="a1">
                    <label>1</label>Game Design &amp; Development, Quinnipiac University, Hamden, Connecticut, 06518, USA</aff>
                <aff id="a2">
                    <label>2</label>Division of Maternal Fetal Medicine, Department of Obstetrics &amp; Gynecology and Women&#x2019;s Health, Montefiore Health System, Bronx, New York, USA</aff>
                <aff id="a3">
                    <label>3</label>Office of Global Engagement, Quinnipiac University, Hamden, Connecticut, 06518, USA</aff>
                <aff id="a4">
                    <label>4</label>Hibiscus Health Caribbean, Bridgetown, Saint Michaels, Barbados</aff>
                <aff id="a5">
                    <label>5</label>Nursing Department, Barbados Community College, Saint Michael, Barbados</aff>
                <aff id="a6">
                    <label>6</label>Barbados Family Planning Association, Bridgetown, Saint Michaels, Barbados</aff>
            </contrib-group>
            <author-notes>
                <corresp id="c1">
                    <label>a</label>
                    <email xlink:href="mailto:elena.bertozzi@quinnipiac.edu">elena.bertozzi@quinnipiac.edu</email>
                </corresp>
                <fn fn-type="conflict">
                    <p>
                        <bold>Competing interests: </bold>Elena Bertozzi is Co-Founder of SolitonZ Games which designed and produced the What's My Method? game.</p>
                </fn>
            </author-notes>
            <pub-date pub-type="epub">
                <day>28</day>
                <month>5</month>
                <year>2024</year>
            </pub-date>
            <pub-date pub-type="collection">
                <year>2024</year>
            </pub-date>
            <volume>8</volume>
            <elocation-id>47</elocation-id>
            <history>
                <date date-type="accepted">
                    <day>25</day>
                    <month>4</month>
                    <year>2024</year>
                </date>
            </history>
            <permissions>
                <copyright-statement>Copyright: &#x00a9; 2024 Bertozzi E et al.</copyright-statement>
                <copyright-year>2024</copyright-year>
                <license xlink:href="https://creativecommons.org/licenses/by/4.0/">
                    <license-p>This is an open access article distributed under the terms of the Creative Commons Attribution Licence, which permits unrestricted use, distribution, and reproduction in any medium, provided the original work is properly cited.</license-p>
                </license>
            </permissions>
            <self-uri content-type="pdf" xlink:href="https://gatesopenresearch.org/articles/8-47/pdf"/>
            <abstract>
                <p>Effective contraceptive education is essential to reducing unwanted pregnancy, increasing uptake of modern contraceptive methods, and thoughtfully planning desired births. New WHO and family planning organization guidelines recommend situating contraceptive education and counseling within a broader context of self-care that emphasizes individual agency and reproductive empowerment. Digital health interventions, and games for health specifically, have been validated as effective and scalable tools for self-guided and interactive health education, especially among younger tech-savvy individuals. Barbados currently supplements provider-based contraceptive counseling with analog materials (pamphlets and posters) and informational videos that play on a screen in the waiting room. This study seeks to conduct a formative evaluation of the What&#x2019;s My Method? (WMM) game intervention as a tool to support contraceptive counseling and increase reproductive empowerment among childbearing persons in Barbados. We test-deployed the WMM game in Bridgetown, Barbados, conducting focus groups and sample deployments with prototypes of the WMM game among three groups of stakeholders (youth contraception ambassadors: n=8; healthcare providers: n=7; and nursing students: n=27) to determine acceptability of the intervention, efficacy of the game as a learning tool, and willingness to adopt the tool in their healthcare context. Feedback on acceptability of the game was largely positive. Constructive feedback informed modifications and improvements to the game. Assessment of contraceptive knowledge gain was complicated by several factors and resulted in modifications of the questionnaire and methodology for measurement of efficacy. Results indicate that the WMM game is well-reviewed and accepted by the target audience of childbearing people and healthcare professionals who would be deploying it. The questionnaire did not perform as intended for our target audience. This pilot testing has informed the design and outcome selection for the upcoming RCT with the final version of the game.</p>
            </abstract>
            <kwd-group kwd-group-type="author">
                <kwd>contraceptive self-efficacy</kwd>
                <kwd>videogame</kwd>
                <kwd>reproductive empowerment</kwd>
                <kwd>family planning method</kwd>
                <kwd>education</kwd>
            </kwd-group>
            <funding-group>
                <award-group id="fund-1">
                    <funding-source>Galpin Internationalization Grant</funding-source>
                </award-group>
                <funding-statement>This work was supported by the Gates Foundation [OPP1161938; Childbearing Intentions and Family Planning Game] and by a Galpin Internationalization grant from Quinnipiac University.</funding-statement>
            </funding-group>
        </article-meta>
    </front>
    <body>
        <sec sec-type="intro">
            <title>Introduction</title>
            <sec>
                <title>Background</title>
                <p>Contraceptive self-care has been identified by the WHO as critical to achieving milestones for female empowerment and well-being (
                    <xref ref-type="bibr" rid="ref-40">World Health Organization, 2022</xref>). This focus reflects a shift in global public health attitudes regarding contraceptive education from a top-down approach&#x2014;where childbearing persons receive directives, often from healthcare professionals and/or government institutions&#x2014;towards an approach that encourages self-determination and agency (
                    <xref ref-type="bibr" rid="ref-15">Hamidi 
                        <italic toggle="yes">et al</italic>., 2018</xref>). Reproductive empowerment is a means of implementing and interpreting contraceptive self-care and has been defined as:</p>
                <list list-type="bullet">
                    <list-item>
                        <label/>
                        <p>Both a transformative process and an outcome, whereby individuals expand their capacity to make informed decisions about their reproductive lives, amplify their ability to participate meaningfully in public and private discussions related to sexuality, reproductive health and fertility, and act on their preferences to achieve desired reproductive outcomes, free from violence, retribution, or fear. (
                            <xref ref-type="bibr" rid="ref-12">Edmeades 
                                <italic toggle="yes">et al</italic>., 2018</xref>)</p>
                    </list-item>
                </list>
                <p>Improving pregnancy planning and preventing unintended pregnancy remains a WHO priority, is one of the US Healthy People Objectives (
                    <xref ref-type="bibr" rid="ref-25">Office of Disease Prevention and Health Promotion, 2023</xref>), and is also a desired outcome in Barbados. Implementing contraceptive counseling and education through the lens of reproductive empowerment requires that the focus of the intervention be on the childbearing persons and their needs. Ideally, this counseling also reviews the many non-contraceptive benefits of hormonal birth control methods, including menstrual regulation and relief of symptoms associated with conditions such as endometriosis and polycystic ovarian syndrome (PCOS). Some studies have been done to measure the impact of contraceptive self-efficacy on increased adoption of contraception. However, most studies focused solely on condom use (
                    <xref ref-type="bibr" rid="ref-7">Burke 
                        <italic toggle="yes">et al</italic>., 2021</xref>; 
                    <xref ref-type="bibr" rid="ref-32">Whiting-Collins 
                        <italic toggle="yes">et al</italic>., 2020</xref>). These studies show that higher rates of self-efficacy relative to condoms result in increased condom use and recommend additional research measuring efficacy with other forms of contraception (
                    <xref ref-type="bibr" rid="ref-7">Burke 
                        <italic toggle="yes">et al</italic>., 2021</xref>).</p>
                <p>There is clear need for improvement in contraceptive counseling in Barbados. As per the United Nations dashboard, the contraceptive prevalence rate for any method among women aged 15&#x2013;49 was 50% in 2022 (
                    <xref ref-type="bibr" rid="ref-30">United Nations, 2023</xref>). Providers report that abortion is often used as birth control, and the fertility rate is declining (1.6 live births per woman in 2023). Childbearing women receive advice from older women propagating unsupported concerns about side effects which makes them reluctant to adopt modern methods. Low uptake of modern contraceptive methods leads to outcomes including unwanted pregnancy and use of abortion as birth control (
                    <xref ref-type="bibr" rid="ref-2">Bearak 
                        <italic toggle="yes">et al</italic>., 2022</xref>; 
                    <xref ref-type="bibr" rid="ref-9">Claridge, 2021</xref>; 
                    <xref ref-type="bibr" rid="ref-14">Gipson 
                        <italic toggle="yes">et al</italic>., 2008</xref>). </p>
                <p>Barbados includes contraceptive counseling (CC) as a standard part of post-natal and pre-abortion care. CC is an essential aspect of healthcare for women and child-bearing individuals. This counseling should be thorough, unbiased, and tailored to each patient&#x2019;s specific needs and health conditions. Providing CC is complicated by patient use of the internet which provides a plethora of information that can be difficult to filter and interpret as well as abundant misinformation which must be countered. The proliferation of misinformation is accelerated by emerging technology use such as influencers on TikTok and YouTube (
                    <xref ref-type="bibr" rid="ref-26">Pfender &amp; Devlin, 2023</xref>). Barbadian clinics have not integrated digital technology to support CC; information about methods is delivered using pamphlets and posters on the walls.</p>
                <p>Digital health interventions have been identified as successful high-impact practices to support healthy reproductive behaviors (
                    <xref ref-type="bibr" rid="ref-10">Dehlendorf 
                        <italic toggle="yes">et al</italic>., 2019</xref>; 
                    <xref ref-type="bibr" rid="ref-29">Stephenson 
                        <italic toggle="yes">et al</italic>., 2020</xref>). Digital health interventions supporting contraceptive self-efficacy include SMS campaigns (
                    <xref ref-type="bibr" rid="ref-8">Chukwu 
                        <italic toggle="yes">et al</italic>., 2021</xref>; 
                    <xref ref-type="bibr" rid="ref-20">Laidlaw 
                        <italic toggle="yes">et al</italic>., 2017</xref>), artificial intelligence-based chatbots (
                    <xref ref-type="bibr" rid="ref-31">Wang 
                        <italic toggle="yes">et al</italic>., 2022</xref>), and interactive websites (
                    <xref ref-type="bibr" rid="ref-21">Lepore 
                        <italic toggle="yes">et al</italic>., 2024</xref>) which provide information and offer tools to help choose appropriate methods. These interventions demonstrate that digital media are an effective way to reach the target audience and communicate information about reproductive health.</p>
                <p>Demonstrating the impact of interventions on reproductive empowerment is complicated.</p>
                <p>There are many different forms of contraception, which vary among mechanism of action, route of administration, frequency of dosing, side effects, and many other factors. Understanding and remembering these nuances is difficult even for healthcare providers. Videogames can be useful for this type of learning. Games are a validated means of presenting complex information in a format that provides context-based learning, encourages engagement through fun, and rewards success (
                    <xref ref-type="bibr" rid="ref-28">Squire, 2011</xref>). People are more likely to seek out information and remember it if it is contextualized and relevant (
                    <xref ref-type="bibr" rid="ref-1">Bado, 2022</xref>). Games focused on sexual health and education have been successfully deployed in other settings (
                    <xref ref-type="bibr" rid="ref-3">Bertozzi 
                        <italic toggle="yes">et al.</italic>, 2018</xref>; 
                    <xref ref-type="bibr" rid="ref-13">Fiellin 
                        <italic toggle="yes">et al.</italic>, 2017</xref>; 
                    <xref ref-type="bibr" rid="ref-17">Haruna 
                        <italic toggle="yes">et al.</italic>, 2018</xref>). This project is a development and pre-implementation study to prepare for a randomized controlled trial which will determine if the WMM game is effective in improving reproductive empowerment and contraceptive self-efficacy. Our process was informed by the Exploration, Preparation, Implementation, Sustainment (EPIS) framework to ensure consideration of the complex interactions between the game intervention (WMM), the intended audience, and healthcare providers (
                    <xref ref-type="bibr" rid="ref-24">Nilsen 
                        <italic toggle="yes">et al.</italic>, 2020</xref>) .</p>
            </sec>
            <sec>
                <title>Aims</title>
                <p>The aims of this research were to: 1) Conduct focus groups and interviews to determine the acceptability of the WMM game as a tool to educate players about contraception and improve reproductive empowerment.  2) Observe players to determine any usability issues or technical difficulties. 3) Assess the utility and accuracy of the pre/post questionnaire. 4) Conduct interviews with key stakeholders (healthcare providers at clinics, faculty at nursing school) to understand their needs and desired outcomes.</p>
            </sec>
        </sec>
        <sec>
            <title>Videogame intervention: The What&#x2019;s My Method? game</title>
            <sec>
                <title>Development</title>
                <p>The first version of the WMM game was designed to be integrated into the 
                    <bold>My Future Family</bold> (MFF) game funded by Grand Challenges in Global Health
                    <sup>
                        <xref ref-type="other" rid="FN1">1</xref>
                    </sup> in 2016 and deployed
                    <sup>
                        <xref ref-type="other" rid="FN2">2</xref>
                    </sup> in Mysore and Chennai, India in 2017 and 2018 (
                    <xref ref-type="bibr" rid="ref-3">Bertozzi 
                        <italic toggle="yes">et al.</italic>, 2018</xref>; 
                    <xref ref-type="bibr" rid="ref-5">Bertozzi 
                        <italic toggle="yes">et al.</italic>, 2021</xref>). The MFF game was aimed at school-age children who otherwise received no, or very little, education about sex. It provided information about sexual and reproductive anatomy, and collected data from gameplay about family planning intentions. Post deployment qualitative interviews in India recommended adding information about contraceptive methods and modalities (
                    <xref ref-type="bibr" rid="ref-5">Bertozzi 
                        <italic toggle="yes">et al</italic>., 2021</xref>). The WMM game was designed to deliver information about contraception by providing patients with foundational knowledge about methods, and then engaging their new knowledge to assist in-game couples with method selection. Each couple has different lifestyles, physiological needs, and personal preferences. Players have to research methodologies and side effects of methods to find the one/s best suited to each couple.</p>
                <p>Due to the COVID pandemic, the WMM addendum was never deployed or tested in the field, though we were able to conduct informal usability testing on reproductive-age volunteers. In 2023, we began a partnership with the Barbados Family Planning Association and received funding to update WMM as a standalone game focused on supporting contraceptive counseling. The original version of WMM had three sections: Reproductive Anatomy, Contraceptive Methods and Modalities, and Couples. One of the goals of this pre-implementation study was to test and design updates for each of these sections, including integration of lessons learned from past deployments and addressing the needs of the new target audience informed by the EPIS framework (
                    <xref ref-type="bibr" rid="ref-24">Nilsen 
                        <italic toggle="yes">et al.</italic>, 2020</xref>). Game development is an iterative process requiring testing on the target audience during the production cycle to ensure that the game is fun, easy to use and understand, and achieves the study goals. We scheduled the pre-implementation study after building a prototype of the new graphic design for the game but before the design had been extended to the Couples section.</p>
            </sec>
            <sec>
                <title>Reproductive anatomy</title>
                <p>The focus groups in India for the MFF game (
                    <xref ref-type="bibr" rid="ref-3">Bertozzi 
                        <italic toggle="yes">et al.</italic>, 2018</xref>) revealed that players could not understand how contraception works without understanding sexual reproductive anatomy. Thus, the MFF game includes a section where players identify reproductive body parts and their functions. An unanticipated positive result of this part of the game was that it normalized study participant verbalization of anatomic vocabulary during gameplay and in post play conversations (i.e. many students were very uncomfortable saying &#x201c;vagina&#x201d; before playing the game, and afterwards did so without hesitation). We attempted a formal pre/post assessment of knowledge gain in the MFF deployment using a drag and drop quiz, but this was not well-received by players and many declined to complete the post-game assessment 
                    <xref ref-type="bibr" rid="ref-5">Bertozzi 
                        <italic toggle="yes">et al.</italic>, 2021</xref>. Lessons learned from the India deployments guided our decision to both retain a reproductive anatomy section in the new WMM game and to use this segment as a knowledge assessment tool. This section assesses players knowledge of the names and functions of the body parts by seeing how quickly they solve the puzzle and how often they make mistakes. In the RCT we will see if familiarity with reproductive anatomy and normalization of pronouncing the terms facilitates frank conversations with healthcare providers as part of our assessment of reproductive empowerment.</p>
                <p>Players in Barbados tested the updated Reproductive Anatomy section (
                    <xref ref-type="fig" rid="f1">Figure 1</xref>). When they click on a term, it is highlighted and a question appears. They then select the dashed oval which is the correct answer. If the selection is correct. The text fills the oval which becomes green. If the answer is incorrect the dashed oval flashes red and the text is not placed there. Players cannot proceed until each oval is filled correctly.</p>
                <fig fig-type="figure" id="f1" orientation="portrait" position="float">
                    <label>Figure 1. </label>
                    <caption>
                        <title>Screenshots from revised reproductive anatomy section deployed in Barbados.</title>
                        <p>Reproduced with permission from SolitonZ Games.</p>
                    </caption>
                    <graphic orientation="portrait" position="float" xlink:href="https://gatesopenresearch-files.f1000.com/manuscripts/16726/aaad7dba-516e-4942-80a1-d0dcc523e6da_figure1.gif"/>
                </fig>
            </sec>
            <sec>
                <title>Contraceptive methods and modalities</title>
                <p>The first version of the WMM game includes seven methods: Condom, Pill, Copper IUD, Hormonal IUD, Tubal Ligation/Vasectomy, Depo-Provera Injection, and None (
                    <xref ref-type="fig" rid="f2">Figure 2</xref>). Players can see a short animation for each method that explains how it works and how it might not. For example, a condom works by physically covering the penis and blocking sperm from entering the vagina and therefore reaching the egg. It does not work if you forget to buy one or use it incorrectly. The animations engage players by depicting cartoon facial expressions of emotions such as concern, embarrassment, and delight. This section of the game was very well received by test users in the India deployment (
                    <xref ref-type="bibr" rid="ref-5">Bertozzi 
                        <italic toggle="yes">et al</italic>., 2021</xref>). However, healthcare provider reviewers requested that the game include significantly more information, including additional methods, discussion of side effects (both positive and negative), general efficacy (how often the method fails), duration of the method, and timeline for return to fertility.</p>
                <fig fig-type="figure" id="f2" orientation="portrait" position="float">
                    <label>Figure 2. </label>
                    <caption>
                        <title>Screenshots of condom information from original WMM methods section.</title>
                        <p>Reproduced with permission from SolitonZ Games.</p>
                    </caption>
                    <graphic orientation="portrait" position="float" xlink:href="https://gatesopenresearch-files.f1000.com/manuscripts/16726/aaad7dba-516e-4942-80a1-d0dcc523e6da_figure2.gif"/>
                </fig>
                <p>One of the goals of our games is to provide complex, scientifically accurate information in formats that can be understood across a wide range of literacy levels. The addition of so much more information about each method required the design of a series of icons to represent side effects of different methods as well as icons that could communicate Return to Fertility, Failure Rate, and Duration. In Barbados, participants tested the legibility of the new icons and the revised Contraceptive Methods section, which at that point included 2 methods (
                    <xref ref-type="fig" rid="f3">Figure 3</xref>).</p>
                <fig fig-type="figure" id="f3" orientation="portrait" position="float">
                    <label>Figure 3. </label>
                    <caption>
                        <title>New icons and Methods section tested in Barbados.</title>
                        <p>Reproduced with permission from SolitonZ Games.</p>
                    </caption>
                    <graphic orientation="portrait" position="float" xlink:href="https://gatesopenresearch-files.f1000.com/manuscripts/16726/aaad7dba-516e-4942-80a1-d0dcc523e6da_figure3.gif"/>
                </fig>
            </sec>
            <sec>
                <title>Couples</title>
                <p>We deployed the original version of the Couples section of the game (
                    <xref ref-type="fig" rid="f4">Figure 4</xref>) to test the game mechanic and engagement among the Barbadian target audience and explained that the interface would be updated with the new graphic design and that Indian avatars and narratives would be replaced with Barbadian ones for the RCT. Players can view 4 couples each with different needs and characteristics which are communicated both in on-screen text and voiceovers. Players select a method at the bottom of the screen and the couples respond to that selection in a way that explains its features. For example, Reyansh is not interested in getting her tubes tied because she is afraid of the surgery and because she wants more children, but her partner would consider it.</p>
                <fig fig-type="figure" id="f4" orientation="portrait" position="float">
                    <label>Figure 4. </label>
                    <caption>
                        <title>Screenshots from original WMM game deployed in Barbados.</title>
                        <p>Reproduced with permission from SolitonZ Games.</p>
                    </caption>
                    <graphic orientation="portrait" position="float" xlink:href="https://gatesopenresearch-files.f1000.com/manuscripts/16726/aaad7dba-516e-4942-80a1-d0dcc523e6da_figure4.gif"/>
                </fig>
            </sec>
        </sec>
        <sec sec-type="methods">
            <title>Methods</title>
            <sec>
                <title>Theoretical frameworks</title>
                <p>Interactive learning games leverage several theoretical models for achieving desired behavior change. These include social learning theory and social cognitive theory, both of which posit that there are interrelated factors (embedded learning, repetition, enjoyment) that influence behavior after the completion of the experience (
                    <xref ref-type="bibr" rid="ref-19">Krath 
                        <italic toggle="yes">et al</italic>., 2021</xref>). The efficacy of games for health and app-based gamification of health-focused education is now well established (
                    <xref ref-type="bibr" rid="ref-16">Haoran 
                        <italic toggle="yes">et al</italic>., 2019</xref>; 
                    <xref ref-type="bibr" rid="ref-27">Sharifzadeh 
                        <italic toggle="yes">et al</italic>., 2020</xref>).  The WMM game is designed to employ known educational learning game principles, utilize research-standard tools (
                    <xref ref-type="bibr" rid="ref-11">Duncan 
                        <italic toggle="yes">et al</italic>., 2014</xref>), and include learning through empathy. Empathy is increasingly recognized as an important component of healthcare education and science learning (
                    <xref ref-type="bibr" rid="ref-22">Mikkonen 
                        <italic toggle="yes">et al</italic>., 2015</xref>; 
                    <xref ref-type="bibr" rid="ref-33">Zeyer &amp; Dillon, 2019</xref>).</p>
                <p>Conversations about sexual reproduction, contraception, and family planning are often complicated by shame, cultural expectations, and countless other factors. Providing information in a format that allows participants privacy and freedom to explore topics without judgement can enable more constructive conversations (
                    <xref ref-type="bibr" rid="ref-3">Bertozzi 
                        <italic toggle="yes">et al.</italic>, 2018</xref>).  The WMM game evokes empathetic behavior by asking players to help avatar couples in the game and select a method that best meets the couples&#x2019; physiological and behavioral needs. To succeed, players must literally listen to the expressed needs of the people in the game and observe their interactions with their partners. The in-game conversations reinforce information about how methods work, how they fail, and how side effects can be both negative and positive in different circumstances. They are motivated by compelling narratives to find the information needed to pick an optimal method that will satisfy both partners. The RCT seeks to determine if this engaged and contextualized approach improves patients&#x2019; reproductive self-efficacy and reported satisfaction with contraceptive counseling.</p>
                <p>Measuring contraceptive knowledge is complicated by many factors including health literacy, assumptions about what constitutes basic contraceptive education, and participant willingness to complete a quiz. It is difficult to word text-based questions to accurately determine respondent knowledge. Our pre and post test questions were based on simplified questions from the Contraceptive Knowledge Assessment (CKA) (
                    <xref ref-type="bibr" rid="ref-18">Haynes 
                        <italic toggle="yes">et al</italic>., 2017</xref>). We sought to determine if this metric would be useful for our purposes or if we should focus on collecting performance metrics from player behavior.</p>
            </sec>
            <sec>
                <title>Design</title>
                <p>The design of the pre-implementation study sought to ensure that the research, game design, and implementation teams had input from all stakeholders. We conducted focus groups and collected both qualitative and quantitative data.</p>
            </sec>
            <sec sec-type="subjects">
                <title>Participants</title>
                <p>As this was a development and pre-implementation study no sampling was involved. We included anyone who volunteered to participate from our target demographics including: staff and administration from the Barbados Family Planning Association (BFPA), faculty and students from the Nursing program at Barbados Community College (BCC), and volunteers from the Youth Advocacy Movement (YAM), representing the childbearing persons who are the target audience of the intervention (
                    <xref ref-type="table" rid="T1">Table 1</xref>). Staff and administrators at the BFPA are our partners and agreed to participate in usability testing as healthcare providers. The BFPA is a hybrid governmental and non-profit organization. It was incorporated in 1967 by an Act of Parliament and provides sexual and reproductive health information, clinical services, education, training, community outreach, and research. A recruitment call was sent out to the YAM volunteers who meet at BFPA prior to conducting community outreach. Nursing students from BCC were recruited as play-testers who would serve both as representatives of our target population (childbearing persons in Barbados) and as future healthcare providers. BCC provides education and training for entry-level nurses and post-registration nursing programs in Barbados and attracts students from across the Caribbean. There is a renewed focus on supporting research among nursing faculty that provides experiential education opportunities for students. BCC faculty and students will be involved in the RCT.</p>
                <table-wrap id="T1" orientation="portrait" position="anchor">
                    <label>Table 1. </label>
                    <caption>
                        <title>Participants.</title>
                    </caption>
                    <table content-type="article-table" frame="hsides">
                        <thead>
                            <tr>
                                <th align="left" colspan="1" rowspan="1" valign="top">Category</th>
                                <th align="left" colspan="1" rowspan="1" valign="top">Participants
                                    <break/>(N=42), n (%)</th>
                                <th align="left" colspan="1" rowspan="1" valign="top">Demographics</th>
                            </tr>
                        </thead>
                        <tbody>
                            <tr>
                                <td align="left" colspan="1" rowspan="1" valign="top">YAM ambassadors</td>
                                <td align="left" colspan="1" rowspan="1" valign="top">8 (22)</td>
                                <td align="left" colspan="1" rowspan="1" valign="top">4 female, 4 male ages (20&#x2013;25)</td>
                            </tr>
                            <tr>
                                <td align="left" colspan="1" rowspan="1" valign="top">BFPA staff and affiliates</td>
                                <td align="left" colspan="1" rowspan="1" valign="top">7 (18)</td>
                                <td align="left" colspan="1" rowspan="1" valign="top">6 female, 1 male ages (32&#x2013;53)</td>
                            </tr>
                            <tr>
                                <td align="left" colspan="1" rowspan="1" valign="top">BCC nursing students</td>
                                <td align="left" colspan="1" rowspan="1" valign="top">27(70)</td>
                                <td align="left" colspan="1" rowspan="1" valign="top">18 female, 9 male ages (20&#x2013;35)</td>
                            </tr>
                        </tbody>
                    </table>
                </table-wrap>
            </sec>
            <sec sec-type="methods">
                <title>Procedures</title>
                <p>Playtesting and focus groups with each category of participant were conducted on separate days during October 2023. The research team had 6 tablets loaded with 2 files. One was a beta version of the new WMM game with a tutorial for understanding the side effect icons, new icons to represent birth control methods, the Reproductive Anatomy puzzle, and a test version of the Methods section. The other was the original WMM with the Methods and Couples sections.</p>
                <p>The protocol began by having participants complete paper consent forms. They then were provided with a QR code to access the pre-test (
                    <xref ref-type="fig" rid="f5">Figure 5</xref>) on their phones. The pre-test asks for optional gender identification and 5 questions about contraceptive knowledge. Following submission of the pre-test, tablets were handed to participants. They were encouraged to work together if desired (which was necessary when participants outnumbered tablets). The following instructions were provided: Play the new version of the WMM game which includes a tutorial section with icons, the Reproductive Anatomy puzzle, and a Methods section; and then play the Couples game from the older version. They were told they could stop playing at any time and could ask questions or receive help if they were confused about how to proceed. When they were finished playing, they were provided with a different QR code for the post-test (
                    <xref ref-type="fig" rid="f6">Figure 6</xref>) to be completed on their phones. The post-test consists of 10 questions that collect player and usability feedback on the play experience and then 5 questions about contraceptive knowledge that mirror those in the pretest, though the wording is different. Data was collected using anonymized Outlook forms stored on secure academic servers.</p>
                <fig fig-type="figure" id="f5" orientation="portrait" position="float">
                    <label>Figure 5. </label>
                    <caption>
                        <title>Participant knowledge pre-test.</title>
                    </caption>
                    <graphic orientation="portrait" position="float" xlink:href="https://gatesopenresearch-files.f1000.com/manuscripts/16726/aaad7dba-516e-4942-80a1-d0dcc523e6da_figure5.gif"/>
                </fig>
                <fig fig-type="figure" id="f6" orientation="portrait" position="float">
                    <label>Figure 6. </label>
                    <caption>
                        <title>Participant knowledge post-test.</title>
                    </caption>
                    <graphic orientation="portrait" position="float" xlink:href="https://gatesopenresearch-files.f1000.com/manuscripts/16726/aaad7dba-516e-4942-80a1-d0dcc523e6da_figure6.gif"/>
                </fig>
                <p>The YAM ambassadors who agreed to participate met with researchers at BFPA main clinic. They were all given the consent forms, then pre-test, tablets, and post-test. Some participants chose to play next to each other and discuss the process. Others sat apart and completed it on their own. Following the experience, researchers led an informal discussion where more feedback and suggestions were collected.</p>
                <p>The BFPA staff and affiliates who participated met individually with researchers. They went through the same protocol and then provided additional verbal feedback about playing the game as a participant and how they would use the game as part of their provision of healthcare to childbearing patients and their partners.</p>
                <p>BCC students participated in a classroom setting. Participants clustered in groups of 3 or 4 per tablet and played communally after having completed the pre-test individually on phones.</p>
            </sec>
            <sec>
                <title>Analysis</title>
                <p>Researchers observed and documented participant behavior during playtesting sessions and post play group discussion. Salient and recurring themes were collected, and technical and usability issues were noted.</p>
                <p>40 participants completed the pre-test, and 33 completed the post-test. Average time needed for all three steps was 47 minutes.  Given that all of the participants in the pre-implementation study are either healthcare professionals or undergoing related training, we expected high rates of knowledge of reproductive anatomy and contraceptive knowledge. We sought to determine the usability of the Reproductive Anatomy puzzle and comprehensibility of the questionnaire items. </p>
                <p>The pre-test consisted of 5 multiple choice questions about contraceptive knowledge. These questions were coded as either correct or incorrect. The post-test asked the same 5 questions in a different order with slightly different wording and were coded as correct or incorrect. Feedback on the overall playability and acceptability of the game and on specific parts of the game was collected using a 1-5 Likert scale.</p>
            </sec>
            <sec>
                <title>Ethical considerations</title>
                <p>All the study procedures were approved by the Quinnipiac Institutional Review Board (protocol #15923, approved 10/17/2023). Given the sensitive nature of the topics covered by the game, participants were informed in advance that they would see representations of reproductive processes and reproductive anatomy. They were also informed both in the consent document and verbally that they could stop participating at any time without penalty. All participants completed a written informed consent form prior to participating in the study and were verbally advised that they could stop at any time. 2 participants did not complete the pre-test and 7 did not complete the post-test. Thus our comparison of pre/post results utilized a data set of 33 responses.</p>
            </sec>
        </sec>
        <sec sec-type="results">
            <title>Results</title>
            <p>A two-tailed comparison of mean pre and post scores on the five contraceptive knowledge questions did not show significant improvement after having played WMM (
                <xref ref-type="table" rid="T2">Table 2</xref>). One of the questions (5) had more variations in the means. Given that the possible answers are binary (correct/ incorrect) we conducted a McNemar&#x2019;s test on the 33 individual responses which demonstrated slightly more impact from playing WMM, but still not significant. (
                <xref ref-type="table" rid="T3">Table 3</xref>).</p>
            <table-wrap id="T2" orientation="portrait" position="anchor">
                <label>Table 2. </label>
                <caption>
                    <title>Analysis of scores on pre/post contraceptive knowledge.</title>
                </caption>
                <table content-type="article-table" frame="hsides">
                    <thead>
                        <tr>
                            <th align="left" colspan="3" rowspan="1" valign="top">t-Test: Paired Two Sample for Means comparing all
                                <break/>Pre/Post</th>
                        </tr>
                        <tr>
                            <th align="center" colspan="1" rowspan="1" valign="top"/>
                            <th align="center" colspan="1" rowspan="1" valign="top">
                                <italic toggle="yes">Variable 1</italic>
                            </th>
                            <th align="center" colspan="1" rowspan="1" valign="top">
                                <italic toggle="yes">Variable 2</italic>
                            </th>
                        </tr>
                    </thead>
                    <tbody>
                        <tr>
                            <td align="left" colspan="1" rowspan="1" valign="top">Mean</td>
                            <td align="right" colspan="1" rowspan="1" valign="top">3.18181818</td>
                            <td align="right" colspan="1" rowspan="1" valign="top">3.454545455</td>
                        </tr>
                        <tr>
                            <td align="left" colspan="1" rowspan="1" valign="top">Variance</td>
                            <td align="right" colspan="1" rowspan="1" valign="top">1.40340909</td>
                            <td align="right" colspan="1" rowspan="1" valign="top">1.255681818</td>
                        </tr>
                        <tr>
                            <td align="left" colspan="1" rowspan="1" valign="top">Observations</td>
                            <td align="right" colspan="1" rowspan="1" valign="top">33</td>
                            <td align="right" colspan="1" rowspan="1" valign="top">33</td>
                        </tr>
                        <tr>
                            <td align="left" colspan="1" rowspan="1" valign="top">Pearson Correlation</td>
                            <td align="right" colspan="1" rowspan="1" valign="top">0.33598899</td>
                            <td align="right" colspan="1" rowspan="1" valign="top"/>
                        </tr>
                        <tr>
                            <td align="left" colspan="1" rowspan="1" valign="top">Hypothesized Mean Difference</td>
                            <td align="right" colspan="1" rowspan="1" valign="top">0</td>
                            <td align="right" colspan="1" rowspan="1" valign="top"/>
                        </tr>
                        <tr>
                            <td align="left" colspan="1" rowspan="1" valign="top">df</td>
                            <td align="right" colspan="1" rowspan="1" valign="top">32</td>
                            <td align="right" colspan="1" rowspan="1" valign="top"/>
                        </tr>
                        <tr>
                            <td align="left" colspan="1" rowspan="1" valign="top">t Stat</td>
                            <td align="right" colspan="1" rowspan="1" valign="top">-1.1785871</td>
                            <td align="right" colspan="1" rowspan="1" valign="top"/>
                        </tr>
                        <tr>
                            <td align="left" colspan="1" rowspan="1" valign="top">P(T&lt;=t) one-tail</td>
                            <td align="right" colspan="1" rowspan="1" valign="top">0.12362626</td>
                            <td align="right" colspan="1" rowspan="1" valign="top"/>
                        </tr>
                        <tr>
                            <td align="left" colspan="1" rowspan="1" valign="top">t Critical one-tail</td>
                            <td align="right" colspan="1" rowspan="1" valign="top">1.69388875</td>
                            <td align="right" colspan="1" rowspan="1" valign="top"/>
                        </tr>
                        <tr>
                            <td align="left" colspan="1" rowspan="1" valign="top">P(T&lt;=t) two-tail</td>
                            <td align="right" colspan="1" rowspan="1" valign="top">0.24725252</td>
                            <td align="right" colspan="1" rowspan="1" valign="top"/>
                        </tr>
                        <tr>
                            <td align="left" colspan="1" rowspan="1" valign="top">t Critical two-tail</td>
                            <td align="right" colspan="1" rowspan="1" valign="top">2.03693334</td>
                            <td align="right" colspan="1" rowspan="1" valign="top"/>
                        </tr>
                    </tbody>
                </table>
            </table-wrap>
            <table-wrap id="T3" orientation="portrait" position="anchor">
                <label>Table 3. </label>
                <table content-type="article-table" frame="hsides">
                    <thead>
                        <tr>
                            <th align="left" colspan="2" rowspan="1" valign="top">McNemar's test to compare individual scores
                                <break/>on Answer 5</th>
                        </tr>
                    </thead>
                    <tbody>
                        <tr>
                            <td align="left" colspan="1" rowspan="1" valign="top">Wrong answer Pre -&gt;correct on Post</td>
                            <td align="right" colspan="1" rowspan="1" valign="top">12</td>
                        </tr>
                        <tr>
                            <td align="left" colspan="1" rowspan="1" valign="top">Correct on Pre to incorrect on Post</td>
                            <td align="right" colspan="1" rowspan="1" valign="top">4</td>
                        </tr>
                        <tr>
                            <td align="left" colspan="1" rowspan="1" valign="top">No change</td>
                            <td align="right" colspan="1" rowspan="1" valign="top">17</td>
                        </tr>
                        <tr>
                            <td align="left" colspan="1" rowspan="1" valign="top">Chi Square</td>
                            <td align="right" colspan="1" rowspan="1" valign="top">3.0625</td>
                        </tr>
                        <tr>
                            <td align="left" colspan="1" rowspan="1" valign="top">Alpha</td>
                            <td align="right" colspan="1" rowspan="1" valign="top">0.05</td>
                        </tr>
                        <tr>
                            <td align="left" colspan="1" rowspan="1" valign="top">Critical Value</td>
                            <td align="right" colspan="1" rowspan="1" valign="top">3.84145882</td>
                        </tr>
                        <tr>
                            <td align="left" colspan="1" rowspan="1" valign="top">p value</td>
                            <td align="right" colspan="1" rowspan="1" valign="top">0.08011831</td>
                        </tr>
                    </tbody>
                </table>
            </table-wrap>
            <p>The results of the contraceptive knowledge questionnaire were thus of limited use in assessing contraceptive knowledge. Some questions were universally answered correctly in both pre- and post- tests, while others appeared to demonstrate confusing patterns of information acquisition. The question regarding which method also prevents STIs was universally answered correctly in both pre- and post- tests. However, participants tended to confuse the copper and hormonal IUDs. While 27.5% incorrectly indicated that copper IUDs emit hormones in the pre-test, this increased to 45% in the post-test even though participants correctly assigned the copper IUD to an avatar who refused hormone-based methods in the Couples game. There was also wide variation in response to the question assessing contraceptive efficacy, as participants felt it was unclear which parameters (usability, compliance, etc.) should be given priority when choosing an answer. The one question that demonstrated the greatest difference in pre and post responses was: &#x201c;Which method can cause heavier menstrual bleeding?&#x201d; however this difference was not significant (
                <xref ref-type="table" rid="T3">Table 3</xref>).</p>
            <p>The feedback on game elements demonstrated high acceptance of the game as a learning tool and positive impact on reproductive empowerment. Respondents reported appreciation of learning through interactivity, clarity of the animations and audio that demonstrated how each method works, and enjoyment of the humor and narratives in the game. On a Likert scale of 1-5: the response to &#x201c;How likely are you to remember what you learned in this game the next time you need to pick a method of birth control?&#x201d; was 4.18 and &#x201c;How well did the animations demonstrate how each method of birth control works?&#x201d; was 4.48.</p>
            <p>Almost all respondents (32 of 33) said that they would have played the game for a longer period if there were more couple scenarios in the game for them to solve.  This supported our observational findings during testing. As both individuals and groups played the game, they were told several times that they could stop whenever they wanted to. In almost every case, participants wanted to keep playing until they found the optimal method for each of the four couples in the game.</p>
            <p>Suggestions for improvement also matched observational findings. The majority of players were able to easily intuit how to play the different parts of the game based on familiarity with other phone games. However, some needed assistance and indicated that the game should have a better tutorial explaining the game mechanics. Participants recommended improving the legibility of some of the icons related to side effects, methods, return to fertility, and failure rate.  There were also recommendations to make the avatars, narratives, and voiceovers representative of Barbadians.</p>
            <p>Several male respondents indicated that the couples&#x2019; narratives should be more inclusive of male perspectives and that solutions should also include the male partner. For example, the male could offer to drive his partner to medical appointments if she was otherwise likely to miss them.</p>
        </sec>
        <sec sec-type="discussion">
            <title>Discussion</title>
            <p>This pre-implementation study provided both the research and game design teams with information critical to improvements to the WMM game and to the study design for the RCT.</p>
            <sec>
                <title>WMM</title>
                <p>Although the prototype included a short tutorial demonstrating how to tap on an icon to see what it represented, some participants wanted more instruction. The WMM will therefore have a more complete tutorial depicting a cartoon hand completing the necessary actions.</p>
                <p>Participants were unable to understand the icon we developed for decreased libido and suggested a new one which we implemented (
                    <xref ref-type="fig" rid="f7">Figure 7</xref>).</p>
                <fig fig-type="figure" id="f7" orientation="portrait" position="float">
                    <label>Figure 7. </label>
                    <caption>
                        <title>Previous and current icons for decreased libido based on user recommendations.</title>
                        <p>Reproduced with permission from SolitonZ Games.</p>
                    </caption>
                    <graphic orientation="portrait" position="float" xlink:href="https://gatesopenresearch-files.f1000.com/manuscripts/16726/aaad7dba-516e-4942-80a1-d0dcc523e6da_figure7.gif"/>
                </fig>
                <p>We noted that when participants tested the revised Methods section, they did not pay attention to the icons on the top of the screen that indicate duration, failure rate, and return to fertility for each method. This was also confirmed by participants during group discussions. They found the design to be visually overwhelming and could not interpret the icons or pay attention to the information they conveyed. This information led to a redesign of the Methods interface (
                    <xref ref-type="fig" rid="f8">Figure 8</xref>). In the revision, after the animation demonstrates how a method works, it is replaced by new icons and the information is reinforced by color and placement.</p>
                <fig fig-type="figure" id="f8" orientation="portrait" position="float">
                    <label>Figure 8. </label>
                    <caption>
                        <title>Screenshots from tested and revised Methods sections.</title>
                        <p>Note icons on top of screen at left. Reproduced with permission from SolitonZ Games.</p>
                    </caption>
                    <graphic orientation="portrait" position="float" xlink:href="https://gatesopenresearch-files.f1000.com/manuscripts/16726/aaad7dba-516e-4942-80a1-d0dcc523e6da_figure8.gif"/>
                </fig>
                <p>The Couples section of the game will be supplemented given its popularity with participants. The narratives and avatars will be representative of Barbadians with input from our local partners. Male partners will be more fully characterized with a more active role in the discussion of the methods and how they fit into family life. It will also create an opportunity to provide more detailed information about positive and negative side effects and the likelihood of experiencing them.</p>
            </sec>
            <sec>
                <title>Questionnaire</title>
                <p>Given the limited utility of the pre/post questionnaire for assessing knowledge gain, we plan to change both the process and the measure for the RCT. There will no longer be a pre- or post-test quiz. Our primary outcome will instead be a measure of Contraceptive Self-Efficacy (CSE) based on a validated survey published in 2020. The purpose of the RCT is to measure the impact of the WMM game on participants&#x2019; sense of agency, education, and empowerment around contraceptive decision-making. For this reason, it is important that the primary outcome not be based on the ultimate choice of contraception, but rather on a measure of personal capabilities around behavior change. CSE is a person&#x2019;s belief in their own ability to succeed in contraceptive management, initiation, and continued use (
                    <xref ref-type="bibr" rid="ref-32">Lillian Whiting-Collins 
                        <italic toggle="yes">et al</italic>., 2020</xref>).</p>
                <p>The scale to measure Contraceptive Self-Efficacy among women in sub-Saharan Africa (CSESSA) is a modification of the original CSE and is divided into three sub-scales measuring &#x201c;Husband/partner communication,&#x201d; &#x201c;Provider communication,&#x201d; and &#x201c;Choosing and managing a method.&#x201d; This tool was selected for its high relevance to our study and for its focus on participant-directed education and decision-making.</p>
            </sec>
            <sec>
                <title>Limitations</title>
                <p>As in a previous deployment, our post-intervention assessment was hampered by some participants choosing not to complete the post-game survey. Providing participants with some form of compensation appears to be necessary to ensure a higher completion rate. We will plan the RCT with some incentives to achieve this. We also acknowledge that having the participants in this deployment having to toggle between two different games may have affected the results. The final version of the game will integrate all sections into one app.</p>
            </sec>
        </sec>
        <sec sec-type="conclusions">
            <title>Conclusions</title>
            <p>The health of childbearing persons is significantly impacted by their ability to make choices about their reproductive health. Thoughtful and informative family planning education can support sexual and reproductive empowerment and self-efficacy. In the era of the internet and digital technology, there are nearly endless options for people to obtain information about contraceptive methods. The feedback that we collected demonstrates that the WMM game is well-accepted by the target audience and delivers clearly presented and thorough information in an interactive format. Participants in this pilot test were enthusiastic about the game and almost universally desired additional time for game-play. Several usability flaws and user interface problems were identified and will be corrected. The deployment did not encounter any technical difficulties which suggests that deployment to actual healthcare settings is feasible. The pre/post knowledge gain assessment tool we tested was confusing for participants and did not provide useful data. Qualitative responses provided useful suggestions for improvement and validation of the game as an intervention. This pre-implementation study informed important changes to the deployment process, the WMM game, and to the choice of primary outcome for the RCT.</p>
        </sec>
    </body>
    <back>
        <sec sec-type="data-availability">
            <title>Data availability</title>
            <sec>
                <title>Underlying data</title>
                <p>OSF: What's My Method Barbados Pilot. 
                    <ext-link ext-link-type="uri" xlink:href="https://doi.org/10.17605/OSF.IO/Q2W6U">https://doi.org/10.17605/OSF.IO/Q2W6U</ext-link> (
                    <xref ref-type="bibr" rid="ref-6">Bertozzi &amp; Bertozzi-Villa, 2024</xref>).</p>
                <p>This project contains the following underlying data:</p>
                <list list-type="bullet">
                    <list-item>
                        <p>WMMBarbados_Results_Publish.xlsx</p>
                    </list-item>
                    <list-item>
                        <p>WMM_Feedback_Publish.xlsx</p>
                    </list-item>
                    <list-item>
                        <p>Documentation of the observation</p>
                    </list-item>
                </list>
            </sec>
            <sec>
                <title>Extended data</title>
                <p>This project contains the following extended data:</p>
                <list list-type="bullet">
                    <list-item>
                        <p>Pretest Questionnaire- What&#x2019;s My Method_Contraceptive Knowledge.pdf</p>
                    </list-item>
                    <list-item>
                        <p>Posttest Questionnaire &#x2013; What&#x2019;s My Method_Player Feedback.pdg</p>
                    </list-item>
                    <list-item>
                        <p>Interview and focus group topic guides</p>
                    </list-item>
                </list>
                <p>Data are available under the terms of the 
                    <ext-link ext-link-type="uri" xlink:href="http://creativecommons.org/publicdomain/zero/1.0/">Creative Commons Zero "No rights reserved" data waiver</ext-link> (CC0 1.0 Public domain dedication).</p>
            </sec>
        </sec>
        <ack>
            <title>Acknowledgements</title>
            <p>The SolitonZ Games team that built the What&#x2019;s My Method? Game includes: Liam Day (Lead Programmer), Magda Gourinchas (Lead Artist), Zachary Kohlberg, Jacob Kohlberg, Matthew Merritt, Michael Merritt, and Elena Bertozzi (Game Designer and Team Lead).</p>
        </ack>
        <fn-group>
            <fn>
                <p id="FN1">
                    <sup>1</sup> 
                    <ext-link ext-link-type="uri" xlink:href="https://gcgh.grandchallenges.org/grant/childbearing-intentions-and-family-planning-game">https://gcgh.grandchallenges.org/grant/childbearing-intentions-and-family-planning-game</ext-link>
                </p>
            </fn>
            <fn>
                <p id="FN2">
                    <sup>2</sup> 
                    <ext-link ext-link-type="uri" xlink:href="https://familyplanninggame.qu.edu/index.html">https://familyplanninggame.qu.edu/index.html</ext-link>
                </p>
            </fn>
        </fn-group>
        <ref-list>
            <ref id="ref-1">
                <mixed-citation publication-type="journal">
                    <person-group person-group-type="author">

                        <name name-style="western">
                            <surname>Bado</surname>
                            <given-names>N</given-names>
                        </name>
</person-group>:
                    <article-title>Game-based learning pedagogy: a review of the literature.</article-title>
                    <source>

                        <italic toggle="yes">Interact Learn Envir.</italic>
</source>
                    <year>2022</year>;<volume>30</volume>(<issue>5</issue>):<fpage>936</fpage>&#x2013;<lpage>948</lpage>.
                    <pub-id pub-id-type="doi">10.1080/10494820.2019.1683587</pub-id>
                </mixed-citation>
            </ref>
            <ref id="ref-2">
                <mixed-citation publication-type="journal">
                    <person-group person-group-type="author">

                        <name name-style="western">
                            <surname>Bearak</surname>
                            <given-names>JM</given-names>
                        </name>

                        <name name-style="western">
                            <surname>Popinchalk</surname>
                            <given-names>A</given-names>
                        </name>

                        <name name-style="western">
                            <surname>Beavin</surname>
                            <given-names>C</given-names>
                        </name>

                        <etal/>
</person-group>:
                    <article-title>Country-specific estimates of unintended pregnancy and abortion incidence: a global comparative analysis of levels in 2015&#x2013;2019.</article-title>
                    <source>

                        <italic toggle="yes">BMJ Glob Health.</italic>
</source>
                    <year>2022</year>;<volume>7</volume>(<issue>3</issue>):
                    <elocation-id>e007151</elocation-id>.
                    <pub-id pub-id-type="pmid">35332057</pub-id>
                    <pub-id pub-id-type="doi">10.1136/bmjgh-2021-007151</pub-id>
                    <pub-id pub-id-type="pmcid">8943721</pub-id>
                </mixed-citation>
            </ref>
            <ref id="ref-6">
                <mixed-citation publication-type="data">
                    <person-group person-group-type="author">

                        <name name-style="western">
                            <surname>Bertozzi</surname>
                            <given-names>E</given-names>
                        </name>

                        <name name-style="western">
                            <surname>Bertozzi-Villa</surname>
                            <given-names>C</given-names>
                        </name>
</person-group>:
                    <data-title>What&#x2019;s my method barbados Pilot.</data-title>[datasetp.<year>2024</year>.
                    <ext-link ext-link-type="uri" xlink:href="http://www.doi.org/10.17605/OSF.IO/Q2W6U">http://www.doi.org/10.17605/OSF.IO/Q2W6U</ext-link>
                </mixed-citation>
            </ref>
            <ref id="ref-3">
                <mixed-citation publication-type="journal">
                    <person-group person-group-type="author">

                        <name name-style="western">
                            <surname>Bertozzi</surname>
                            <given-names>E</given-names>
                        </name>

                        <name name-style="western">
                            <surname>Bertozzi-Villa</surname>
                            <given-names>A</given-names>
                        </name>

                        <name name-style="western">
                            <surname>Kulkarni</surname>
                            <given-names>P</given-names>
                        </name>

                        <etal/>
</person-group>:
                    <article-title>Collecting family planning intentions and providing reproductive health information using a tablet-based video game in India
[version 2; peer review: 2 approved].</article-title>
                    <source>

                        <italic toggle="yes">Gates Open Res.</italic>
</source>
                    <year>2018</year>;<volume>2</volume>:<fpage>20</fpage>.
                    <pub-id pub-id-type="pmid">29984358</pub-id>
                    <pub-id pub-id-type="doi">10.12688/gatesopenres.12818.2</pub-id>
                    <pub-id pub-id-type="pmcid">6030399</pub-id>
                </mixed-citation>
            </ref>
            <ref id="ref-5">
                <mixed-citation publication-type="journal">
                    <person-group person-group-type="author">

                        <name name-style="western">
                            <surname>Bertozzi</surname>
                            <given-names>E</given-names>
                        </name>

                        <name name-style="western">
                            <surname>Bertozzi-Villa</surname>
                            <given-names>A</given-names>
                        </name>

                        <name name-style="western">
                            <surname>Padankatti</surname>
                            <given-names>S</given-names>
                        </name>

                        <etal/>
</person-group>:
                    <article-title>Outcomes assessment pitfalls: challenges to quantifying knowledge gain in a sex education game [version 3; peer review: 2 approved, 1 not approved].</article-title>
                    <source>

                        <italic toggle="yes">Gates Open Res.</italic>
</source>
                    <year>2021</year>;<volume>4</volume>:<fpage>73</fpage>.
                    <pub-id pub-id-type="pmid">33824946</pub-id>
                    <pub-id pub-id-type="doi">10.12688/gatesopenres.13129.3</pub-id>
                    <pub-id pub-id-type="pmcid">7993112</pub-id>
                </mixed-citation>
            </ref>
            <ref id="ref-7">
                <mixed-citation publication-type="journal">
                    <person-group person-group-type="author">

                        <name name-style="western">
                            <surname>Burke</surname>
                            <given-names>HM</given-names>
                        </name>

                        <name name-style="western">
                            <surname>Ridgeway</surname>
                            <given-names>K</given-names>
                        </name>

                        <name name-style="western">
                            <surname>Murray</surname>
                            <given-names>K</given-names>
                        </name>

                        <etal/>
</person-group>:
                    <article-title>Reproductive empowerment and contraceptive self-care: a systematic review.</article-title>
                    <source>

                        <italic toggle="yes">Sex Reprod Health Matters.</italic>
</source>
                    <year>2021</year>;<volume>29</volume>(<issue>2</issue>): 2090057.
                    <pub-id pub-id-type="pmid">35892261</pub-id>
                    <pub-id pub-id-type="doi">10.1080/26410397.2022.2090057</pub-id>
                    <pub-id pub-id-type="pmcid">9336472</pub-id>
                </mixed-citation>
            </ref>
            <ref id="ref-8">
                <mixed-citation publication-type="journal">
                    <person-group person-group-type="author">

                        <name name-style="western">
                            <surname>Chukwu</surname>
                            <given-names>E</given-names>
                        </name>

                        <name name-style="western">
                            <surname>Gilroy</surname>
                            <given-names>S</given-names>
                        </name>

                        <name name-style="western">
                            <surname>Addaquay</surname>
                            <given-names>K</given-names>
                        </name>

                        <etal/>
</person-group>:
                    <article-title>Formative study of mobile phone use for family planning among young people in Sierra Leone: global systematic survey.</article-title>
                    <source>

                        <italic toggle="yes">JMIR Form Res.</italic>
</source>
                    <year>2021</year>;<volume>5</volume>(<issue>11</issue>):
                    <elocation-id>e23874</elocation-id>.
                    <pub-id pub-id-type="pmid">34766908</pub-id>
                    <pub-id pub-id-type="doi">10.2196/23874</pub-id>
                    <pub-id pub-id-type="pmcid">8663572</pub-id>
                </mixed-citation>
            </ref>
            <ref id="ref-9">
                <mixed-citation publication-type="journal">
                    <person-group person-group-type="author">

                        <name name-style="western">
                            <surname>Claridge</surname>
                            <given-names>AM</given-names>
                        </name>
</person-group>:
                    <article-title>Pregnancy intentions of first-time mothers and their children's outcomes: unraveling reciprocal pathways.</article-title>
                    <source>

                        <italic toggle="yes">J Marriage Fam.</italic>
</source>
                    <year>2021</year>;<volume>83</volume>(<issue>4</issue>):<fpage>942</fpage>&#x2013;<lpage>960</lpage>.
                    <pub-id pub-id-type="doi">10.1111/jomf.12757</pub-id>
                </mixed-citation>
            </ref>
            <ref id="ref-10">
                <mixed-citation publication-type="journal">
                    <person-group person-group-type="author">

                        <name name-style="western">
                            <surname>Dehlendorf</surname>
                            <given-names>C</given-names>
                        </name>

                        <name name-style="western">
                            <surname>Fitzpatrick</surname>
                            <given-names>J</given-names>
                        </name>

                        <name name-style="western">
                            <surname>Fox</surname>
                            <given-names>E</given-names>
                        </name>

                        <etal/>
</person-group>:
                    <article-title>Cluster randomized trial of a patient-centered contraceptive decision support tool, 
                        <italic toggle="yes">My Birth Control</italic>.</article-title>
                    <source>

                        <italic toggle="yes">Am J Obstet Gynecol.</italic>
</source>
                    <year>2019</year>;<volume>220</volume>(<issue>6</issue>):<fpage>565.e561</fpage>&#x2013;<lpage>565.e512</lpage>.
                    <pub-id pub-id-type="pmid">30763545</pub-id>
                    <pub-id pub-id-type="doi">10.1016/j.ajog.2019.02.015</pub-id>
                </mixed-citation>
            </ref>
            <ref id="ref-11">
                <mixed-citation publication-type="journal">
                    <person-group person-group-type="author">

                        <name name-style="western">
                            <surname>Duncan</surname>
                            <given-names>LR</given-names>
                        </name>

                        <name name-style="western">
                            <surname>Hieftje</surname>
                            <given-names>KD</given-names>
                        </name>

                        <name name-style="western">
                            <surname>Culyba</surname>
                            <given-names>S</given-names>
                        </name>

                        <etal/>
</person-group>:
                    <article-title>Game playbooks: tools to guide multidisciplinary teams in developing videogame-based behavior change interventions.</article-title>
                    <source>

                        <italic toggle="yes">Transl Behav Med.</italic>
</source>
                    <year>2014</year>;<volume>4</volume>(<issue>1</issue>):<fpage>108</fpage>&#x2013;<lpage>116</lpage>.
                    <pub-id pub-id-type="pmid">24653781</pub-id>
                    <pub-id pub-id-type="doi">10.1007/s13142-013-0246-8</pub-id>
                    <pub-id pub-id-type="pmcid">3958589</pub-id>
                </mixed-citation>
            </ref>
            <ref id="ref-12">
                <mixed-citation publication-type="web">
                    <person-group person-group-type="author">

                        <name name-style="western">
                            <surname>Edmeades</surname>
                            <given-names>J</given-names>
                        </name>

                        <name name-style="western">
                            <surname>Hinson</surname>
                            <given-names>L</given-names>
                        </name>

                        <name name-style="western">
                            <surname>Sebany</surname>
                            <given-names>M</given-names>
                        </name>

                        <etal/>
</person-group>:
                    <article-title>A conceptual framework for reproductive empowerment: empowering individuals and couples to improve their health</article-title>. Washington DC: International Center for Research on Women,<year>2018</year>.
                    <ext-link ext-link-type="uri" xlink:href="https://www.icrw.org/wp-content/uploads/2018/10/Reproductive-Empowerment-Background-Paper_100318-FINAL.pdf">Reference Source</ext-link>
                </mixed-citation>
            </ref>
            <ref id="ref-13">
                <mixed-citation publication-type="journal">
                    <person-group person-group-type="author">

                        <name name-style="western">
                            <surname>Fiellin</surname>
                            <given-names>LE</given-names>
                        </name>

                        <name name-style="western">
                            <surname>Hieftje</surname>
                            <given-names>KD</given-names>
                        </name>

                        <name name-style="western">
                            <surname>Pendergrass</surname>
                            <given-names>TM</given-names>
                        </name>

                        <etal/>
</person-group>:
                    <article-title>Video game intervention for sexual risk reduction in minority adolescents: randomized controlled trial.</article-title>
                    <source>

                        <italic toggle="yes">J Med Internet Res.</italic>
</source>
                    <year>2017</year>;<volume>19</volume>(<issue>9</issue>):
                    <elocation-id>e314</elocation-id>.
                    <pub-id pub-id-type="pmid">28923788</pub-id>
                    <pub-id pub-id-type="doi">10.2196/jmir.8148</pub-id>
                    <pub-id pub-id-type="pmcid">5625130</pub-id>
                </mixed-citation>
            </ref>
            <ref id="ref-14">
                <mixed-citation publication-type="journal">
                    <person-group person-group-type="author">

                        <name name-style="western">
                            <surname>Gipson</surname>
                            <given-names>JD</given-names>
                        </name>

                        <name name-style="western">
                            <surname>Koenig</surname>
                            <given-names>MA</given-names>
                        </name>

                        <name name-style="western">
                            <surname>Hindin</surname>
                            <given-names>MJ</given-names>
                        </name>
</person-group>:
                    <article-title>The effects of unintended pregnancy on infant, child, and parental health: a review of the literature.</article-title>
                    <source>

                        <italic toggle="yes">Stud Fam Plann.</italic>
</source>
                    <year>2008</year>;<volume>39</volume>(<issue>1</issue>):<fpage>18</fpage>&#x2013;<lpage>38</lpage>.
                    <pub-id pub-id-type="pmid">18540521</pub-id>
                    <pub-id pub-id-type="doi">10.1111/j.1728-4465.2008.00148.x</pub-id>
                </mixed-citation>
            </ref>
            <ref id="ref-15">
                <mixed-citation publication-type="journal">
                    <person-group person-group-type="author">

                        <name name-style="western">
                            <surname>Hamidi</surname>
                            <given-names>OP</given-names>
                        </name>

                        <name name-style="western">
                            <surname>Deimling</surname>
                            <given-names>T</given-names>
                        </name>

                        <name name-style="western">
                            <surname>Lehman</surname>
                            <given-names>E</given-names>
                        </name>

                        <etal/>
</person-group>:
                    <article-title>High self-efficacy is associated with prescription contraceptive use.</article-title>
                    <source>

                        <italic toggle="yes">Womens Health Issues.</italic>
</source>
                    <year>2018</year>;<volume>28</volume>(<issue>6</issue>):<fpage>509</fpage>&#x2013;<lpage>513</lpage>.
                    <pub-id pub-id-type="pmid">30131220</pub-id>
                    <pub-id pub-id-type="doi">10.1016/j.whi.2018.04.006</pub-id>
                    <pub-id pub-id-type="pmcid">6345511</pub-id>
                </mixed-citation>
            </ref>
            <ref id="ref-16">
                <mixed-citation publication-type="journal">
                    <person-group person-group-type="author">

                        <name name-style="western">
                            <surname>Haoran</surname>
                            <given-names>G</given-names>
                        </name>

                        <name name-style="western">
                            <surname>Bazakidi</surname>
                            <given-names>E</given-names>
                        </name>

                        <name name-style="western">
                            <surname>Zary</surname>
                            <given-names>N</given-names>
                        </name>
</person-group>:
                    <article-title>Serious games in health professions education: review of trends and learning efficacy.</article-title>
                    <source>

                        <italic toggle="yes">Yearb Med Inform.</italic>
</source>
                    <year>2019</year>;<volume>28</volume>(<issue>1</issue>):<fpage>240</fpage>&#x2013;<lpage>248</lpage>.
                    <pub-id pub-id-type="pmid">31022747</pub-id>
                    <pub-id pub-id-type="doi">10.1055/s-0039-1677904</pub-id>
                    <pub-id pub-id-type="pmcid">6697512</pub-id>
                </mixed-citation>
            </ref>
            <ref id="ref-17">
                <mixed-citation publication-type="journal">
                    <person-group person-group-type="author">

                        <name name-style="western">
                            <surname>Haruna</surname>
                            <given-names>H</given-names>
                        </name>

                        <name name-style="western">
                            <surname>Hu</surname>
                            <given-names>X</given-names>
                        </name>

                        <name name-style="western">
                            <surname>Chu</surname>
                            <given-names>SKW</given-names>
                        </name>

                        <etal/>
</person-group>:
                    <article-title>Improving sexual health education programs for adolescent students through game-based learning and gamification.</article-title>
                    <source>

                        <italic toggle="yes">Int J Environ Res Public Health.</italic>
</source>
                    <year>2018</year>;<volume>15</volume>(<issue>9</issue>):<fpage>2027</fpage>.
                    <pub-id pub-id-type="pmid">30227642</pub-id>
                    <pub-id pub-id-type="doi">10.3390/ijerph15092027</pub-id>
                    <pub-id pub-id-type="pmcid">6163657</pub-id>
                </mixed-citation>
            </ref>
            <ref id="ref-18">
                <mixed-citation publication-type="journal">
                    <person-group person-group-type="author">

                        <name name-style="western">
                            <surname>Haynes</surname>
                            <given-names>MC</given-names>
                        </name>

                        <name name-style="western">
                            <surname>Ryan</surname>
                            <given-names>N</given-names>
                        </name>

                        <name name-style="western">
                            <surname>Saleh</surname>
                            <given-names>M</given-names>
                        </name>

                        <etal/>
</person-group>:
                    <article-title>Contraceptive knowledge assessment: validity and reliability of a novel contraceptive research tool.</article-title>
                    <source>

                        <italic toggle="yes">Contraception.</italic>
</source>
                    <year>2017</year>;<volume>95</volume>(<issue>2</issue>):<fpage>190</fpage>&#x2013;<lpage>197</lpage>.
                    <pub-id pub-id-type="pmid">27621043</pub-id>
                    <pub-id pub-id-type="doi">10.1016/j.contraception.2016.09.002</pub-id>
                </mixed-citation>
            </ref>
            <ref id="ref-19">
                <mixed-citation publication-type="journal">
                    <person-group person-group-type="author">

                        <name name-style="western">
                            <surname>Krath</surname>
                            <given-names>J</given-names>
                        </name>

                        <name name-style="western">
                            <surname>Sch&#x00fc;rmann</surname>
                            <given-names>L</given-names>
                        </name>

                        <name name-style="western">
                            <surname>von Korflesch</surname>
                            <given-names>HFO</given-names>
                        </name>
</person-group>:
                    <article-title>Revealing the theoretical basis of gamification: a systematic review and analysis of theory in research on gamification, serious games and game-based learning.</article-title>
                    <source>

                        <italic toggle="yes">Comput Hum Behav.</italic>
</source>
                    <year>2021</year>;<volume>125</volume>:
                    <elocation-id>106963</elocation-id>.
                    <pub-id pub-id-type="doi">10.1016/j.chb.2021.106963</pub-id>
                </mixed-citation>
            </ref>
            <ref id="ref-20">
                <mixed-citation publication-type="journal">
                    <person-group person-group-type="author">

                        <name name-style="western">
                            <surname>Laidlaw</surname>
                            <given-names>R</given-names>
                        </name>

                        <name name-style="western">
                            <surname>Dixon</surname>
                            <given-names>D</given-names>
                        </name>

                        <name name-style="western">
                            <surname>Morse</surname>
                            <given-names>T</given-names>
                        </name>

                        <etal/>
</person-group>:
                    <article-title>Using participatory methods to design an mHealth intervention for a low income country, a case study in Chikwawa, Malawi.</article-title>
                    <source>

                        <italic toggle="yes">BMC Med Inform Decis Mak.</italic>
</source>
                    <year>2017</year>;<volume>17</volume>(<issue>1</issue>): 98.
                    <pub-id pub-id-type="pmid">28679428</pub-id>
                    <pub-id pub-id-type="doi">10.1186/s12911-017-0485-6</pub-id>
                    <pub-id pub-id-type="pmcid">5498888</pub-id>
                </mixed-citation>
            </ref>
            <ref id="ref-21">
                <mixed-citation publication-type="journal">
                    <person-group person-group-type="author">

                        <name name-style="western">
                            <surname>Lepore</surname>
                            <given-names>C</given-names>
                        </name>

                        <name name-style="western">
                            <surname>McNamara</surname>
                            <given-names>M</given-names>
                        </name>

                        <name name-style="western">
                            <surname>Miclette</surname>
                            <given-names>K</given-names>
                        </name>

                        <etal/>
</person-group>:
                    <article-title>Assessment of a novel interactive website to inform adolescent and young adult decision-making about contraception.</article-title>
                    <source>

                        <italic toggle="yes">J Pediatr Adolesc Gynecol.</italic>
</source>
                    <year>2024</year>;<volume>37</volume>(<issue>2</issue>):<fpage>149</fpage>&#x2013;<lpage>155</lpage>.
                    <pub-id pub-id-type="pmid">37935279</pub-id>
                    <pub-id pub-id-type="doi">10.1016/j.jpag.2023.10.006</pub-id>
                </mixed-citation>
            </ref>
            <ref id="ref-22">
                <mixed-citation publication-type="journal">
                    <person-group person-group-type="author">

                        <name name-style="western">
                            <surname>Mikkonen</surname>
                            <given-names>K</given-names>
                        </name>

                        <name name-style="western">
                            <surname>Kyng&#x00e4;s</surname>
                            <given-names>H</given-names>
                        </name>

                        <name name-style="western">
                            <surname>K&#x00e4;&#x00e4;ri&#x00e4;inen</surname>
                            <given-names>M</given-names>
                        </name>
</person-group>:
                    <article-title>Nursing students&#x2019; experiences of the empathy of their teachers: a qualitative study.</article-title>
                    <source>

                        <italic toggle="yes"> Adv Health Sci Educ Theory Pract.</italic>
</source>
                    <year>2015</year>;<volume>20</volume>(<issue>3</issue>):<fpage>669</fpage>&#x2013;<lpage>682</lpage>.
                    <pub-id pub-id-type="pmid">25266998</pub-id>
                    <pub-id pub-id-type="doi">10.1007/s10459-014-9554-0</pub-id>
                </mixed-citation>
            </ref>
            <ref id="ref-24">
                <mixed-citation publication-type="book">
                    <person-group person-group-type="editor">

                        <name name-style="western">
                            <surname>Nilsen</surname>
                            <given-names>P</given-names>
                        </name>

                        <name name-style="western">
                            <surname>Birken</surname>
                            <given-names>SA</given-names>
                        </name>

                        <name name-style="western">
                            <surname>Moullin</surname>
                            <given-names>JC</given-names>
                        </name>

                        <etal/>
</person-group>:
                    <article-title>Chapter 2 Exploration, Preparation, Implementation, Sustainment (EPIS) framework</article-title>.
                    <italic toggle="yes">Handbook on Implementation Science</italic>. Edward Elgar Publishing,
                    <ext-link ext-link-type="uri" xlink:href="http://dx.doi.org/10.4337/9781788975995">https://doi.org/10.4337/9781788975995</ext-link>.<year>2020</year>;<fpage>32</fpage>&#x2013;<lpage>61</lpage>.
                    <pub-id pub-id-type="doi">10.4337/9781788975995.00009</pub-id>
                </mixed-citation>
            </ref>
            <ref id="ref-25">
                <mixed-citation publication-type="web">
                    <collab>Office of Disease Prevention and Health Promotion</collab>:
                    <article-title>Family Planning</article-title>. U.S. Department of Health and Human Services.<year>2023</year>; Retrieved December 4.
                    <ext-link ext-link-type="uri" xlink:href="https://health.gov/healthypeople/objectives-and-data/browse-objectives/family-planning">Reference Source</ext-link>
                </mixed-citation>
            </ref>
            <ref id="ref-26">
                <mixed-citation publication-type="journal">
                    <person-group person-group-type="author">

                        <name name-style="western">
                            <surname>Pfender</surname>
                            <given-names>EJ</given-names>
                        </name>

                        <name name-style="western">
                            <surname>Devlin</surname>
                            <given-names>MM</given-names>
                        </name>
</person-group>:
                    <article-title>What do social media influencers say about birth control? a content analysis of youtube vlogs about birth control.</article-title>
                    <source>

                        <italic toggle="yes">Health Commun.</italic>
</source>
                    <year>2023</year>;<volume>38</volume>(<issue>14</issue>):<fpage>3336</fpage>&#x2013;<lpage>3345</lpage>.
                    <pub-id pub-id-type="pmid">36642835</pub-id>
                    <pub-id pub-id-type="doi">10.1080/10410236.2022.2149091</pub-id>
                </mixed-citation>
            </ref>
            <ref id="ref-27">
                <mixed-citation publication-type="journal">
                    <person-group person-group-type="author">

                        <name name-style="western">
                            <surname>Sharifzadeh</surname>
                            <given-names>N</given-names>
                        </name>

                        <name name-style="western">
                            <surname>Kharrazi</surname>
                            <given-names>H</given-names>
                        </name>

                        <name name-style="western">
                            <surname>Nazari</surname>
                            <given-names>E</given-names>
                        </name>

                        <etal/>
</person-group>:
                    <article-title>Health education serious games targeting health care providers, patients, and public health users: scoping review.</article-title>
                    <source>

                        <italic toggle="yes">JMIR Serious Games.</italic>
</source>
                    <year>2020</year>;<volume>8</volume>(<issue>1</issue>):
                    <elocation-id>e13459</elocation-id>.
                    <pub-id pub-id-type="pmid">32134391</pub-id>
                    <pub-id pub-id-type="doi">10.2196/13459</pub-id>
                    <pub-id pub-id-type="pmcid">7082739</pub-id>
                </mixed-citation>
            </ref>
            <ref id="ref-28">
                <mixed-citation publication-type="web">
                    <person-group person-group-type="author">

                        <name name-style="western">
                            <surname>Squire</surname>
                            <given-names>K</given-names>
                        </name>
</person-group>:
                    <article-title>Video games and learning: teaching and participatory culture in the digital age</article-title>. Technology, Education--Connections (the TEC Series). ERIC,<year>2011</year>.
                    <ext-link ext-link-type="uri" xlink:href="https://www.google.com/url?sa=t&amp;source=web&amp;rct=j&amp;opi=89978449&amp;url=https://journalhosting.ucalgary.ca/index.php/ajer/article/view/55771/42636&amp;ved=2ahUKEwjPycnAoO6FAxWn4TgGHWaCDw0QFnoECC4QAQ&amp;usg=AOvVaw0pscYJsv4q6C39_wPvaEi6">Reference Source</ext-link>
                </mixed-citation>
            </ref>
            <ref id="ref-29">
                <mixed-citation publication-type="journal">
                    <person-group person-group-type="author">

                        <name name-style="western">
                            <surname>Stephenson</surname>
                            <given-names>J</given-names>
                        </name>

                        <name name-style="western">
                            <surname>Bailey</surname>
                            <given-names>JV</given-names>
                        </name>

                        <name name-style="western">
                            <surname>Gubijev</surname>
                            <given-names>A</given-names>
                        </name>

                        <etal/>
</person-group>:
                    <article-title>An interactive website for informed contraception choice: randomised evaluation of 
                        <italic toggle="yes">contraception choices</italic>.</article-title>
                    <source>

                        <italic toggle="yes">Digit Health.</italic>
</source>
                    <year>2020</year>;<volume>6</volume>: 2055207620936435.
                    <pub-id pub-id-type="pmid">32704380</pub-id>
                    <pub-id pub-id-type="doi">10.1177/2055207620936435</pub-id>
                    <pub-id pub-id-type="pmcid">7359649</pub-id>
                </mixed-citation>
            </ref>
            <ref id="ref-30">
                <mixed-citation publication-type="web">
                    <collab>United Nations</collab>:
                    <article-title>World population dashboard</article-title>. United Nations.<year>2023</year>.
                    <ext-link ext-link-type="uri" xlink:href="https://www.unfpa.org/data/world-population-dashboard">Reference Source</ext-link>
                </mixed-citation>
            </ref>
            <ref id="ref-31">
                <mixed-citation publication-type="journal">
                    <person-group person-group-type="author">

                        <name name-style="western">
                            <surname>Wang</surname>
                            <given-names>H</given-names>
                        </name>

                        <name name-style="western">
                            <surname>Gupta</surname>
                            <given-names>S</given-names>
                        </name>

                        <name name-style="western">
                            <surname>Singhal</surname>
                            <given-names>A</given-names>
                        </name>

                        <etal/>
</person-group>:
                    <article-title>An artificial intelligence chatbot for young people's sexual and reproductive health in india (snehai): instrumental case study.</article-title>
                    <source>

                        <italic toggle="yes">J Med Internet Res.</italic>
</source>
                    <year>2022</year>;<volume>24</volume>(<issue>1</issue>):
                    <elocation-id>e29969</elocation-id>.
                    <pub-id pub-id-type="pmid">34982034</pub-id>
                    <pub-id pub-id-type="doi">10.2196/29969</pub-id>
                    <pub-id pub-id-type="pmcid">8764609</pub-id>
                </mixed-citation>
            </ref>
            <ref id="ref-32">
                <mixed-citation publication-type="journal">
                    <person-group person-group-type="author">

                        <name name-style="western">
                            <surname>Whiting-Collins</surname>
                            <given-names>L</given-names>
                        </name>

                        <name name-style="western">
                            <surname>Grenier</surname>
                            <given-names>L</given-names>
                        </name>

                        <name name-style="western">
                            <surname>Winch</surname>
                            <given-names>PJ</given-names>
                        </name>

                        <etal/>
</person-group>:
                    <article-title>Measuring Contraceptive Self-Efficacy in Sub-Saharan Africa: development and validation of the CSESSA scale in Kenya and Nigeria.</article-title>
                    <source>

                        <italic toggle="yes">Contracept X.</italic>
</source>
                    <year>2020</year>;<volume>2</volume>:
                    <elocation-id>100041</elocation-id>.
                    <pub-id pub-id-type="pmid">33145490</pub-id>
                    <pub-id pub-id-type="doi">10.1016/j.conx.2020.100041</pub-id>
                    <pub-id pub-id-type="pmcid">7591732</pub-id>
                </mixed-citation>
            </ref>
            <ref id="ref-40">
                <mixed-citation publication-type="web">
                    <collab>World Health Organization</collab>:
                    <article-title>WHO guideline on self-care interventions for health and well-being</article-title>. (9240030905). World Health Organization,<year>2022</year>.
                    <ext-link ext-link-type="uri" xlink:href="https://www.who.int/publications/i/item/9789240052192">Reference Source</ext-link>
                </mixed-citation>
            </ref>
            <ref id="ref-33">
                <mixed-citation publication-type="journal">
                    <person-group person-group-type="author">

                        <name name-style="western">
                            <surname>Zeyer</surname>
                            <given-names>A</given-names>
                        </name>

                        <name name-style="western">
                            <surname>Dillon</surname>
                            <given-names>J</given-names>
                        </name>
</person-group>:
                    <article-title>The role of empathy for learning in complex science|environment|health contexts.</article-title>
                    <source>

                        <italic toggle="yes">Int J Sci Educ.</italic>
</source>
                    <year>2019</year>;<volume>41</volume>(<issue>3</issue>):<fpage>297</fpage>&#x2013;<lpage>315</lpage>.
                    <pub-id pub-id-type="doi">10.1080/09500693.2018.1549371</pub-id>
                </mixed-citation>
            </ref>
        </ref-list>
    </back>
    <sub-article article-type="reviewer-report" id="report37843">
        <front-stub>
            <article-id pub-id-type="doi">10.21956/gatesopenres.16726.r37843</article-id>
            <title-group>
                <article-title>Reviewer response for version 1</article-title>
            </title-group>
            <contrib-group>
                <contrib contrib-type="author">
                    <name>
                        <surname>Torres-Cort&#x00e9;s</surname>
                        <given-names>Betzabe</given-names>
                    </name>
                    <xref ref-type="aff" rid="r37843a1">1</xref>
                    <role>Referee</role>
                </contrib>
                <aff id="r37843a1">
                    <label>1</label>Psychology, Universidad de O'Higgins (Ringgold ID: 519092), Rancagua, O'Higgins Region, Chile</aff>
            </contrib-group>
            <author-notes>
                <fn fn-type="conflict">
                    <p>
                        <bold>Competing interests: </bold>No competing interests were disclosed.</p>
                </fn>
            </author-notes>
            <pub-date pub-type="epub">
                <day>25</day>
                <month>9</month>
                <year>2024</year>
            </pub-date>
            <permissions>
                <copyright-statement>Copyright: &#x00a9; 2024 Torres-Cort&#x00e9;s B</copyright-statement>
                <copyright-year>2024</copyright-year>
                <license xlink:href="https://creativecommons.org/licenses/by/4.0/">
                    <license-p>This is an open access peer review report distributed under the terms of the Creative Commons Attribution Licence, which permits unrestricted use, distribution, and reproduction in any medium, provided the original work is properly cited.</license-p>
                </license>
            </permissions>
            <related-article ext-link-type="doi" id="relatedArticleReport37843" related-article-type="peer-reviewed-article" xlink:href="10.12688/gatesopenres.15376.1"/>
            <custom-meta-group>
                <custom-meta>
                    <meta-name>recommendation</meta-name>
                    <meta-value>reject</meta-value>
                </custom-meta>
            </custom-meta-group>
        </front-stub>
        <body>
            <p>Summary: The study presents the design and preliminary evaluation of a gaming intervention for reproductive empowerment and contraceptive self-efficacy.</p>
            <p> Title:&#x00a0;it should not include the word "development," as the manuscript does not describe the design of the intervention (which was completed prior to the study)</p>
            <p> Methods:</p>
            <p> - I understand that the theoretical foundations refer to what was used in designing the intervention. However, I suggest moving them to the Introduction section, as the Methods section should only include aspects directly related to the study, not the design of the intervention itself.</p>
            <p> - A clearer definition of the methodology is needed. Is it a mixed-methods study or a study with both qualitative and quantitative phases? I recommend referring to established authors for guidance.</p>
            <p> - Design and pre-implementation studies can indeed include a sample. I recommend reviewing this statement.</p>
            <p> - I recommend including a specific section for data collection techniques, as they are currently integrated into the "Procedure" section, which is not appropriate</p>
            <p> - Observation and documentation of participant behavior are data collection techniques. It is recommended to include them as such and provide a detailed description: Was a structured guideline used for observation and documentation, or was it done in an unstructured manner?</p>
            <p> - The description of the pre-test should be included in the Data Collection Techniques section, not in the Analysis section</p>
            <p> - The Analysis section needs to be revised, as it currently does not specify the techniques used for analyzing quantitative and qualitative data. It is unclear how the results were obtained.</p>
            <p> Results:</p>
            <p> - I recommend highlighting which results correspond to each data collection technique (focus groups, observation, documentation and pre-post test). The current presentation requires the reader to infer this information.</p>
            <p> Methods and results:</p>
            <p> - The study indicates that it is based on the EPIS framework; however, the Methods/Results section does not reference the stages of this framework or specify which parts of the study correspond to each stage. I suggest including this information.</p>
            <p> </p>
            <p> Overall, it can be said that the study lacks replicability: its methodology is unclear, it claims to be based on the EPIS framework but does not incorporate this framework in the methods or results sections, it does not adequately identify its data collection techniques, and it lacks detailed descriptions of its data analysis methods. As a result, it is impossible to determine the source of the results or how to replicate the study.</p>
            <p>Is the rationale for developing the new method (or application) clearly explained?</p>
            <p>Yes</p>
            <p>Is the description of the method technically sound?</p>
            <p>Partly</p>
            <p>Are the conclusions about the method and its performance adequately supported by the findings presented in the article?</p>
            <p>Yes</p>
            <p>If any results are presented, are all the source data underlying the results available to ensure full reproducibility?</p>
            <p>No</p>
            <p>Are sufficient details provided to allow replication of the method development and its use by others?</p>
            <p>No</p>
            <p>Reviewer Expertise:</p>
            <p>Designing Health Interventions from an Implementation Science Perspective</p>
            <p>I confirm that I have read this submission and believe that I have an appropriate level of expertise to state that I do not consider it to be of an acceptable scientific standard, for reasons outlined above.</p>
        </body>
        <sub-article article-type="response" id="comment3739-37843">
            <front-stub>
                <contrib-group>
                    <contrib contrib-type="author">
                        <name>
                            <surname>Bertozzi</surname>
                            <given-names>Elena</given-names>
                        </name>
                        <aff>Game Design &amp; Development, Quinnipiac University, Hamden, CT, USA</aff>
                    </contrib>
                </contrib-group>
                <author-notes>
                    <fn fn-type="conflict">
                        <p>
                            <bold>Competing interests: </bold>No competing interests were disclosed.</p>
                    </fn>
                </author-notes>
                <pub-date pub-type="epub">
                    <day>15</day>
                    <month>10</month>
                    <year>2024</year>
                </pub-date>
            </front-stub>
            <body>
                <p>Thank you very much for this helpful review that informed us about gaps in the way that we explained the process and purpose of the study. We have made significant edits to the text to clarify and address your notes in detail below.</p>
                <p> &#x00a0; 
                    <list list-type="order">
                        <list-item>
                            <p>&#x201c;Development&#x201d;</p>
                        </list-item>
                    </list> </p>
                <p> Game design is an iterative process wherein the game is repeatedly tested and modified in response to feedback from the intended audience. The game intervention in this study was not completed prior to the study. The study was an important step in the process of ensuring that the game meets the needs of the healthcare professionals who would be deploying it within a health care setting, that it effectively communicates the desired information, and that the intended audience is motivated to engage and interact with it. The latter half of the paper details the changes that we made to the intervention as a result of this work. Thank you for pointing out that we did not make this clear in the abstract and introduction. We have modified the text to clarify. Given your expertise in Implementation Science, your comments are much appreciated.</p>
                <p> &#x00a0; 
                    <list list-type="order">
                        <list-item>
                            <p>Methods</p>
                        </list-item>
                    </list> </p>
                <p> The theoretical foundations we cite do inform the study. The study is part of the process of improving the WMM game itself and assessing stakeholder feedback on it.</p>
                <p> </p>
                <p> We included more citations and details about the methods of the study.</p>
                <p> </p>
                <p> The analysis section was revised to provide the requested information</p>
                <p> &#x00a0; 
                    <list list-type="order">
                        <list-item>
                            <p>Results</p>
                        </list-item>
                    </list> </p>
                <p> Tables and captions were added to clarify source and type of data and analysis</p>
                <p> </p>
                <p> The note that we should include more of the EPIS framework in the paper is also very helpful. We have added information about the process steps of the framework and how implementation science differs from other kinds of study design.</p>
                <p> </p>
                <p> Given that the study aims to &#x201c;conduct a formative evaluation of the What&#x2019;s My Method? (WMM) game intervention as a tool to support contraceptive counseling&#x201d; as we state in the abstract, the data that we collected was focused on assessing the value of our proposed questionnaire as a tool for quantifying knowledge gain and on the attitudes of the healthcare providers and intended audience for this specific game. Thus the only numerical data we are reporting are 1) A comparison of the pre and post test scores for knowledge gain, and the means of the Likert scale scores for acceptability of the intervention. Based on your and the other reviewers&#x2019; feedback we have edited those tables for clarity.</p>
                <p> </p>
                <p> In terms of replicability &#x2013; as a development and pre-implementation study of a very specific game intervention in a specific healthcare environment, this study is not designed to be replicable. It is a step in the process of preparing for the RCT of the intervention which will be. Although many games for health have been designed and developed, few have actually been integrated into healthcare environments. We hope that this paper will help support others who seek to do so.</p>
                <p> </p>
                <p> Thanks, we appreciate your comments and expertise.</p>
            </body>
        </sub-article>
    </sub-article>
    <sub-article article-type="reviewer-report" id="report37838">
        <front-stub>
            <article-id pub-id-type="doi">10.21956/gatesopenres.16726.r37838</article-id>
            <title-group>
                <article-title>Reviewer response for version 1</article-title>
            </title-group>
            <contrib-group>
                <contrib contrib-type="author">
                    <name>
                        <surname>Undie</surname>
                        <given-names>Chi-Chi</given-names>
                    </name>
                    <xref ref-type="aff" rid="r37838a1">1</xref>
                    <role>Referee</role>
                </contrib>
                <aff id="r37838a1">
                    <label>1</label>Population Council, Nairobi, Kenya</aff>
            </contrib-group>
            <author-notes>
                <fn fn-type="conflict">
                    <p>
                        <bold>Competing interests: </bold>No competing interests were disclosed.</p>
                </fn>
            </author-notes>
            <pub-date pub-type="epub">
                <day>17</day>
                <month>9</month>
                <year>2024</year>
            </pub-date>
            <permissions>
                <copyright-statement>Copyright: &#x00a9; 2024 Undie CC</copyright-statement>
                <copyright-year>2024</copyright-year>
                <license xlink:href="https://creativecommons.org/licenses/by/4.0/">
                    <license-p>This is an open access peer review report distributed under the terms of the Creative Commons Attribution Licence, which permits unrestricted use, distribution, and reproduction in any medium, provided the original work is properly cited.</license-p>
                </license>
            </permissions>
            <related-article ext-link-type="doi" id="relatedArticleReport37838" related-article-type="peer-reviewed-article" xlink:href="10.12688/gatesopenres.15376.1"/>
            <custom-meta-group>
                <custom-meta>
                    <meta-name>recommendation</meta-name>
                    <meta-value>approve-with-reservations</meta-value>
                </custom-meta>
            </custom-meta-group>
        </front-stub>
        <body>
            <p>This paper presents developmental and pre-implementation evidence to inform an upcoming RCT around the use of an exciting game intervention to enhance contraceptive self-efficacy in Barbados. It is an interesting paper which could be strengthened by outlining the methods more clearly. Suggestions for rectifying this and other issues are provided below. 
                <list list-type="bullet">
                    <list-item>
                        <p>The rationale for developing the game intervention is convincingly laid out in the Background section, with extensive arguments building up to the need for new studies that demonstrate self-efficacy related to other contraceptive methods (beyond condoms alone), which in turn result in increased use of those methods. In addition, evidence around lower than desired CPR in the study context, the use of abortion as birth control, etc., is outlined. These arguments imply that the game intervention will play a key role in the uptake of contraception. However, in the Questionnaire section of the paper, the authors indicate that &#x201c;[I]t is important that the primary outcome not be based on the ultimate choice of contraception, but rather on a measure of personal capabilities around behavior change.&#x201d; This assertion is fine in its own right, but it is misaligned with the extensive arguments laid out in the Background section. Some of the language in Background section therefore needs to be tempered to maintain the point that the end goal has to do with personal capabilities around behavior change, rather than actual contraceptive use/uptake/choice.</p>
                    </list-item>
                    <list-item>
                        <p>The Analysis section does not provide a description of the qualitative data analysis. The procedures for conducting the focus group discussions (FGDs) are also not laid out. They are also referred to as an &#x2018;informal discussion&#x2019; in the Procedures section, which makes it unclear to the reader whether several FGDs were carried out, or a single informal discussion. The authors do not indicate how many FGDs were conducted, and with which respondent categories. Little information is provided on how exactly the observations were carried out (were they structured in any way?). In general, the procedures, analysis, and results around the qualitative data seem to have been almost overlooked/omitted, compared to what obtains for the quantitative data. It makes the conclusions less persuasive than they could have been. The vagueness around the qualitative methods makes them challenging for others to use, replicate, and reproduce.</p>
                    </list-item>
                    <list-item>
                        <p>Participants section: The first sentence indicates that &#x2018;no sampling was involved.&#x2019; Instead, it should indicate that a combination of convenience sampling and purposive (or purposeful) sampling was employed &#x2013; convenience because whoever volunteered was enrolled in whatever numbers, but purposeful because the volunteers did have to represent specific categories of people in order to be eligible to participate.</p>
                    </list-item>
                    <list-item>
                        <p>The Conclusions indicate that &#x2018;the game is well-accepted by the target audience and delivers clearly presented and thorough information in an interactive format.&#x2019;&#x00a0; This seems to gloss over the confusion participants experienced in distinguishing between the copper and hormonal IUDs, and in understanding some of the contraceptive self-efficacy parameters, for example. It would be best to temper the language in this sentence. Furthermore, the target audience of childbearing people used by the study is arguably a really select group. The study used youth contraception ambassadors and nursing students, rather than just &#x2018;regular&#x2019; youth, who are probably not as sensitized as the ambassadors or nursing students, and who are, therefore, really the ones that need to develop contraceptive self-efficacy.&#x00a0; The use of these groups, as opposed to less informed youth should be discussed in the Limitations section.</p>
                    </list-item>
                    <list-item>
                        <p>Male participants wanted to see a more inclusive approach to men featured in the game stories. What does this mean for the reproductive autonomy of childbearing persons? Are there any implications? If so, it would be helpful to reflect on this in the Discussion.</p>
                    </list-item>
                    <list-item>
                        <p>Design section: This section currently states: &#x2018;We conducted focus group discussions and collected both qualitative and quantitative data.&#x2019; Since FGDs are qualitative data, it should instead state something like: &#x2018;We conducted both qualitative (focus group discussions) and quantitative (pre- and post-test survey) data.&#x2019;</p>
                    </list-item>
                    <list-item>
                        <p>&#x201c;If the selection is correct.&#x201d; This sentence is hanging, and is probably meant to be a part of the sentence that precedes it.</p>
                    </list-item>
                </list>
            </p>
            <p>Is the rationale for developing the new method (or application) clearly explained?</p>
            <p>Partly</p>
            <p>Is the description of the method technically sound?</p>
            <p>Partly</p>
            <p>Are the conclusions about the method and its performance adequately supported by the findings presented in the article?</p>
            <p>Partly</p>
            <p>If any results are presented, are all the source data underlying the results available to ensure full reproducibility?</p>
            <p>Partly</p>
            <p>Are sufficient details provided to allow replication of the method development and its use by others?</p>
            <p>Partly</p>
            <p>Reviewer Expertise:</p>
            <p>NA</p>
            <p>I confirm that I have read this submission and believe that I have an appropriate level of expertise to confirm that it is of an acceptable scientific standard, however I have significant reservations, as outlined above.</p>
        </body>
        <sub-article article-type="response" id="comment3738-37838">
            <front-stub>
                <contrib-group>
                    <contrib contrib-type="author">
                        <name>
                            <surname>Bertozzi</surname>
                            <given-names>Elena</given-names>
                        </name>
                        <aff>Game Design &amp; Development, Quinnipiac University, Hamden, CT, USA</aff>
                    </contrib>
                </contrib-group>
                <author-notes>
                    <fn fn-type="conflict">
                        <p>
                            <bold>Competing interests: </bold>No competing interests were disclosed.</p>
                    </fn>
                </author-notes>
                <pub-date pub-type="epub">
                    <day>15</day>
                    <month>10</month>
                    <year>2024</year>
                </pub-date>
            </front-stub>
            <body>
                <p>We very much appreciate the note that we needed to clarify how user feedback was collected. This should also be clearer now that we have included more details about the EPIS framework as suggested by Reviewer #3. We list the revisions below in the order suggested by your review. 
                    <list list-type="bullet">
                        <list-item>
                            <p>Modified the first paragraph of the Methods section to clarify that we did not conduct formal focus group sessions where qualitative data was collected. The qualitative responses we report all come from the questionnaires.</p>
                        </list-item>
                        <list-item>
                            <p>Thank you very much for this helpful language re: sampling which we included.</p>
                        </list-item>
                        <list-item>
                            <p>Added text in the Conclusion explaining how we modified the game in response to player confusion in the specified areas and how this fits into the EPIS framework.</p>
                        </list-item>
                        <list-item>
                            <p>Added a qualification to the Limitations Section and also noted it in the Participants section.</p>
                        </list-item>
                        <list-item>
                            <p>We edited the section about including males to reflect the diversity in parenting arrangements.</p>
                        </list-item>
                        <list-item>
                            <p>We reworded the Design section to clarify this point and appreciate your catching the typo.</p>
                        </list-item>
                    </list> Thank you very much for your time and attention to our work. We appreciate your help in improving the paper.</p>
            </body>
        </sub-article>
    </sub-article>
    <sub-article article-type="reviewer-report" id="report37423">
        <front-stub>
            <article-id pub-id-type="doi">10.21956/gatesopenres.16726.r37423</article-id>
            <title-group>
                <article-title>Reviewer response for version 1</article-title>
            </title-group>
            <contrib-group>
                <contrib contrib-type="author">
                    <name>
                        <surname>Mengelkoch</surname>
                        <given-names>Summer</given-names>
                    </name>
                    <xref ref-type="aff" rid="r37423a1">1</xref>
                    <role>Referee</role>
                    <uri content-type="orcid">https://orcid.org/0000-0002-1845-2107</uri>
                </contrib>
                <aff id="r37423a1">
                    <label>1</label>University of California, LA, CA, USA</aff>
            </contrib-group>
            <author-notes>
                <fn fn-type="conflict">
                    <p>
                        <bold>Competing interests: </bold>No competing interests were disclosed.</p>
                </fn>
            </author-notes>
            <pub-date pub-type="epub">
                <day>5</day>
                <month>8</month>
                <year>2024</year>
            </pub-date>
            <permissions>
                <copyright-statement>Copyright: &#x00a9; 2024 Mengelkoch S</copyright-statement>
                <copyright-year>2024</copyright-year>
                <license xlink:href="https://creativecommons.org/licenses/by/4.0/">
                    <license-p>This is an open access peer review report distributed under the terms of the Creative Commons Attribution Licence, which permits unrestricted use, distribution, and reproduction in any medium, provided the original work is properly cited.</license-p>
                </license>
            </permissions>
            <related-article ext-link-type="doi" id="relatedArticleReport37423" related-article-type="peer-reviewed-article" xlink:href="10.12688/gatesopenres.15376.1"/>
            <custom-meta-group>
                <custom-meta>
                    <meta-name>recommendation</meta-name>
                    <meta-value>approve-with-reservations</meta-value>
                </custom-meta>
            </custom-meta-group>
        </front-stub>
        <body>
            <p>The authors present the results of a pre-implementation study assessing What&#x2019;s My Method (WMM), a digital health game designed to teach users about contraception and enhance reproductive self-efficacy. After describing the game, the authors discuss primarily qualitative outcomes assessed in pre-implementation users (
                <italic>N</italic> = 42), and some non-significant outcomes of contraceptive knowledge tests. The game seems quite useful and interesting, and certainly deserves additional study. I was impressed with the design and approach here. In general, the game and study are well-founded and I am excited to hear the results of the RCT. Although of acceptable standards, there are a few points which I believe require additional clarification and reorganization before this is suitable for indexing. Overall, I found this to be a well written report and an interesting, if preliminary, research study. Below, I detail my major comments and suggestions and hope that my feedback improves the overall quality of the current report.</p>
            <p> Major Comments (in order of importance):</p>
            <p> Discussion: 
                <list list-type="order">
                    <list-item>
                        <p>The knowledge gleaned from pre/post quizzes is really important to research. Even if the results don't come out as expected or desired, that is no reason to pivot away and leave knowledge entirely unassessed. Instead, I would HIGHLY recommend also including a pre/post test of knowledge, but considering it a secondary (or even exploratory) outcome for the RCT. Not including it at all because the results came out weird in a small and unincentivized group of participants seems misguided. I think the authors will miss out on really valuable information by entirely relying on efficacy measures. Accurate knowledge is far more empowering than a perception of knowledge that leaves users unprepared to make informed contraceptive choices.&#x00a0;</p>
                    </list-item>
                </list> Results: 
                <list list-type="order">
                    <list-item>
                        <p>Table 2 is very difficult to understand. The authors would benefit from considering and including information that will be useful to the reader. First, the authors should label variable 1 and variable 2 as pre/post scores, if that is what they are. I&#x2019;m assuming that the mean is the mean of the number correct out of five? So higher scores here are better? The authors should be clear about the numbers they are presenting and how they are scored. It is standard to round off values at two decimal places. What measure of variance is listed? Why did the authors hypothesize a mean difference of 0? It seems that the statement might represent a null hypothesis, which is typically not the focus in a paper of this nature. Why are there more t-values than p-values? Perhaps the last line is missing? Instead of presenting all possible t-statistics, the authors should be presenting the stat that is the most appropriate for their analysis (likely the two-tailed) or explain to the reader why multiple stats are presented.</p>
                    </list-item>
                    <list-item>
                        <p>Instead of only presenting values for a few of the feedback questions, all should be presented in a table, as these are the key results interpreted here.</p>
                    </list-item>
                </list> Introduction: 
                <list list-type="order">
                    <list-item>
                        <p>At the outset of the paper, the authors explain that contraceptive counseling and education should be implemented through a &#x201c;lens of reproductive empowerment&#x201d;. They follow with some of these statements: &#x201c;Ideally, this counseling also reviews the many non-contraceptive benefits of hormonal birth control methods,&#x201d; &#x2026; and &#x201c;Childbearing women receive advice from older women propagating unsupported concerns about side effects, which makes them reluctant to adopt modern methods.&#x201d; without citations, implying these are the authors&#x2019; observations or opinions. It is not empowering to highlight benefits without discussing risks, nor to dismiss very real and well-supported side effects of hormonal contraceptives as &#x201c;unsupported concerns&#x201d; because they don&#x2019;t occur in all women under all conditions. It is far more empowering to women to explain the balance of potential costs alongside potential benefits of contraceptive options, allowing women to make fully informed choices that align with their priorities and values. The authors should consider the tone at the outset of the paper, and soften or modify these statements, as the purpose of this paper is not to glorify hormonal contraceptives, but rather to promote reproductive empowerment through gamified contraceptive education in a population where these actions are likely to be very beneficial.&#x00a0;</p>
                    </list-item>
                    <list-item>
                        <p>Before explaining the history and development of the WMM game, it would be helpful to give the reader a brief overview of the game, either just before or just after the aims.</p>
                    </list-item>
                    <list-item>
                        <p>Figure 1: Update the figure to be accurate... sperm aren't produced in fallopian tubes, as the figure implies (I assume this is just a labeling error and not a real screenshot from the game)</p>
                    </list-item>
                    <list-item>
                        <p>Figure 4: Please make the text legible, even if it requires editing the screengrab by overlaying text that is less blurry!</p>
                    </list-item>
                </list> Methods: 
                <list list-type="order">
                    <list-item>
                        <p>The theoretical frameworks section belongs in the introduction, not the methods.</p>
                    </list-item>
                    <list-item>
                        <p>Participants: the statement that no sampling was involved is a bit overstated. It was not random sampling, but sampling still occurred.</p>
                    </list-item>
                </list> Minor comments (the authors should not feel obligated respond to these minor comments unless they would like to, they are simply provided in case they are helpful in improving the quality of the manuscript):</p>
            <p> Abstract: 
                <list list-type="order">
                    <list-item>
                        <p>&#x201c;Results indicate that the WMM game is well-reviewed and accepted&#x201d; Did the authors mean well-received?</p>
                    </list-item>
                    <list-item>
                        <p>Last line: instead of referencing &#x201c;the&#x201d; upcoming RCT, &#x201c;an&#x201d; upcoming RCT is more appropriate.</p>
                    </list-item>
                </list> Introduction: 
                <list list-type="order">
                    <list-item>
                        <p>First line: typical to spell out abbreviations the first time they are used (WHO)</p>
                    </list-item>
                    <list-item>
                        <p>&#x201c;Implementing contraceptive counseling and education through the lens of reproductive empowerment requires that the focus of the intervention be on the childbearing persons and their needs.&#x201d; Remove the &#x201c;the&#x201d; before intervention</p>
                    </list-item>
                    <list-item>
                        <p>If this was meant to be a header, please format as such &#x201c;Demonstrating the impact of interventions on reproductive empowerment is complicated.&#x201d;</p>
                    </list-item>
                    <list-item>
                        <p>The first time the WMM acronym is used, please define it so the reader knows what it is!</p>
                    </list-item>
                    <list-item>
                        <p>&#x201c;Game development is an iterative process requiring testing on the target audience during the production cycle to ensure that the game is fun, easy to use and understand, and achieves the study goals.&#x201d; Here, I don&#x2019;t think &#x201c;achieves the study goals&#x201d; is a part of game development. Perhaps the word game goals would be more accurate.</p>
                    </list-item>
                </list> Methods: 
                <list list-type="order">
                    <list-item>
                        <p>Consider using a screen grab for Fig 6 in which you have removed the blue lines indicating a grammatical error in #1</p>
                    </list-item>
                    <list-item>
                        <p>The statement: &#x201c;2 participants did not complete the pre-test and 7 did not complete the post-test. Thus our comparison of pre/post results utilized a data set of 33 responses.&#x201d; Does not belong under the header &#x201c;ethical considerations&#x201d;</p>
                    </list-item>
                </list> Discussion: really impressed with the way user feedback was used to improve the game!</p>
            <p>Is the rationale for developing the new method (or application) clearly explained?</p>
            <p>Yes</p>
            <p>Is the description of the method technically sound?</p>
            <p>Yes</p>
            <p>Are the conclusions about the method and its performance adequately supported by the findings presented in the article?</p>
            <p>Partly</p>
            <p>If any results are presented, are all the source data underlying the results available to ensure full reproducibility?</p>
            <p>Yes</p>
            <p>Are sufficient details provided to allow replication of the method development and its use by others?</p>
            <p>Partly</p>
            <p>Reviewer Expertise:</p>
            <p>I am a social psychologist, with extensive experience in women's health research, including impacts of hormones and hormonal contraceptive use on stress reactivity, persistence, and depression. I also have extensive experience in psychoneuroimmunology and implementation of clinical interventions.</p>
            <p>I confirm that I have read this submission and believe that I have an appropriate level of expertise to confirm that it is of an acceptable scientific standard, however I have significant reservations, as outlined above.</p>
        </body>
        <sub-article article-type="response" id="comment3737-37423">
            <front-stub>
                <contrib-group>
                    <contrib contrib-type="author">
                        <name>
                            <surname>Bertozzi</surname>
                            <given-names>Elena</given-names>
                        </name>
                        <aff>Game Design &amp; Development, Quinnipiac University, Hamden, CT, USA</aff>
                    </contrib>
                </contrib-group>
                <author-notes>
                    <fn fn-type="conflict">
                        <p>
                            <bold>Competing interests: </bold>No competing interests were disclosed.</p>
                    </fn>
                </author-notes>
                <pub-date pub-type="epub">
                    <day>15</day>
                    <month>10</month>
                    <year>2024</year>
                </pub-date>
            </front-stub>
            <body>
                <p>Thank you for this thorough and thoughtful feedback. We have implemented your suggestions. Please see details below.</p>
                <p> Suggestion for finding a way to effectively measure knowledge gain noted.</p>
                <p> Results Section 
                    <list list-type="order">
                        <list-item>
                            <p>Reformatted Table 2 to improve legibility and include requested data</p>
                        </list-item>
                        <list-item>
                            <p>Added Table 4 with values for all feedback questions.</p>
                        </list-item>
                    </list> Introduction: 
                    <list list-type="order">
                        <list-item>
                            <p>Added text about risks and citation for benefits of hormonal birth control. We revised text to make clear that these are non-scientific concerns about contraceptive side effects.</p>
                        </list-item>
                        <list-item>
                            <p>Added a brief description of WMM before the Aims section</p>
                        </list-item>
                        <list-item>
                            <p>Corrected screenshot label &#x2013; Thanks for catching this!</p>
                        </list-item>
                        <list-item>
                            <p>Redid screenshots and improved resolution. (see figures Figure 1B, and new Figures 4a and 4b)</p>
                        </list-item>
                    </list> </p>
                <p> Methods: 
                    <list list-type="order">
                        <list-item>
                            <p>Moved theoretical frameworks section to appropriate location</p>
                        </list-item>
                        <list-item>
                            <p>Eliminated phrase stating &#x201c;no sampling&#x201d; and revised text</p>
                        </list-item>
                    </list> Thank you for the individual editing suggestions all of which we implemented. We very much appreciate your time and assistance in improving our paper.</p>
            </body>
        </sub-article>
    </sub-article>
</article>
