<?xml version="1.0" encoding="UTF-8"?><!DOCTYPE article PUBLIC "-//NLM//DTD JATS (Z39.96) Journal Publishing DTD v1.2 20190208//EN" "http://jats.nlm.nih.gov/publishing/1.2/JATS-journalpublishing1.dtd"><article xmlns:mml="http://www.w3.org/1998/Math/MathML" xmlns:xlink="http://www.w3.org/1999/xlink" article-type="research-article" dtd-version="1.2" xml:lang="en">
    <front>
        <journal-meta>
            <journal-id journal-id-type="pmc">Gates Open Res</journal-id>
            <journal-title-group>
                <journal-title>Gates Open Research</journal-title>
            </journal-title-group>
            <issn pub-type="epub">2572-4754</issn>
            <publisher>
                <publisher-name>F1000 Research Limited</publisher-name>
                <publisher-loc>London, UK</publisher-loc>
            </publisher>
        </journal-meta>
        <article-meta>
            <article-id pub-id-type="doi">10.12688/gatesopenres.12877.1</article-id>
            <article-categories>
                <subj-group subj-group-type="heading">
                    <subject>Research Article</subject>
                </subj-group>
                <subj-group>
                    <subject>Articles</subject>
                </subj-group>
            </article-categories>
            <title-group>
                <article-title>Requirements elicitation for a blockchain vaccine supply chain management web/mobile application</article-title>
                <fn-group content-type="pub-status">
                    <fn>
                        <p>[version 1; peer review: 3 approved with reservations, 1 not approved]</p>
                    </fn>
                </fn-group>
            </title-group>
            <contrib-group>
                <contrib contrib-type="author" corresp="yes">
                    <name>
                        <surname>Mindila</surname>
                        <given-names>Agnes N.</given-names>
                    </name>
                    <role content-type="http://credit.niso.org/">Conceptualization</role>
                    <role content-type="http://credit.niso.org/">Data Curation</role>
                    <role content-type="http://credit.niso.org/">Formal Analysis</role>
                    <role content-type="http://credit.niso.org/">Funding Acquisition</role>
                    <role content-type="http://credit.niso.org/">Investigation</role>
                    <role content-type="http://credit.niso.org/">Methodology</role>
                    <role content-type="http://credit.niso.org/">Project Administration</role>
                    <role content-type="http://credit.niso.org/">Resources</role>
                    <role content-type="http://credit.niso.org/">Software</role>
                    <role content-type="http://credit.niso.org/">Supervision</role>
                    <role content-type="http://credit.niso.org/">Validation</role>
                    <role content-type="http://credit.niso.org/">Visualization</role>
                    <role content-type="http://credit.niso.org/">Writing &#x2013; Original Draft Preparation</role>
                    <role content-type="http://credit.niso.org/">Writing &#x2013; Review &amp; Editing</role>
                    <uri content-type="orcid">https://orcid.org/0000-0003-1807-5927</uri>
                    <xref ref-type="corresp" rid="c1">a</xref>
                    <xref ref-type="aff" rid="a1">1</xref>
                </contrib>
                <contrib contrib-type="author" corresp="yes">
                    <name>
                        <surname>Wafula</surname>
                        <given-names>Joseph M.</given-names>
                    </name>
                    <role content-type="http://credit.niso.org/">Conceptualization</role>
                    <role content-type="http://credit.niso.org/">Data Curation</role>
                    <role content-type="http://credit.niso.org/">Funding Acquisition</role>
                    <role content-type="http://credit.niso.org/">Investigation</role>
                    <role content-type="http://credit.niso.org/">Methodology</role>
                    <role content-type="http://credit.niso.org/">Project Administration</role>
                    <role content-type="http://credit.niso.org/">Software</role>
                    <role content-type="http://credit.niso.org/">Visualization</role>
                    <role content-type="http://credit.niso.org/">Writing &#x2013; Original Draft Preparation</role>
                    <role content-type="http://credit.niso.org/">Writing &#x2013; Review &amp; Editing</role>
                    <xref ref-type="corresp" rid="c2">b</xref>
                    <xref ref-type="aff" rid="a1">1</xref>
                </contrib>
                <contrib contrib-type="author" corresp="no">
                    <name>
                        <surname>Ratemo</surname>
                        <given-names>Harriet A. M.</given-names>
                    </name>
                    <role content-type="http://credit.niso.org/">Data Curation</role>
                    <role content-type="http://credit.niso.org/">Formal Analysis</role>
                    <role content-type="http://credit.niso.org/">Funding Acquisition</role>
                    <role content-type="http://credit.niso.org/">Investigation</role>
                    <role content-type="http://credit.niso.org/">Methodology</role>
                    <role content-type="http://credit.niso.org/">Project Administration</role>
                    <role content-type="http://credit.niso.org/">Visualization</role>
                    <role content-type="http://credit.niso.org/">Writing &#x2013; Original Draft Preparation</role>
                    <xref ref-type="aff" rid="a1">1</xref>
                </contrib>
                <contrib contrib-type="author" corresp="no">
                    <name>
                        <surname>Tabu</surname>
                        <given-names>Collins</given-names>
                    </name>
                    <role content-type="http://credit.niso.org/">Formal Analysis</role>
                    <role content-type="http://credit.niso.org/">Investigation</role>
                    <role content-type="http://credit.niso.org/">Methodology</role>
                    <role content-type="http://credit.niso.org/">Resources</role>
                    <role content-type="http://credit.niso.org/">Validation</role>
                    <xref ref-type="aff" rid="a2">2</xref>
                </contrib>
                <contrib contrib-type="author" corresp="no">
                    <name>
                        <surname>Charo</surname>
                        <given-names>Joyce</given-names>
                    </name>
                    <role content-type="http://credit.niso.org/">Formal Analysis</role>
                    <role content-type="http://credit.niso.org/">Investigation</role>
                    <role content-type="http://credit.niso.org/">Methodology</role>
                    <role content-type="http://credit.niso.org/">Resources</role>
                    <role content-type="http://credit.niso.org/">Validation</role>
                    <xref ref-type="aff" rid="a2">2</xref>
                </contrib>
                <contrib contrib-type="author" corresp="no">
                    <name>
                        <surname>Silali</surname>
                        <given-names>Catherine</given-names>
                    </name>
                    <role content-type="http://credit.niso.org/">Formal Analysis</role>
                    <role content-type="http://credit.niso.org/">Investigation</role>
                    <role content-type="http://credit.niso.org/">Methodology</role>
                    <role content-type="http://credit.niso.org/">Validation</role>
                    <xref ref-type="aff" rid="a2">2</xref>
                </contrib>
                <aff id="a1">
                    <label>1</label>Department of Computing, Jomo Kenyatta University of Agriculture and technology, Nairobi, Kenya, 00200, Kenya</aff>
                <aff id="a2">
                    <label>2</label>NVIP, Ministry of Health, Kenya, Nairobi, Kenya, Kenya</aff>
            </contrib-group>
            <author-notes>
                <corresp id="c1">
                    <label>a</label>
                    <email xlink:href="mailto:amindila@icsit.jkuat.ac.ke">amindila@icsit.jkuat.ac.ke</email>
                </corresp>
                <corresp id="c2">
                    <label>b</label>
                    <email xlink:href="mailto:muliaro@icsit.jkuat.ac.ke">muliaro@icsit.jkuat.ac.ke</email>
                </corresp>
                <fn fn-type="conflict">
                    <p>No competing interests were disclosed.</p>
                </fn>
            </author-notes>
            <pub-date pub-type="epub">
                <day>26</day>
                <month>4</month>
                <year>2019</year>
            </pub-date>
            <pub-date pub-type="collection">
                <year>2019</year>
            </pub-date>
            <volume>3</volume>
            <elocation-id>1420</elocation-id>
            <history>
                <date date-type="accepted">
                    <day>11</day>
                    <month>4</month>
                    <year>2019</year>
                </date>
            </history>
            <permissions>
                <copyright-statement>Copyright: &#x00a9; 2019 Mindila AN et al.</copyright-statement>
                <copyright-year>2019</copyright-year>
                <license xlink:href="https://creativecommons.org/licenses/by/4.0/">
                    <license-p>This is an open access article distributed under the terms of the Creative Commons Attribution Licence, which permits unrestricted use, distribution, and reproduction in any medium, provided the original work is properly cited.</license-p>
                </license>
            </permissions>
            <self-uri content-type="pdf" xlink:href="https://gatesopenresearch.org/articles/3-1420/pdf"/>
            <abstract>
                <p>

                    <bold>Background:</bold> This paper presents the processes carried out to be able to develop/gather the complete and right requirements to developing a secure and effective blockchain system for the vaccine supply chain. The paper hence presents the requirements elicitation activity of the Blockchain web/mobile application for vaccine supply chain.</p>
                <p>

                    <bold>Methods:</bold> A mixed-methods methodology was applied. The methods employed were; document review, survey, focus group workshops, interviews, observation, brainstorming, brainwriting Unified Modeling Language and system dynamics.</p>
                <p>

                    <bold>Results:</bold> The paper present results of each of the methods used in requirements elicitation for eight themes, namely: temperature monitoring; quality, suitability and capacity of transport facilities; information systems and supportive management functions; storage quality, suitability and capacity; maintenance of cold-chain equipment; vaccine distribution; vaccine management policies and stock management. The results presented gave understanding of the operation of the existing vaccine supply chain and the requirements for the blockchain web/mobile application for vaccine supply chain. </p>
                <p>

                    <bold>Conclusions:</bold> The requirements for the development of the desired vaccine supply chain web/mobile application were captured and documented.</p>
            </abstract>
            <kwd-group kwd-group-type="author">
                <kwd>Blockchain</kwd>
                <kwd>Vaccine Supply Chain</kwd>
                <kwd>Requirements Elicitation</kwd>
                <kwd>System dynamics</kwd>
            </kwd-group>
            <funding-group>
                <award-group id="fund-1" xlink:href="http://www.gatesfoundation.org">
                    <funding-source>Gates Foundation</funding-source>
                    <award-id>OPP1182320</award-id>
                </award-group>
                <funding-statement>This work is supported by a GCE Grant form the Gates Foundation (number OPP1182320).</funding-statement>
                <funding-statement>
                    <italic>The funders had no role in study design, data collection and analysis, decision to publish, or preparation of the manuscript.</italic>
                </funding-statement>
            </funding-group>
        </article-meta>
    </front>
    <body>
        <sec sec-type="intro">
            <title>Introduction</title>
            <p>Requirement elicitation is an important activity in the software requirement engineering phase of software development. If this activity is not given the importance it requires it becomes one of the reasons why a project could fail and not fulfill the customers&#x2019; needs. This phase ensures that requirements are complete and correct and that communication and association between the stakeholders for the success of the software requirement is achieved. When requirements are complete and correct, they guarantee that the developed application is a quality product and fulfills the user needs (
                <xref ref-type="bibr" rid="ref-9">Abbasi 
                    <italic toggle="yes">et al.</italic>, 2015</xref>).</p>
            <sec>
                <title>Requirements elicitation project</title>
                <p>The requirements elicitation reported in this paper concerns development of a blockchain vaccine supply chain management web/mobile software application, the aim of which is to provide end-to-end visibility of the vaccine supply chain. This project has been necessitated by the needs of low- and medium-income countries (LMICs), where the vaccines supply chain is faced with challenges, which leads to problems including expiration and/or lack of needed vaccines, wastages and stock-outs (
                    <xref ref-type="bibr" rid="ref-3">Yadav, 2015</xref>). Faced with shortage in resources and shrinking development partners support, LMICs need to employ strategies that are more efficient and effective. This will enable countries to achieve universal health and save many lives, especially those of children.</p>
                <p>This paper posits that one way of achieving this efficiency is by employing blockchain technology. Blockchain technology has attracted tremendous interest from a wide range of stakeholders, which include finance, healthcare, utilities, real estate and government agencies. This is due to its characteristics as a shared, distributed and fault-tolerant database/ledger that every participant in the network can share and where any single transaction is witnessed and verified by the network nodes. Information exchanged is resilient to manipulation. Blockchain&#x2019;s decentralized architecture can be leveraged to develop an assured data provenance capability for vaccine supply chain management (
                    <xref ref-type="bibr" rid="ref-5">Liang 
                        <italic toggle="yes">et al.</italic>, 2017</xref>). The project for which requirement elicitation is reported is based on permissioned blockchain, since the vaccine supply chain participants in healthcare are more likely to be altruistic and operate under real identities than are users of a highly anonymous, unregulated system like Bitcoin, which employs public blockchain. The application will be deployed in a private network owned by actors in the vaccine supply chain. Cryptographic data structure and a consensus algorithm will be employed. Self-executing and self-enforcing smart contracts are relevant in the vaccine supply chain permissioned blockchain because there will be need to verify correctness of transactions and enforcement of predefined rules.</p>
                <p>There are several requirement elicitation techniques which can be used to define the scope of a project and gather user requirement. 
                    <xref ref-type="bibr" rid="ref-9">Abbasi 
                        <italic toggle="yes">et al.</italic>, 2015</xref> categorizes them into four namely: classical traditional techniques (interviews, questionnaires and surveys), cognitive analytical techniques (card sorting, laddering and repertory grids), modern and group elicitation techniques (brainstorming, joint application development (JAD), prototyping) and social analysis (ethnography, direct observation, passive observation) (
                    <xref ref-type="bibr" rid="ref-9">Abbasi 
                        <italic toggle="yes">et al.</italic>, 2015</xref>).</p>
            </sec>
        </sec>
        <sec sec-type="methods">
            <title>Methods</title>
            <sec>
                <title>Requirements elicitation design</title>
                <p>The project adopted mixed methods (
                    <xref ref-type="bibr" rid="ref-7">Creswell, 2003</xref>), using different methods from the categories discussed by 
                    <xref ref-type="bibr" rid="ref-9">Abbasi 
                        <italic toggle="yes">et al.</italic> (2015)</xref>. This was based on the fact that no single requirements elicitation method is capable of giving all the software requirements, meaning it is important to use several methods (
                    <xref ref-type="bibr" rid="ref-8">Yousuf &amp; Asger, 2015</xref>).</p>
                <p>The requirements elicitation methods were:</p>
                <list list-type="bullet">
                    <list-item>
                        <label>i. </label>
                        <p>Reviewing of World Health Organization (WHO) documentations, International bodies e.g GAVI, vaccine supply chain management literature review and Effective Vaccine Management Criteria, Blockchain, Consensus models, Provenance models, Internet of Things (IoT) and radio frequency identification (RFID) and Sensor networks. Here literature review is treated as one of the methods of requirements elicitation.</p>
                    </list-item>
                    <list-item>
                        <label>ii. </label>
                        <p>LimeSurvey questionnaires designed.</p>
                    </list-item>
                    <list-item>
                        <label>iii. </label>
                        <p>Focus Group workshops with the National Vaccine and Immunization Program (NVIP ) department of the Ministry of Health (MoH)</p>
                    </list-item>
                    <list-item>
                        <label>iv. </label>
                        <p>Interview with Kenya Medical Supply Agency (KEMSA).</p>
                    </list-item>
                    <list-item>
                        <label>v. </label>
                        <p>A focus group workshop with the National Logistics Working Group.</p>
                    </list-item>
                    <list-item>
                        <label>vi. </label>
                        <p>Requirements elicitation workshop was organized with 30 participants from Mombasa, Migori and Nakuru counties. Besides the representation from the counties, we had representation from the NVIP department, and the Regional Store managers from Mombasa, Kisumu and Nakuru. The aim of the workshop was to understand the processes, the SOPs, the actors, the stakeholders and the rules that govern vaccine supply chain. This employed focus group method where brainwriting and brainstorming were used.</p>
                    </list-item>
                    <list-item>
                        <label>vii. </label>
                        <p>Site visit of Mombasa County. The places visited were, Regional store, Sub-County store, Health Facility, and County offices. Passive observation was employed as a method of requirements elicitation.</p>
                    </list-item>
                </list>
                <p>
                    <xref ref-type="table" rid="T1">Table 1</xref> shows the range of participants in all of the seven methods employed in the requirements elicitation activity.</p>
                <table-wrap id="T1" orientation="portrait" position="anchor">
                    <label>Table 1. </label>
                    <caption>
                        <title>Participants across all the methods used for requirements elicitation.</title>
                    </caption>
                    <table content-type="article-table" frame="hsides">
                        <thead>
                            <tr>
                                <th align="left" colspan="1" rowspan="1">Participants Affiliations</th>
                                <th align="left" colspan="1" rowspan="1">Roles</th>
                                <th align="left" colspan="1" rowspan="1">Actual Number</th>
                            </tr>
                        </thead>
                        <tbody>
                            <tr>
                                <td colspan="1" rowspan="1" valign="top">Mombasa County</td>
                                <td colspan="1" rowspan="1" valign="top">1.Sub-County EPI logistician
                                    <break/>2. Nurse-In-Charge of Health Facility
                                    <break/>3. County EPI logistician
                                    <break/>4.Regional Store Manager (KEMSA)</td>
                                <td colspan="1" rowspan="1" valign="top">4</td>
                            </tr>
                            <tr>
                                <td colspan="1" rowspan="1" valign="top">Migori County</td>
                                <td colspan="1" rowspan="1" valign="top">1.Sub-County EPI logistician
                                    <break/>2. Nurse-In-Charge of Health Facility
                                    <break/>3. County EPI logistician
                                    <break/>4. Regional Store Manager (KEMSA)</td>
                                <td colspan="1" rowspan="1" valign="top">4</td>
                            </tr>
                            <tr>
                                <td colspan="1" rowspan="1" valign="top">Nakuru County</td>
                                <td colspan="1" rowspan="1" valign="top">1.Sub-County EPI logistician
                                    <break/>2. Nurse-In-Charge of Health Facility
                                    <break/>3. County EPI logistician
                                    <break/>4. Regional Store Manager (KEMSA)</td>
                                <td colspan="1" rowspan="1" valign="top">4</td>
                            </tr>
                            <tr>
                                <td colspan="1" rowspan="1" valign="top">National Logistics Working Group</td>
                                <td colspan="1" rowspan="1" valign="top">NVIP
                                    <break/>JSI
                                    <break/>UNICEF
                                    <break/>CHAI</td>
                                <td colspan="1" rowspan="1" valign="top">12</td>
                            </tr>
                            <tr>
                                <td colspan="1" rowspan="1" valign="top">Nakuru</td>
                                <td colspan="1" rowspan="1" valign="top">Regional Store Manager</td>
                                <td colspan="1" rowspan="1" valign="top">1</td>
                            </tr>
                            <tr>
                                <td colspan="1" rowspan="1" valign="top">Mombasa</td>
                                <td colspan="1" rowspan="1" valign="top">Regional store Manger</td>
                                <td colspan="1" rowspan="1" valign="top">1</td>
                            </tr>
                            <tr>
                                <td colspan="1" rowspan="1" valign="top">Kisumu</td>
                                <td colspan="1" rowspan="1" valign="top">Regional Store Manager</td>
                                <td colspan="1" rowspan="1" valign="top">1</td>
                            </tr>
                            <tr>
                                <td colspan="1" rowspan="1" valign="top">NVIP</td>
                                <td colspan="1" rowspan="1" valign="top">National EPI Logistician
                                    <break/>National Store Manager
                                    <break/>National Store Assistant Managers
                                    <break/>National Medical Engineer</td>
                                <td colspan="1" rowspan="1" valign="top">6</td>
                            </tr>
                            <tr>
                                <td colspan="1" rowspan="1" valign="top">JKUAT</td>
                                <td colspan="1" rowspan="1" valign="top">Principal Investigator and Collaborators</td>
                                <td colspan="1" rowspan="1" valign="top">8</td>
                            </tr>
                            <tr>
                                <td colspan="1" rowspan="1" valign="top">KEMSA</td>
                                <td colspan="1" rowspan="1" valign="top">ICT Manager</td>
                                <td colspan="1" rowspan="1" valign="top">1</td>
                            </tr>
                            <tr>
                                <td colspan="1" rowspan="1" valign="top">Denominator</td>
                                <td colspan="1" rowspan="1" valign="top">Visiting team and fellow GCE
                                    <break/>Grantees of the Bill and Melinda Gates
                                    <break/>Foundation for level 4.</td>
                                <td colspan="1" rowspan="1" valign="top">2</td>
                            </tr>
                        </tbody>
                    </table>
                </table-wrap>
            </sec>
            <sec>
                <title>Requirements elicitation structural representation</title>
                <p>Stock and flow diagramming from the system dynamics field will be used to represent the analysis of the methods of requirements elicitation discussed. The goals, processes, rules and challenges, as described by the stakeholders in the vaccine supply chain, are represented in the stock and flow diagramming. Besides, this method is specifically important because vaccine supply chain management involves complex, dynamic decision-making tasks. The complexity is due to the environmental context, which is characterized by a number of variables to manage temporal spacing of consecutive decisions and availability of needed information. The second source of complexity is the structure of the underlying system. This systemic complexity is the result of interactions between system components in such a way that the system produces outputs that are not easily predictable (
                    <xref ref-type="bibr" rid="ref-1">&#x00d6;zg&#x00fc;n &amp; Barlas, 2015</xref>). This paper posits that by appreciating time delays, feedback and remaining sensitive to nonlinearities, vaccine supply chain management can be efficient and accurate, saving countries investment and enabling vaccines to be at the right place at the right time in correct quantities, hence saving more lives.</p>
                <p>The dynamic model of vaccine stock management is unique for several reasons: First, there should be no &#x201c;backlog of orders&#x201d; in this context as children need to get the vaccine in the right time. Second, in cases of temperature variations, the whole vaccine stock risks losing potency. Third, in cases of disease outbreaks, there is the possibility of whole-stock being consumed in a short time. Fourth, the final customers are varied and need to be reached even when they have not &#x201c;ordered&#x201d; through vaccine drives.</p>
                <p>The work builds upon the model proposed by 
                    <xref ref-type="bibr" rid="ref-2">Gon&#x00e7;alves (2003)</xref>, capturing a supply chain with a single supplier offering a unique, non-substitutable product to retailers (
                    <xref ref-type="bibr" rid="ref-4">Villa 
                        <italic toggle="yes">et al.</italic>, 2015</xref>). 
                    <xref ref-type="fig" rid="f1">Figure 1</xref> displays the simplest structure of our supply chain structure. The structure shows the supplier, who in most LMICs is the UNICEF Supply Division. The retailer represents the specific country and the final customers are the last mile health facilities, which is where vaccines are administered to children.</p>
                <fig fig-type="figure" id="f1" orientation="portrait" position="float">
                    <label>Figure 1. </label>
                    <caption>
                        <title>Vaccine supply chain in low- and middle-income countries.</title>
                    </caption>
                    <graphic orientation="portrait" position="float" xlink:href="https://gatesopenresearch-files.f1000.com/manuscripts/13968/fb3ecc6d-3180-4dfc-b437-b2e0f2be69e6_figure1.gif"/>
                </fig>
            </sec>
            <sec>
                <title>Data collection tools used for each method</title>
                <p>

                    <bold>

                        <italic toggle="yes">Literature review.</italic>
</bold> A.N., J.M. and H.A.M., from Jomo Kenyatta University of Agriculture and Technology, held weekly workshops over a span of 3 months. Prior to every workshop every team member was tasked with carrying out research on the topic scheduled to be next discussed. After the specified period of time members were tasked with writing white papers on the topics. The topics that were discussed and white papers written concern:</p>
                <list list-type="bullet">
                    <list-item>
                        <label>1. </label>
                        <p>International aspects of vaccines</p>
                    </list-item>
                    <list-item>
                        <label>2. </label>
                        <p>Use of provenance in the blockchain</p>
                    </list-item>
                    <list-item>
                        <label>3. </label>
                        <p>Use of RFID and blockchain in cold chain</p>
                    </list-item>
                    <list-item>
                        <label>4. </label>
                        <p>Blockchain security</p>
                    </list-item>
                    <list-item>
                        <label>5. </label>
                        <p>Blockchain</p>
                    </list-item>
                    <list-item>
                        <label>6. </label>
                        <p>Internet of things (IoT) for blockchain network design</p>
                    </list-item>
                    <list-item>
                        <label>7. </label>
                        <p>Vaccine supply chain</p>
                    </list-item>
                </list>
                <p>All white papers can be accessed on 
                    <ext-link ext-link-type="uri" xlink:href="https://doi.org/10.17605/OSF.IO/PTUAG">OSF</ext-link> (
                    <xref ref-type="bibr" rid="ref-18">Mindila, 2018</xref>).</p>
                <p>

                    <bold>

                        <italic toggle="yes">LimeSurvey questionnaire design.</italic>
</bold> After completion of phase 1, A.N.M, J.M.W and H.A.M designed questions in 
                    <ext-link ext-link-type="uri" xlink:href="C:\Users\Amindila\Downloads\v">LimeSurvey</ext-link> software based on part of the literature review in phase two and the World Health Organization (WHO) Effective Vaccine Management (EVM) criteria (
                    <xref ref-type="bibr" rid="ref-17">WHO, 2018</xref>). The questionnaires were sent to the NVIP department staff for verification before administration. The questionnaires were administered to the 30 members who later also attended the Requirements elicitation workshop. The list of the questionnaire recipients can be accessed on 
                    <ext-link ext-link-type="uri" xlink:href="https://doi.org/10.17605/OSF.IO/PTUAG">OSF</ext-link> (
                    <xref ref-type="bibr" rid="ref-18">Mindila, 2018</xref>). The LimeSurvey questionnaire design can be accessed on 
                    <ext-link ext-link-type="uri" xlink:href="https://doi.org/10.17605/OSF.IO/PTUAG">OSF</ext-link> (
                    <xref ref-type="bibr" rid="ref-18">Mindila, 2018</xref>).</p>
                <p>The themes of the survey were:</p>
                <list list-type="bullet">
                    <list-item>
                        <label>1. </label>
                        <p>Policy and Regulations: MoH Kenya</p>
                    </list-item>
                    <list-item>
                        <label>2. </label>
                        <p>Policy and Regulations from partners perspective</p>
                    </list-item>
                    <list-item>
                        <label>3. </label>
                        <p>Service delivery level vaccine stock management</p>
                    </list-item>
                    <list-item>
                        <label>4. </label>
                        <p>Sub county level Vaccines Stock Management</p>
                    </list-item>
                    <list-item>
                        <label>5. </label>
                        <p>National/Primary level vaccine stock management</p>
                    </list-item>
                    <list-item>
                        <label>6. </label>
                        <p>Maintenance</p>
                    </list-item>
                    <list-item>
                        <label>7. </label>
                        <p>Vaccine management policies</p>
                    </list-item>
                    <list-item>
                        <label>8. </label>
                        <p>Information systems and supportive management functions</p>
                    </list-item>
                    <list-item>
                        <label>9. </label>
                        <p>National logistics working group/National Vaccines and Immunization programme</p>
                    </list-item>
                    <list-item>
                        <label>10. </label>
                        <p>Storage and Transport Capacity</p>
                    </list-item>
                    <list-item>
                        <label>11. </label>
                        <p>Vaccines distribution</p>
                    </list-item>
                    <list-item>
                        <label>12. </label>
                        <p>Quality and suitability of building, equipment and transport facilities</p>
                    </list-item>
                    <list-item>
                        <label>13. </label>
                        <p>Temperature monitoring</p>
                    </list-item>
                </list>
                <p>

                    <bold>

                        <italic toggle="yes">Focus group workshops with the NVIP department of the Ministry of Health (MoH).</italic>
</bold> J.M.W and the NVIP head C.T. held a meeting on 8
                    <sup>th</sup> March 2018 at the NVIP offices where J.M.W introduced the project and obtained verbal permission from C.T., the NVIP head, to collaborate with a promise to issue a formal consent letter. NVIP head C.T accepted to be the project owner. Consent for the National Vaccine and Immunization Program (NVIP) focus group was obtained verbally from C.T., who also gave permission to the concerned staff to attend the focus group meetings. The participants were informed and invited by the head of NVIP. Consent was not sought from participants as these was considered low-risk interactions</p>
                <p>The NVIP head C.T. invited A.N.M, J.M.W and H.A.M to present the project concept to the National Logistics Working Group meeting on 12
                    <sup>th</sup> March 2018 at the NVIP offices at DFH conference room and gathered their views.</p>
                <p>
                    <underline>
                        <italic toggle="yes">Focus group workshop 1</italic>
                    </underline>
                </p>
                <p> A.N.M, J.M.W and H.A.M held a meeting on 25
                    <sup>th</sup> April 2018 with the NVIP department at NVIP offices Division of Family Health and introduced the Blockchain vaccine supply chain mobile/web application project. The minutes of what was discussed can be accessed on 
                    <ext-link ext-link-type="uri" xlink:href="https://doi.org/10.17605/OSF.IO/PTUAG">OSF</ext-link> (
                    <xref ref-type="bibr" rid="ref-18">Mindila, 2018</xref>).</p>
                <p>
                    <underline>
                        <italic toggle="yes">Focus group workshop 2</italic>
                    </underline>
                </p>
                <p>A.N.M, J.M.W and H.A.M, C.S and J.C held a 1-day follow-up workshop on 26th April 2018 at DFH offices, where a team from NVIP was selected to work together with the project team. The project team shared the project timeline, a concept paper so as to enable the head of NVIP C.T. to obtain written consent. The project outline shared with NVIP can be accessed on 
                    <ext-link ext-link-type="uri" xlink:href="https://doi.org/10.17605/OSF.IO/PTUAG">OSF</ext-link> (
                    <xref ref-type="bibr" rid="ref-18">Mindila, 2018</xref>).</p>
                <p>
                    <underline>
                        <italic toggle="yes">Focus group workshop 3</italic>
                    </underline>
                </p>
                <p>A.N.M, J.M.W and H.A.M. and the NVIP team held a one-day brainstorming session on 27th April 2018 that was guided by the LimeSurvey questionnaires designed in phase 2.</p>
                <p>
                    <underline>
                        <italic toggle="yes">Focus group workshop 4</italic>
                    </underline>
                </p>
                <p>A.N, J.M and H.A.M and Joyce Charo of NVIP held a 1-day workshop on 17
                    <sup>th</sup> May 2018 where the survey structure was designed.</p>
                <p>
                    <bold>
                        <italic toggle="yes">Discussion with KEMSA.</italic>
                    </bold> The Project team had an introductory meeting with the CEO of KEMSA Mr. Fredrick Wanyonyi at the KEMSA headquarters on 19
                    <sup>th</sup> February 2018 where the team introduced the intention of the visit and the project. The CEO was receptive and linked us to the ICT manager, with whom we later organized for a meeting and thereafter held an interview with the ICT manager of KEMSA The communication that followed between J.M.W and Mr. Samuel Wataku can be accessed on 
                    <ext-link ext-link-type="uri" xlink:href="https://doi.org/10.17605/OSF.IO/PTUAG">OSF</ext-link> (
                    <xref ref-type="bibr" rid="ref-18">Mindila, 2018</xref>) as meeting with Prof. Wafula. The outcome of the meeting yielded collaboration themes which are captured as KEMSA CEO letter which can be accessed on 
                    <ext-link ext-link-type="uri" xlink:href="https://doi.org/10.17605/OSF.IO/PTUAG">OSF</ext-link> (
                    <xref ref-type="bibr" rid="ref-18">Mindila, 2018</xref>).</p>
                <p>

                    <bold>

                        <italic toggle="yes">Focus group workshop with the National Logistics Working Group.</italic>
</bold> The project team introduced the project to the National Logistics Working Group (NLWG), whose members were UNICEF, NVIP, John Snow, Inc. (JSI) and Clinton Health Access Initiative (CHAI) in a 1-day sensitization meeting made possible by the NVIP head. The presentation made by A.N.M. can be accessed on 
                    <ext-link ext-link-type="uri" xlink:href="https://doi.org/10.17605/OSF.IO/PTUAG">OSF</ext-link> (
                    <xref ref-type="bibr" rid="ref-18">Mindila, 2018</xref>).</p>
                <p>

                    <bold>

                        <italic toggle="yes">Requirements elicitation workshop.</italic>
</bold> The requirements elicitation workshop was held in Mombasa and involved 30 participants. The participants were drawn from Mombasa county, Migori county and Nakuru county. Besides the representation from the counties, there was representation from the NVIP department and the Regional Store managers from Mombasa, Kisumu and Nakuru and KEMSA. The project team were the facilitators. The aim of the workshop was to understand the processes, the SOPs, the actors, the stakeholders and the rules that govern vaccine supply chain. The workshop was guided by LimeSurvey questionnaires that were developed in phase 2.</p>
                <p>The workshop adopted brainwriting or card sorting (
                    <xref ref-type="bibr" rid="ref-9">Abbasi 
                        <italic toggle="yes">et al.</italic>, 2015</xref>) and brainstorming as the techniques for elicitation. It was guided by eight themes that had been prepared by the project team. A total of 12 participants from the three counties of Migori, Mombasa and Nakuru, whose positions were County EPIs, Sub-county EPIs, nurses in charge and Data clerks, took part in the workshop. There were three regional store managers drawn from Nakuru, Mombasa and Kisumu. There were six NVIP participants and eight project team members.</p>
                <p>Four colors were selected and each color represented two themes. Every participant was assigned one of the selected colors randomly. Each participant was issued eight cards comprising of two green, two pink, two blue and two yellow in color to write responses to questions posed per color coded theme. The themes were coded as follows: Blue-Information system and supportive functions; Blue, Vaccine Management policies; Pink, Maintenance of Cold Chain Equipment; Pink, Temperature Monitoring; Yellow, Stock Management; Yellow, Storage; Green, Vaccine Distribution; Green, Transport.</p>
                <p>In this workshop, a presenter prompted questions highlighting the important information needed on different themes. All participants then filled in their responses on cards provided. After posing four questions covering the first four themes, the participants broke out into 4 groups according to the color they were assigned randomly to discuss cards with color that match that of the group They would then discuss their responses and pin up a summary of answers generated. Anyone from the different groups added more answers on the pinned summary during rotation period. This process was repeated for the four remaining themes. After the group discussions, a plenary session was held where summaries and way forward were presented by the panelists.</p>
                <p>The eight themes were as structured below:</p>
                <p>
                    <underline>
                        <italic toggle="yes">Temperature monitoring</italic>
                    </underline>
                </p>
                <p>I am responsible for tracking vaccine temperature and at any one point in time I need to know the temperatures under which all vaccines are kept whether in stores or in transit. How may you make sure that you support me and what would be the challenges? How do we address them?</p>
                <p>
                    <underline>
                        <italic toggle="yes">Quality, suitability and capacity of transport facilities</italic>
                    </underline>
                </p>
                <p>I am the officer in charge of vaccines transportation in the country. My duty is to guarantee quality and efficient transport services in the vaccines supply chain. What challenges may I face in ensuring vehicles deliver vaccines in good/favorable conditions within the appropriate timeframe? Also, what can you comment on the communication channels accompanying the vaccines transport services.</p>
                <p>
                    <underline>
                        <italic toggle="yes">Information systems and supportive management functions</italic>
                    </underline>
                </p>
                <p>What tools and systems are used for vaccines data management? Are they sufficient in providing data for decision making?</p>
                <p>
                    <underline>
                        <italic toggle="yes">Storage quality, suitability and capacity</italic>
                    </underline>
                </p>
                <p>There are various requirements for the various vaccines in terms of storage, how do I ensure that these requirements are not violated? If I walked into a particular vaccine store, how easy would it for me to locate a particular vaccine? What informs which vaccines I release upon requisition? Is the current storage sufficient, any contingency plans? Are there any security threats to vaccine storage?</p>
                <p>
                    <underline>
                        <italic toggle="yes">Maintenance of cold chain equipment (CCE)</italic>
                    </underline>
                </p>
                <p>What important information and support would I require to provide effective maintenance of the cold chain equipment? Given that I would like to know the state of all CCE under my jurisdiction in terms of temperature and how much the equipment is holding and be able to respond when am called upon.</p>
                <p>
                    <underline>
                        <italic toggle="yes">Vaccine distribution</italic>
                    </underline>
                </p>
                <p>I am the national/regional/sub-county vaccine distribution officer, what important information and support do I require to undertake effective vaccine delivery?</p>
                <p>
                    <underline>
                        <italic toggle="yes">Vaccine management policies</italic>
                    </underline>
                </p>
                <p>What information do I require for effective vaccine management? </p>
                <p>
                    <underline>
                        <italic toggle="yes">Stock management</italic>
                    </underline>
                </p>
                <p>I need to order the next stock of vaccines, where do I direct my requests to at the various levels right from the health facilities. Can I tell on the fly about various stock levels? What decisions inform the stock ordering policy? How do I know what amounts of stock to order (at the different levels)? How do I mitigate stock-outs and/or reorder delays? How do I know when orders arrive? How do I handle/prevent losses?</p>
            </sec>
            <sec>
                <title>Site visit to Mombasa county</title>
                <p>The visit was to the EPI logistician office, Mombasa subcounty store and Mombasa Regional Store on the 30
                    <sup>th</sup> of May 2018; it involved observations of how the vaccines were stored and labelled, how temperature monitoring was done, and how records were kept and reporting on vaccine movement was conducted. The staff were able to explain the vaccine supply chain processes and some of the challenges they face. Discussions were mostly oral, and pictorial images of what happens at these stores were captured. In a nutshell, the fact finding involved both observation and discussions.</p>
            </sec>
        </sec>
        <sec>
            <title>Ethical approval</title>
            <p>The research was not clinical and did not require human or animal samples the ethical committee approval was not required as advised by MoH. The NVIP head described it as a Systematic research that seeks the &#x201c;How&#x201d; rather than the &#x201c;Whom&#x201d;, and it was a study about how the system works and so did not require ethical approval. For this reason, and due to the low level of risk inherent in this research, formal consent was not sought from participants.</p>
        </sec>
        <sec sec-type="results">
            <title>Results</title>
            <sec>
                <title>Literature review insights</title>
                <p>The literature review method resulted in the output of white papers on different topics. The subheadings below contain important excerpts from the literature review. The full papers can be found on 
                    <ext-link ext-link-type="uri" xlink:href="https://doi.org/10.17605/OSF.IO/PTUAG">OSF</ext-link> (
                    <xref ref-type="bibr" rid="ref-18">Mindila, 2018</xref>).</p>
                <p>

                    <bold>

                        <italic toggle="yes">International aspects of vaccines.</italic>
</bold> The Global Vaccine Action Plan (GVAP) is a framework approved by the World Health Assembly in May 2012 to achieve the Decade of Vaccines vision by delivering universal access to immunization (
                    <xref ref-type="bibr" rid="ref-60">World Health Organization, 2013</xref>). African states have embraced immunization as a cornerstone for Sustainable Development Goals (SDGs) and AU agenda 2063 (Addis Declaration on Immunization, ADI 2017); SDGs call for 
                    <italic toggle="yes">&#x201c;&#x2026;access to safe, effective, quality and affordable medicines and vaccines for all by 2030</italic>&#x201d;. ADI Strategy 2 states the need to &#x201c;Address gaps in immunization and work with key partners to overcome barriers to access and utilization of immunization services&#x201d;. Monitoring national vaccine coverage is key in understanding the execution of immunization programs so as to identify areas within immunization systems needing improvements, and prepare for the introduction of new vaccines (
                    <xref ref-type="bibr" rid="ref-61">World Health Organization, 2016</xref>).With the introduction of new vaccines, developing countries are facing serious challenges in their vaccine supply and logistics systems, especially in resource-constrained environments.</p>
                <p>

                    <bold>

                        <italic toggle="yes">Use of provenance in the blockchain.</italic>
</bold> Provenance provides the history of the origins of all modifications to an entity, agents that have modified the entity through processing it or viewing it, and the particular processes that have altered the entity. (
                    <xref ref-type="bibr" rid="ref-5">Liang 
                        <italic toggle="yes">et al.</italic>, 2017</xref>) Using provenance for instance, one can be able to track pharmaceuticals from the manufacturing laboratory as they move all the way to the end consumer (
                    <xref ref-type="bibr" rid="ref-6">Hasan 
                        <italic toggle="yes">et al.</italic>, 2009</xref>). In the implementation of the blockchain-based solution, the PROV Data Model, as specified under the PROV Standard released by W3C Provenance Working Group in April 2013, will be incorporated (
                    <xref ref-type="bibr" rid="ref-22">Moreau &amp; Missier, 2013</xref>). The data model is composed of an agent, an entity and an activity at the core. It can store information about data, how it was modified, who modified it and when it was modified as well as the locations it has passed through. Therefore, incorporating it into the proposed solution will ensure that every vaccine in the supply chain can be isolated, analyzed, and all the activities associated with it identified hence ensuring full transparency and accountability in the supply chain.</p>
                <p>

                    <bold>

                        <italic toggle="yes">Use of RFID and blockchain in cold chain.</italic>
</bold> To improve traceability and monitoring of vaccines, use of RFID and blockchain technologies is recommended. RFID tags attached to vaccines will be read at pre-determined intervals by RFID readers fitted in transit vehicles and in storage facilities. The data read will be channeled to the blockchain. Small modifications can be done to the RFID tags such that they include some sensors, e.g. temperature sensors. An RFID tag with a sensor can now be aware of the temperature of its environment and can return such data once interrogated by a RFID reader. (
                    <xref ref-type="bibr" rid="ref-11">Ko 
                        <italic toggle="yes">et al.</italic>, 2016</xref>). Beaconing RFID tags are consistently sending information out at predetermined intervals. These tags can be pre-programmed to send data following a pre-programmed schedule. Depending on application at hand, different tagging levels may be selected they include: Pallet level tagging, Case level tagging and Item level tagging (
                    <xref ref-type="bibr" rid="ref-10">Gaukler, 2011</xref>). Item level tagging combined with smart shelves can help prevent stock outs.</p>
                <p>

                    <bold>

                        <italic toggle="yes">Blockchain.</italic>
</bold> In a blockchain transaction, two cryptographic primitives are used to prevent malicious users breaking the system. A digital signature is used to make sure that the information is signed by the claimed person as well as to test whether the information is modified by some malicious people. The signature process contains signature generation and signature verification. Given a message, the signatory generates a signature by using his private key, and the verifier can use signatory&#x2019;s public key to verify the messages authenticity. Instead of signing on the message directly, a cryptographic hash function is applied to the original message to produce a message digest for performance reason.</p>
                <p>Merkle tree is an important element in blockchain security. It is a binary tree of hashes as proposed by Ralph Merkle (
                    <xref ref-type="bibr" rid="ref-13">Merkle, 1980</xref>; 
                    <xref ref-type="bibr" rid="ref-12">Merkle, 1988</xref>) that is used to verify data integrity efficiently and securely. One of the strengths of Merkle tree is that there is no need to recompute the hash of all data if one data block changes.</p>
            </sec>
            <sec>
                <title>IoT for blockchain network design</title>
                <p>A network model based on multi-layer distributed accounting can be regarded as an organic combination of the centric network and decentralized network, which not only effectively utilizes the performance and capabilities of the cloud server, but also significantly improves the overall security and reliability of the IoT with taking advantages of blockchain techniques. Such a network is made up of entities and processes. The entities in an IoT environment are objects. This include equipment, facilities, users, APIs and other abstract things, so long as they have networking capabilities. These nodes have capabilities like data transmission, analysis and storage, edge layers that are made up of a certain number of objects with a central node that manages them and is able to interface to a superior layer. The node is self-independent in that it only communicates to the central node in order to achieve optimization. This can however be modified depending on the scenario and performance needs. A high level layer is a network entity consisting of a number of nodes that employ the blockchain protocol (
                    <xref ref-type="bibr" rid="ref-16">Li &amp; Zhang, 2017</xref>)</p>
                <p>

                    <bold>

                        <italic toggle="yes">Vaccine supply chain.</italic>
</bold> Well-functioning supply chains, delivering medicines, vaccines and other health products, are critical to the provision of health services. Most governments in LMICs, especially in sub-Saharan Africa, choose a distribution model where the government procures drugs and distributes them to health clinics using a publicly run central medical store (CMS) and a government-owned transport fieet. There is considerable heterogeneity in the governance structure of the CMS in different countries. In some cases, products funded by some bilateral agencies are procured directly by the international agency or a procurement agent acting on its behalf. Once medicines arrive in the country, the most common structure for distribution involves warehousing and storage at the CMS. Depending upon the geography, administrative structure and the number of health facilities in a country, products are first distributed from the CMS to regional and district medical stores that supply hospitals, health centers and community health workers (
                    <xref ref-type="bibr" rid="ref-3">Yadav, 2015</xref>).</p>
                <p>In LMICs, information about demand, stock levels and timely use of vaccines is poorly kept, affecting timely supply leading to expirees and/or lack of needed vaccines (
                    <xref ref-type="bibr" rid="ref-20">WHO 
                        <italic toggle="yes">et al.</italic>, 2010</xref>; 
                    <xref ref-type="bibr" rid="ref-19">WHO, 2014</xref>).</p>
                <p>There is also the risk of poor product quality and counterfeiting that LMICs face. Remedy would be to help manage this information along the supply chain by enabling visibility and transparency of information from the start to end of the supply chain for all authorized persons. This requires accurate data collection and secure data storage to enable a flow of trusted information between parties and to enable forecasting and any other analytics required (
                    <xref ref-type="bibr" rid="ref-20">WHO 
                        <italic toggle="yes">et al.</italic>, 2010</xref>).</p>
            </sec>
            <sec>
                <title>Results of Focus group workshops 1 and 2</title>
                <p>A.N.M, J.M.W and H.A.M held a meeting on 25
                    <sup>th</sup> April 2018 with the NVIP department at NVIP offices Division of Family Health and introduced the Blockchain vaccine supply chain mobile/web application project. The minutes from this meeting and the project outline shared with NVIP can be accessed on 
                    <ext-link ext-link-type="uri" xlink:href="https://doi.org/10.17605/OSF.IO/PTUAG">OSF</ext-link> (
                    <xref ref-type="bibr" rid="ref-18">Mindila, 2018</xref>).</p>
            </sec>
            <sec>
                <title>Results of focus group workshop 3 with the NVIP department of the MoH</title>
                <p>
                    <xref ref-type="fig" rid="f2">Figure 2</xref> represents the ecosystem of policies and regulations that guide vaccine management in Kenya. Policies developed by WHO which are applied globally, and GVAP and Decade of Vaccines 2010&#x2013;2020 objectives and goals are aligned to reduce morbidity and mortality and address availability of potent vaccines for children. Kenya&#x2019;s policies and plans are aligned to the global objectives and plans.</p>
                <fig fig-type="figure" id="f2" orientation="portrait" position="float">
                    <label>Figure 2. </label>
                    <caption>
                        <title>Institutions and Bodies involved directly with NVIP.</title>
                    </caption>
                    <graphic orientation="portrait" position="float" xlink:href="https://gatesopenresearch-files.f1000.com/manuscripts/13968/fb3ecc6d-3180-4dfc-b437-b2e0f2be69e6_figure2.gif"/>
                </fig>
                <p>
                    <xref ref-type="fig" rid="f3">Figure 3</xref> shows the organizations identified by NVIP as those that offer different forms of support, ranging from technical, financial and infrastructural support. </p>
                <fig fig-type="figure" id="f3" orientation="portrait" position="float">
                    <label>Figure 3. </label>
                    <caption>
                        <title>Policies and regulations guiding vaccine management in Kenya.</title>
                    </caption>
                    <graphic orientation="portrait" position="float" xlink:href="https://gatesopenresearch-files.f1000.com/manuscripts/13968/fb3ecc6d-3180-4dfc-b437-b2e0f2be69e6_figure3.gif"/>
                </fig>
                <p>The brainstorming workshop 3 with the NVIP team provided insight into the processes of estimation of needs, procurement and arrival of vaccines. This was captured in the form of UML diagrams enabled by draw.io software and are represented in 
                    <xref ref-type="fig" rid="f4">Figure 4</xref>&#x2013;
                    <xref ref-type="fig" rid="f8">Figure 8</xref>.</p>
                <fig fig-type="figure" id="f4" orientation="portrait" position="float">
                    <label>Figure 4. </label>
                    <caption>
                        <title>Partner organizations that offer NVIP support.</title>
                    </caption>
                    <graphic orientation="portrait" position="float" xlink:href="https://gatesopenresearch-files.f1000.com/manuscripts/13968/fb3ecc6d-3180-4dfc-b437-b2e0f2be69e6_figure4.gif"/>
                </fig>
                <fig fig-type="figure" id="f5" orientation="portrait" position="float">
                    <label>Figure 5. </label>
                    <caption>
                        <title>Participating parties in forecasting of vaccine needs.</title>
                    </caption>
                    <graphic orientation="portrait" position="float" xlink:href="https://gatesopenresearch-files.f1000.com/manuscripts/13968/fb3ecc6d-3180-4dfc-b437-b2e0f2be69e6_figure5.gif"/>
                </fig>
                <fig fig-type="figure" id="f6" orientation="portrait" position="float">
                    <label>Figure 6. </label>
                    <caption>
                        <title>Process involved in the estimation of vaccine needs.</title>
                    </caption>
                    <graphic orientation="portrait" position="float" xlink:href="https://gatesopenresearch-files.f1000.com/manuscripts/13968/fb3ecc6d-3180-4dfc-b437-b2e0f2be69e6_figure6.gif"/>
                </fig>
                <fig fig-type="figure" id="f7" orientation="portrait" position="float">
                    <label>Figure 7. </label>
                    <caption>
                        <title>Participants in the clearance and arrival process.</title>
                    </caption>
                    <graphic orientation="portrait" position="float" xlink:href="https://gatesopenresearch-files.f1000.com/manuscripts/13968/fb3ecc6d-3180-4dfc-b437-b2e0f2be69e6_figure7.gif"/>
                </fig>
                <fig fig-type="figure" id="f8" orientation="portrait" position="float">
                    <label>Figure 8. </label>
                    <caption>
                        <title>Procurement process.</title>
                    </caption>
                    <graphic orientation="portrait" position="float" xlink:href="https://gatesopenresearch-files.f1000.com/manuscripts/13968/fb3ecc6d-3180-4dfc-b437-b2e0f2be69e6_figure8.gif"/>
                </fig>
                <p>
                    <xref ref-type="fig" rid="f4">Figure 4</xref> shows the participants in the function of estimation of needs. They are health workers, health managers, depot managers and national logistician.</p>
                <p>Estimation of vaccine needs happens at four levels, as illustrated in 
                    <xref ref-type="fig" rid="f5">Figure 5</xref>: at the health facility, at the sub-county stores, at the regional store and finally at the primary store. At health facilities the people involved are the health workers; at the sub-county and regional stores, health workers at managerial level and store managers are involved; and the national logistician is found at the national level.</p>
                <p>
                    <xref ref-type="fig" rid="f6">Figure 6</xref> captures the process of estimation of vaccine needs as explained by the NVIP team. It is a sequential process that moves from right to left and time of events moves from top to bottom. Parameters considered include the target population, the minimum and maximum stock, stock available and amount of vaccine received last for each of the antigens. These parameters are then used to fill a vaccine order sheet. This process happens at all the four levels, every lower level propagates their order sheet to the higher level until the national level receives.</p>
                <p>
                    <xref ref-type="fig" rid="f7">Figure 7</xref> represents the participants in the procurement and arrival process. The process has eight participants, namely: UNICEF country office, NVIP head office, UNICEF Supply Division, Ministry of Health, National Regulatory Authority, Clearing Agent, Clearing Agent warehouse and the Cargo Handling centre. 
                    <xref ref-type="fig" rid="f8">Figure 8</xref> represents the procurement process while 
                    <xref ref-type="fig" rid="f9">Figure 9</xref> represents the processes involved in the arrival process as explained by the NVIP team.</p>
                <fig fig-type="figure" id="f9" orientation="portrait" position="float">
                    <label>Figure 9. </label>
                    <caption>
                        <title>Clearance and arrival process.</title>
                    </caption>
                    <graphic orientation="portrait" position="float" xlink:href="https://gatesopenresearch-files.f1000.com/manuscripts/13968/fb3ecc6d-3180-4dfc-b437-b2e0f2be69e6_figure9.gif"/>
                </fig>
                <p>The process is a sequential flow from left to right and in terms of time the events happen from top to bottom. Four parameters stand out as crucial to the process, this are, arrival dates, five pre-shipment documents, NRA permit and Tax Exemption. This four parameters facilitate the arrival and clearance process.</p>
                <p>Lastly the NVIP team provided our team with insights into the final part of the arrival process that involves vaccine inspection and filling of the Vaccine Arrival Report (VAR). 
                    <xref ref-type="fig" rid="f10">Figure 10</xref> and 
                    <xref ref-type="fig" rid="f11">Figure 11</xref> show the participants and the process respectively.</p>
                <fig fig-type="figure" id="f10" orientation="portrait" position="float">
                    <label>Figure 10. </label>
                    <caption>
                        <title>Vaccine arrival inspection and VAR generation participants.</title>
                    </caption>
                    <graphic orientation="portrait" position="float" xlink:href="https://gatesopenresearch-files.f1000.com/manuscripts/13968/fb3ecc6d-3180-4dfc-b437-b2e0f2be69e6_figure10.gif"/>
                </fig>
                <fig fig-type="figure" id="f11" orientation="portrait" position="float">
                    <label>Figure 11. </label>
                    <caption>
                        <title>Vaccine inspection and VAR completion process.</title>
                    </caption>
                    <graphic orientation="portrait" position="float" xlink:href="https://gatesopenresearch-files.f1000.com/manuscripts/13968/fb3ecc6d-3180-4dfc-b437-b2e0f2be69e6_figure11.gif"/>
                </fig>
            </sec>
            <sec>
                <title>Discussion with KEMSA ICT department</title>
                <p>The discussion with KEMSA gave us insight into the support they offer to NVIP. KEMSA offer infrastructural support to NVIP by way of providing cold chain storage. We further discussed with KEMSA on possibilities of hosting the blockchain application on their servers and we discussed possibilities of integrating the blockchain application with their systems.</p>
            </sec>
            <sec>
                <title>Focus group workshop with the National Logistics Working Group</title>
                <p>The National Logistics working Group workshop provided a forum for us to create awareness of the blockchain solution for vaccine supply chain and the potential it has of enabling visibility, traceability and transparency.</p>
            </sec>
            <sec>
                <title>Requirements elicitation workshop and site visit</title>
                <p>The requirements elicitation workshop and the site visit enabled the research team to understand the process involved in the vaccine supply chain. The integration of the literature review already carried out, the workshop, the focus group meetings and the site visit enabled the team to capture the requirements as discussed in the figures below. The raw data of the requirements elicitation workshop and the data coding done by the research team as editable word tables and excel sheets are provided on 
                    <ext-link ext-link-type="uri" xlink:href="https://doi.org/10.17605/OSF.IO/PTUAG">OSF</ext-link> (
                    <xref ref-type="bibr" rid="ref-18">Mindila, 2018</xref>). They represent all the eight themes discussed in the Methods.</p>
                <p>The requirements of the eight themes discussed and whose survey results are also provided on 
                    <ext-link ext-link-type="uri" xlink:href="https://doi.org/10.17605/OSF.IO/PTUAG">OSF</ext-link> (
                    <xref ref-type="bibr" rid="ref-18">Mindila, 2018</xref>), which also covers the literature review themes discussed are captured in 
                    <xref ref-type="fig" rid="f12">Figure 12</xref>&#x2013;
                    <xref ref-type="fig" rid="f20">Figure 20</xref>.</p>
                <fig fig-type="figure" id="f12" orientation="portrait" position="float">
                    <label>Figure 12. </label>
                    <caption>
                        <title>Use case for capture of unique vaccine digital identifier and capture of temperature readings in storage and in transit.</title>
                    </caption>
                    <graphic orientation="portrait" position="float" xlink:href="https://gatesopenresearch-files.f1000.com/manuscripts/13968/fb3ecc6d-3180-4dfc-b437-b2e0f2be69e6_figure12.gif"/>
                </fig>
                <fig fig-type="figure" id="f13" orientation="portrait" position="float">
                    <label>Figure 13. </label>
                    <caption>
                        <title>Authorized personnel activity diagram on digital recording of wastages or good vaccines at various levels in the vaccine supply chain.</title>
                    </caption>
                    <graphic orientation="portrait" position="float" xlink:href="https://gatesopenresearch-files.f1000.com/manuscripts/13968/fb3ecc6d-3180-4dfc-b437-b2e0f2be69e6_figure13.gif"/>
                </fig>
                <fig fig-type="figure" id="f14" orientation="portrait" position="float">
                    <label>Figure 14. </label>
                    <caption>
                        <title>Vaccine Detail QR capture, location capture and temperature reading capture process.</title>
                    </caption>
                    <graphic orientation="portrait" position="float" xlink:href="https://gatesopenresearch-files.f1000.com/manuscripts/13968/fb3ecc6d-3180-4dfc-b437-b2e0f2be69e6_figure14.gif"/>
                </fig>
                <fig fig-type="figure" id="f15" orientation="portrait" position="float">
                    <label>Figure 15. </label>
                    <caption>
                        <title>The activities involved in maintenance of cold chain equipment.</title>
                    </caption>
                    <graphic orientation="portrait" position="float" xlink:href="https://gatesopenresearch-files.f1000.com/manuscripts/13968/fb3ecc6d-3180-4dfc-b437-b2e0f2be69e6_figure15.gif"/>
                </fig>
                <fig fig-type="figure" id="f16" orientation="portrait" position="float">
                    <label>Figure 16. </label>
                    <caption>
                        <title>The process of maintenance of cold chain equipment.</title>
                    </caption>
                    <graphic orientation="portrait" position="float" xlink:href="https://gatesopenresearch-files.f1000.com/manuscripts/13968/fb3ecc6d-3180-4dfc-b437-b2e0f2be69e6_figure16.gif"/>
                </fig>
                <fig fig-type="figure" id="f17" orientation="portrait" position="float">
                    <label>Figure 17. </label>
                    <caption>
                        <title>Objects involved in maintenance of the cold chain equipment.</title>
                    </caption>
                    <graphic orientation="portrait" position="float" xlink:href="https://gatesopenresearch-files.f1000.com/manuscripts/13968/fb3ecc6d-3180-4dfc-b437-b2e0f2be69e6_figure17.gif"/>
                </fig>
                <fig fig-type="figure" id="f18" orientation="portrait" position="float">
                    <label>Figure 18. </label>
                    <caption>
                        <title>Vaccine stock management processes and objects involved.</title>
                    </caption>
                    <graphic orientation="portrait" position="float" xlink:href="https://gatesopenresearch-files.f1000.com/manuscripts/13968/fb3ecc6d-3180-4dfc-b437-b2e0f2be69e6_figure18.gif"/>
                </fig>
                <fig fig-type="figure" id="f19" orientation="portrait" position="float">
                    <label>Figure 19. </label>
                    <caption>
                        <title>Vaccine distribution process.</title>
                    </caption>
                    <graphic orientation="portrait" position="float" xlink:href="https://gatesopenresearch-files.f1000.com/manuscripts/13968/fb3ecc6d-3180-4dfc-b437-b2e0f2be69e6_figure19.gif"/>
                </fig>
                <fig fig-type="figure" id="f20" orientation="portrait" position="float">
                    <label>Figure 20. </label>
                    <caption>
                        <title>Vaccine distribution objects involved.</title>
                    </caption>
                    <graphic orientation="portrait" position="float" xlink:href="https://gatesopenresearch-files.f1000.com/manuscripts/13968/fb3ecc6d-3180-4dfc-b437-b2e0f2be69e6_figure20.gif"/>
                </fig>
                <p>

                    <bold>

                        <italic toggle="yes">Temperature monitoring and storage.</italic>
</bold> For every vaccine to be traced and remain visible within the entire supply chain, whether in storage or in transit, the Blockchain application needs to have a means of getting the temperature readings and associate the readings to a specific vaccine that has a unique digital identity within the system. At the time the vaccines arrive at the primary store, the personnel in charge will be responsible for getting the details of the vaccines and associated temperature readings. 
                    <xref ref-type="fig" rid="f12">Figure 12</xref> and 
                    <xref ref-type="fig" rid="f13">Figure 13</xref> represents a use case and activity diagram that will achieve this requirement. This will partly involve the tests done to ensure that the vaccine is in good condition at the time of arrival.</p>
                <p>The application will employ RFID technology and QR codes to digitally capture details of each vaccine. The granularity of the capture during the testing can be the vial and the carton boxes or as agreed by the policy makers. The application will employ iBeacon and Eddystone technologies to provide details of temperature conditions and the adherence to FIFO and FEFO policies.</p>
                <p>While on transit, the trucks involved will have iBeacon temperature sensors that will be connected to the drivers&#x2019; mobile phones and will then be able to transmit the temperature readings of the vaccines in transit. The assumption here is that as part of the contract for transportation the driver needs to have a smart phone with the application responsible for temperature readings and location installed and enabled respectively.</p>
                <p>
                    <xref ref-type="fig" rid="f14">Figure 14</xref> illustrates the process involved in capturing the vaccine details and location, and reading vaccine temperature.</p>
                <p>

                    <bold>

                        <italic toggle="yes">Maintenance of CCE and transport vehicles.</italic>
</bold> Maintenance of the CCE and transport vehicles is important in keeping the vaccines viable. 
                    <xref ref-type="fig" rid="f15">Figure 15</xref> and 
                    <xref ref-type="fig" rid="f16">Figure 16</xref> represent the activity diagram and the sequence diagram that captures the process involved. 
                    <xref ref-type="fig" rid="f17">Figure 17</xref> captures the objects involved in maintenance and clearly shows the attributes that would be important in maintenance.</p>
                <p>

                    <bold>

                        <italic toggle="yes">Vaccine stock management.</italic>
</bold> 
                    <xref ref-type="fig" rid="f18">Figure 18</xref> represents the processes involved in vaccine stock management.</p>
                <p>

                    <bold>

                        <italic toggle="yes">Distribution and Transportation of vaccines.</italic>
</bold> 
                    <xref ref-type="fig" rid="f19">Figure 19</xref> represents the processes involved in distribution of the vaccines, and 
                    <xref ref-type="fig" rid="f20">Figure 20</xref> shows the different components of the vaccine distribution process.</p>
            </sec>
        </sec>
        <sec sec-type="conclusions">
            <title>Conclusion</title>
            <p>The requirements elicitation process enabled the researchers to understand the processes and identify the points of intervention where the blockchain vaccine supply chain application will be of value.</p>
            <p>The work will progress by developing the structure of the model using qualitative data, knowledge of the vaccine supply chain domain and reviewing of other relevant previous models by literature review. The designs of the blockchain application will then be developed and finally the application will be developed and tested.</p>
        </sec>
        <sec>
            <title>Data availability</title>
            <sec>
                <title>Underlying data</title>
                <p>Open Science Framework: Blockchain Web/Mobile Application for Vaccine Supply Chain. 
                    <ext-link ext-link-type="uri" xlink:href="https://doi.org/10.17605/OSF.IO/PTUAG">https://doi.org/10.17605/OSF.IO/PTUAG</ext-link> (
                    <xref ref-type="bibr" rid="ref-18">Mindila, 2018</xref>).</p>
                <p>This project contains the following underlying data:</p>
                <list list-type="bullet">
                    <list-item>
                        <p>Folder: Dropbox: MOH PHOTOS (photographs from the site visit to the Ministry of Health)</p>
                    </list-item>
                    <list-item>
                        <p>Folder: Analysis of Raw Data (tables describing the supply chain and responsibilities of groups within this)</p>
                    </list-item>
                    <list-item>
                        <p>Folder: Information Systems and Reporting (notes taken regarding information systems/reporting)</p>
                    </list-item>
                    <list-item>
                        <p>Folder: Maintenance of Cold Chain Equipment (notes taken regarding cold chain equipment maintenance)</p>
                    </list-item>
                    <list-item>
                        <p>Folder: Meeting Minutes</p>
                    </list-item>
                    <list-item>
                        <p>Folder: Project Plans (project outlines)</p>
                    </list-item>
                    <list-item>
                        <p>Folder: Stock Management (notes taken regarding stock management)</p>
                    </list-item>
                    <list-item>
                        <p>Folder: Storage (notes taken regarding the storage of vaccines)</p>
                    </list-item>
                    <list-item>
                        <p>Survey responses.zip (responses to LimeSurvey questionnaire)</p>
                    </list-item>
                    <list-item>
                        <p>Folder: Temperature (notes taken regarding temperature of vaccines)</p>
                    </list-item>
                    <list-item>
                        <p>Folder: Transport (notes taken regarding transport of vaccines)</p>
                    </list-item>
                    <list-item>
                        <p>Folder: Vaccine Distribution (notes taken regarding vaccine distribution)</p>
                    </list-item>
                    <list-item>
                        <p>Folder: Vaccine Management Policies (notes taken regarding vaccine management policies)</p>
                    </list-item>
                </list>
            </sec>
            <sec>
                <title>Extended data</title>
                <p>Open Science Framework: Blockchain Web/Mobile Application for Vaccine Supply Chain. 
                    <ext-link ext-link-type="uri" xlink:href="https://doi.org/10.17605/OSF.IO/PTUAG">https://doi.org/10.17605/OSF.IO/PTUAG</ext-link> (
                    <xref ref-type="bibr" rid="ref-18">Mindila, 2018</xref>).</p>
                <p>This project contains the following extended data:</p>
                <list list-type="bullet">
                    <list-item>
                        <p>Folder: The Survey Questionnaires (questionnaires administered during this study)</p>
                    </list-item>
                    <list-item>
                        <p>Folder: White Papers</p>
                    </list-item>
                </list>
                <p>Data are available under the terms of the 
                    <ext-link ext-link-type="uri" xlink:href="http://creativecommons.org/publicdomain/zero/1.0/">Creative Commons Zero "No rights reserved" data waiver</ext-link> (CC0 1.0 Public domain dedication).</p>
            </sec>
        </sec>
    </body>
    <back>
        <ref-list>
            <ref id="ref-9">
                <mixed-citation publication-type="journal">
                    <person-group person-group-type="author">
				
                        <name name-style="western">
                            <surname>Abbasi</surname>
                            <given-names>MA</given-names>
                        </name>
				
                        <name name-style="western">
                            <surname>Jabeen</surname>
                            <given-names>J</given-names>
                        </name>
				
                        <name name-style="western">
                            <surname>Hafeez</surname>
                            <given-names>Y</given-names>
                        </name>
				
                        <etal/>
			</person-group>:
                    <article-title>Assessment of Requirement Elicitation Tools and Techniques by Various Parameters.</article-title>
                    <source>
				
                        <italic toggle="yes">Softw Eng.</italic>
			</source>
                    <year>2015</year>;<volume>3</volume>(<issue>2</issue>):<fpage>7</fpage>&#x2013;<lpage>11</lpage>.
                    <ext-link ext-link-type="uri" xlink:href="https://pdfs.semanticscholar.org/866d/5d5b6d584a9ec5de27695cb09b7574ff0a7d.pdf">Reference Source</ext-link>
                </mixed-citation>
            </ref>
            <ref id="ref-7">
                <mixed-citation publication-type="journal">
                    <person-group person-group-type="author">
				
                        <name name-style="western">
                            <surname>Creswell</surname>
                            <given-names>JW</given-names>
                        </name>
			</person-group>:
                    <article-title>Research design: Qualitative, quantitative, and mixed methods approaches (2nd ed.)</article-title>. Thousand Oaks, CA: Sage.<year>2003</year>.
                    <ext-link ext-link-type="uri" xlink:href="https://pdfs.semanticscholar.org/73b7/18e508fa943dfb22a9cb5fb17f888239ad0e.pdf">Reference Source</ext-link>
                </mixed-citation>
            </ref>
            <ref id="ref-10">
                <mixed-citation publication-type="journal">
                    <person-group person-group-type="author">
				
                        <name name-style="western">
                            <surname>Gaukler</surname>
                            <given-names>GM</given-names>
                        </name>
			</person-group>:
                    <article-title>Item-level RFID in a retail supply chain with stock-out-based substitution.</article-title>
                    <source>
				
                        <italic toggle="yes">IEEE Transactions on Industrial Informatics.</italic>
			</source>
                    <year>2011</year>;<volume>7</volume>(<issue>2</issue>):<fpage>362</fpage>&#x2013;<lpage>370</lpage>.
                    <pub-id pub-id-type="doi">10.1109/TII.2010.2068305</pub-id>
                </mixed-citation>
            </ref>
            <ref id="ref-2">
                <mixed-citation publication-type="journal">
                    <person-group person-group-type="author">
				
                        <name name-style="western">
                            <surname>Gon&#x00e7;alves</surname>
                            <given-names>P</given-names>
                        </name>
			</person-group>:
                    <article-title>Phantom orders and demand bubbles in supply chains</article-title>. PhD dissertation, Sloan School of Management, MIT, Cambridge, MA.<year>2003</year>.
                    <ext-link ext-link-type="uri" xlink:href="http://web.mit.edu/~paulopg/www/PhD Thesis Goncalves.PDF">Reference Source</ext-link>
                </mixed-citation>
            </ref>
            <ref id="ref-6">
                <mixed-citation publication-type="journal">
                    <person-group person-group-type="author">
				
                        <name name-style="western">
                            <surname>Hasan</surname>
                            <given-names>R</given-names>
                        </name>
				
                        <name name-style="western">
                            <surname>Sion</surname>
                            <given-names>R</given-names>
                        </name>
				
                        <name name-style="western">
                            <surname>Winslett</surname>
                            <given-names>M</given-names>
                        </name>
			</person-group>:
                    <article-title>The Case of the Fake Picasso: Preventing History Forgery with Secure Provenance</article-title>.
                    <italic toggle="yes">Proccedings of the 7th Conference on File and Storage Technologies</italic>.<year>2009</year>;<fpage>1</fpage>&#x2013;<lpage>14</lpage>.
                    <ext-link ext-link-type="uri" xlink:href="https://www.usenix.net/legacy/events/fast09/tech/full_papers/hasan/hasan.pdf">Reference Source</ext-link>
                </mixed-citation>
            </ref>
            <ref id="ref-11">
                <mixed-citation publication-type="journal">
                    <person-group person-group-type="author">
				
                        <name name-style="western">
                            <surname>Ko</surname>
                            <given-names>D</given-names>
                        </name>
				
                        <name name-style="western">
                            <surname>Kwak</surname>
                            <given-names>Y</given-names>
                        </name>
				
                        <name name-style="western">
                            <surname>Choi</surname>
                            <given-names>D</given-names>
                        </name>
				
                        <etal/>
			</person-group>:
                    <article-title>Design of cold chain application framework (CCAF) based on IOT and cloud</article-title>.
                    <italic toggle="yes">Proceedings - 8th International Conference on U- and E-Service, Science and Technology, UNESST, 2015</italic>.<year>2016</year>;<volume>9</volume>(<issue>12</issue>):<fpage>11</fpage>&#x2013;<lpage>13</lpage>.
                    <pub-id pub-id-type="doi">10.1109/UNESST.2015.8</pub-id>
                </mixed-citation>
            </ref>
            <ref id="ref-5">
                <mixed-citation publication-type="journal">
                    <person-group person-group-type="author">
				
                        <name name-style="western">
                            <surname>Liang</surname>
                            <given-names>X</given-names>
                        </name>
				
                        <name name-style="western">
                            <surname>Shetty</surname>
                            <given-names>S</given-names>
                        </name>
				
                        <name name-style="western">
                            <surname>Tosh</surname>
                            <given-names>D</given-names>
                        </name>
				
                        <etal/>
			</person-group>:
                    <article-title>ProvChain: A Blockchain-based Data Provenance Architecture in Cloud Environment with Enhanced Privacy and Availability</article-title>.
                    <italic toggle="yes">2017 17th IEEE/ACM International Symposium on Cluster, Cloud and Grid Computing</italic>.<year>2017</year>.
                    <pub-id pub-id-type="doi">10.1109/CCGRID.2017.8</pub-id>
                </mixed-citation>
            </ref>
            <ref id="ref-16">
                <mixed-citation publication-type="journal">
                    <person-group person-group-type="author">
				
                        <name name-style="western">
                            <surname>Li</surname>
                            <given-names>C</given-names>
                        </name>
				
                        <name name-style="western">
                            <surname>Zhang</surname>
                            <given-names>L</given-names>
                        </name>
			</person-group>:
                    <article-title>A Blockchain Based New Secure Multi-Layer Network Model for Internet of Things</article-title>.
                    <italic toggle="yes">IEEE International Congress on Internet of Things (ICIOT)</italic>.<year>2017</year>.
                    <pub-id pub-id-type="doi">10.1109/IEEE.ICIOT.2017.34</pub-id>
                </mixed-citation>
            </ref>
            <ref id="ref-12">
                <mixed-citation publication-type="journal">
                    <person-group person-group-type="author">
				
                        <name name-style="western">
                            <surname>Merkle</surname>
                            <given-names>RC</given-names>
                        </name>
			</person-group>:
                    <article-title>A digital signature based on a conventional encryption function</article-title>. Berlin, Heidelberg, Springer Berlin Heidelberg.<year>1988</year>;<fpage>369</fpage>&#x2013;<lpage>378</lpage>.
                    <pub-id pub-id-type="doi">10.1007/3-540-48184-2_32</pub-id>
                </mixed-citation>
            </ref>
            <ref id="ref-13">
                <mixed-citation publication-type="journal">
                    <person-group person-group-type="author">
				
                        <name name-style="western">
                            <surname>Merkle</surname>
                            <given-names>RC</given-names>
                        </name>
			</person-group>:
                    <article-title>Protocols for public key cryptosystems.</article-title>
                    <source>
				
                        <italic toggle="yes">Proc of IEEE Symposium on Security and Privacy.</italic>
			</source>
                    <year>1980</year>;<fpage>122</fpage>&#x2013;<lpage>133</lpage>.
                    <pub-id pub-id-type="doi">10.1109/SP.1980.10006</pub-id>
                </mixed-citation>
            </ref>
            <ref id="ref-18">
                <mixed-citation publication-type="journal">
                    <person-group person-group-type="author">
				
                        <name name-style="western">
                            <surname>Mindila</surname>
                            <given-names>AN</given-names>
                        </name>
			</person-group>:
                    <article-title>Blockchain Web/Mobile Application for Vaccine Supply Chain</article-title>. OSF.<year>2018</year>.
                    <ext-link ext-link-type="uri" xlink:href="http://www.doi.org/10.17605/OSF.IO/PTUAG">http://www.doi.org/10.17605/OSF.IO/PTUAG</ext-link>
                </mixed-citation>
            </ref>
            <ref id="ref-22">
                <mixed-citation publication-type="journal">
                    <person-group person-group-type="author">
				
                        <name name-style="western">
                            <surname>Moreau</surname>
                            <given-names>L</given-names>
                        </name>
				
                        <name name-style="western">
                            <surname>Missier</surname>
                            <given-names>P</given-names>
                        </name>
			</person-group>:
                    <article-title>Prov-N: The Provenance Notation. W3C Recommendations</article-title>.<year>2013</year>.
                    <ext-link ext-link-type="uri" xlink:href="http://www.w3.org/TR/2013/REC-prov-n-20130430/">Reference Source</ext-link>
                </mixed-citation>
            </ref>
            <ref id="ref-1">
                <mixed-citation publication-type="journal">
                    <person-group person-group-type="author">
				
                        <name name-style="western">
                            <surname>&#x00d6;zg&#x00fc;n</surname>
                            <given-names>O</given-names>
                        </name>
				
                        <name name-style="western">
                            <surname>Barlas</surname>
                            <given-names>Y</given-names>
                        </name>
			</person-group>:
                    <article-title>Effects of systemic complexity factors on task difficulty in a stock management game.</article-title>
                    <source>
				
                        <italic toggle="yes">Syst Dyn Rev.</italic>
			</source>
                    <year>2015</year>;<volume>31</volume>(<issue>3</issue>):<fpage>115</fpage>&#x2013;<lpage>146</lpage>.
                    <pub-id pub-id-type="doi">10.1002/sdr.1543</pub-id>
                </mixed-citation>
            </ref>
            <ref id="ref-4">
                <mixed-citation publication-type="journal">
                    <person-group person-group-type="author">
				
                        <name name-style="western">
                            <surname>Villa</surname>
                            <given-names>S</given-names>
                        </name>
				
                        <name name-style="western">
                            <surname>Gon&#x00e7;alves</surname>
                            <given-names>P</given-names>
                        </name>
				
                        <name name-style="western">
                            <surname>Arango</surname>
                            <given-names>S</given-names>
                        </name>
			</person-group>:
                    <article-title>Exploring retailers&#x2019; ordering decisions under delays.</article-title>
                    <source>
				
                        <italic toggle="yes">Syst Dyn Rev.</italic>
			</source>
                    <year>2015</year>;<volume>31</volume>(<issue>1&#x2013;2</issue>):<fpage>1</fpage>&#x2013;<lpage>27</lpage>.
                    <pub-id pub-id-type="doi">10.1002/sdr.1527</pub-id>
                </mixed-citation>
            </ref>
            <ref id="ref-17">
                <mixed-citation publication-type="journal">
                    <collab>WHO</collab>:
                    <article-title>World Health Organization, Effective Vaccine Management</article-title>.<year>2018</year>; Accessed 19 November 2017.
                    <ext-link ext-link-type="uri" xlink:href="http://www.who.int/immunization/programmes_systems/supply_chain/evm/en/">Reference Source</ext-link>
                </mixed-citation>
            </ref>
            <ref id="ref-19">
                <mixed-citation publication-type="journal">
                    <collab>WHO</collab>:
                    <article-title>A Call to Action: For National Programmes and the Global Community by the World Health Organization (WHO) Immunization Advisory Group</article-title>. Geneva Switzerland.<year>2014</year>.
                    <ext-link ext-link-type="uri" xlink:href="https://www.who.int/immunization/sage/meetings/2014/october/3_Call-to-Action_FINALWeb.pdf?ua=1">Reference Source</ext-link>
                </mixed-citation>
            </ref>
            <ref id="ref-20">
                <mixed-citation publication-type="journal">
                    <collab>WHO, UNICEF, Gavi Alliance and Path</collab>:
                    <article-title>Effective Vaccine Management Initiative Background</article-title>.<year>2010</year>.
                    <ext-link ext-link-type="uri" xlink:href="https://www.who.int/immunization/programmes_systems/supply_chain/EVM-background.pdf">Reference Source</ext-link>
                </mixed-citation>
            </ref>
            <ref id="ref-60">
                <mixed-citation publication-type="journal">
                    <collab>World Health Organization</collab>:
                    <article-title>Global Vaccine Action Plan. Decade of vaccine collaboration.</article-title>
                    <source>
				
                        <italic toggle="yes">Vaccine.</italic>
			</source>
                    <year>2013</year>;<volume>31 Suppl 2</volume>:<fpage>B5</fpage>&#x2013;<lpage>31</lpage>.
                    <pub-id pub-id-type="pmid">23734366</pub-id>
                    <pub-id pub-id-type="doi">10.1016/j.vaccine.2013.02.015</pub-id>
                </mixed-citation>
            </ref>
            <ref id="ref-61">
                <mixed-citation publication-type="journal">
                    <collab>World Health Organization</collab>:
                    <article-title>Fulfilling A Promise: Ensuring Immunization for All in Africa</article-title>.<year>2016</year>.
                    <ext-link ext-link-type="uri" xlink:href="https://www.afro.who.int/sites/default/files/2017-12/MCIA Report.pdf">Reference Source</ext-link>
                </mixed-citation>
            </ref>
            <ref id="ref-3">
                <mixed-citation publication-type="journal">
                    <person-group person-group-type="author">
				
                        <name name-style="western">
                            <surname>Yadav</surname>
                            <given-names>P</given-names>
                        </name>
			</person-group>:
                    <article-title>Health Product Supply Chains in Developing Countries: Diagnosis of the Root Causes of Underperformance and an Agenda for Reform</article-title>.
                    <source>
				
                        <italic toggle="yes">Health Syst Reform.</italic>
			</source>
                    <year>2015</year>;<volume>1</volume>(<issue>2</issue>):<fpage>142</fpage>&#x2013;<lpage>154</lpage>.
                    <pub-id pub-id-type="doi">10.4161/23288604.2014.968005</pub-id>
                </mixed-citation>
            </ref>
            <ref id="ref-8">
                <mixed-citation publication-type="journal">
                    <person-group person-group-type="author">
				
                        <name name-style="western">
                            <surname>Yousuf</surname>
                            <given-names>M</given-names>
                        </name>
				
                        <name name-style="western">
                            <surname>Asger</surname>
                            <given-names>M</given-names>
                        </name>
			</person-group>:
                    <article-title>Comparison of Various Requirements Elicitation Techniques.</article-title>
                    <source>
				
                        <italic toggle="yes">Int J Comput Appl.</italic>
			</source>
                    <year>2015</year>;<volume>116</volume>(<issue>4</issue>):<fpage>8</fpage>&#x2013;<lpage>15</lpage>.
                    <pub-id pub-id-type="doi">10.5120/20322-2408</pub-id>
                </mixed-citation>
            </ref>
        </ref-list>
    </back>
    <sub-article article-type="reviewer-report" id="report27875">
        <front-stub>
            <article-id pub-id-type="doi">10.21956/gatesopenres.13968.r27875</article-id>
            <title-group>
                <article-title>Reviewer response for version 1</article-title>
            </title-group>
            <contrib-group>
                <contrib contrib-type="author">
                    <name>
                        <surname>Drosatos</surname>
                        <given-names>George</given-names>
                    </name>
                    <xref ref-type="aff" rid="r27875a1">1</xref>
                    <xref ref-type="aff" rid="r27875a2">2</xref>
                    <role>Referee</role>
                    <uri content-type="orcid">https://orcid.org/0000-0002-8130-5775</uri>
                </contrib>
                <aff id="r27875a1">
                    <label>1</label>Democritus University of Thrace, Alexandroupoli, Greece</aff>
                <aff id="r27875a2">
                    <label>2</label>Athena Research Centre, Marousi, Greece</aff>
            </contrib-group>
            <author-notes>
                <fn fn-type="conflict">
                    <p>
                        <bold>Competing interests: </bold>No competing interests were disclosed.</p>
                </fn>
            </author-notes>
            <pub-date pub-type="epub">
                <day>8</day>
                <month>10</month>
                <year>2019</year>
            </pub-date>
            <permissions>
                <copyright-statement>Copyright: &#x00a9; 2019 Drosatos G</copyright-statement>
                <copyright-year>2019</copyright-year>
                <license xlink:href="https://creativecommons.org/licenses/by/4.0/">
                    <license-p>This is an open access peer review report distributed under the terms of the Creative Commons Attribution Licence, which permits unrestricted use, distribution, and reproduction in any medium, provided the original work is properly cited.</license-p>
                </license>
            </permissions>
            <related-article ext-link-type="doi" id="relatedArticleReport27875" related-article-type="peer-reviewed-article" xlink:href="10.12688/gatesopenres.12877.1"/>
            <custom-meta-group>
                <custom-meta>
                    <meta-name>recommendation</meta-name>
                    <meta-value>approve-with-reservations</meta-value>
                </custom-meta>
            </custom-meta-group>
        </front-stub>
        <body>
            <p>This paper is a requirements elicitation study that tries to define the requirements of a blockchain-based vaccine supply chain ecosystem. According to the authors, this ecosystem is focused on the needs of low and medium-income countries (LMICs). The results of this study are based on a series of different methods, such as focus groups and questionnaires.</p>
            <p> </p>
            <p> Overall, the paper has many issues that can be summarized in the following points: 
                <list list-type="bullet">
                    <list-item>
                        <p>The paper should be rewritten in order to fix issues about syntactical and grammatical errors, and basically to give the reader the impression that is a scientific paper and not just a deliverable of a project.</p>
                    </list-item>
                    <list-item>
                        <p>The authors should include in the introduction a background subsection where they will explain what is blockchain and they will also present the different types of blockchain. After that, they should justify why it is important to use blockchain technology in their case and not just an information system that would manage the vaccine supply chain of a LMIC. Also, the usage (or not) of blockchain technology could be the subject of analysis in the methodology.</p>
                    </list-item>
                    <list-item>
                        <p>Also, a section with related works should be included in the paper. In this section, the authors will present what other research works have appeared in the literature and how they are relevant with their work. These research works could be organized to different areas, such as blockchain applications in medicines supply chain (three research works are available in the scoping review (Drosatos and Kaldoudi, 2019
                            <sup>
                                <xref ref-type="bibr" rid="rep-ref-27875-1">1</xref>
                            </sup>), studies about requirements elicitation to medicines/vaccines supply chain (not necessarily with blockchain technology), etc. Thus, in this case, the authors should exclude from their methods the literature review as it will be presented in related works section.</p>
                    </list-item>
                    <list-item>
                        <p>In the methods and especially in Table 1, the authors should provide details about the demographic data (e.g. age group and gender) of participants and how many of them took part in each method. Also, there is a confusion about the 30 participants in the requirement elicitation workshop (e.g. have they included in Table 1?).</p>
                    </list-item>
                    <list-item>
                        <p>In the results, the authors should provide more details about the results of the questionnaires, the focus groups and the workshops. These details, accompanied with some metrics, may cover the issues of completeness and correctness of the results.</p>
                    </list-item>
                    <list-item>
                        <p>Additionally, it is better to describe the results as text inside in the paper (e.g. accompanied with the participants agreement or disagreement) and avoid too many figures with use cases and sequence diagrams. All these diagrams could be included as an Appendix section.</p>
                    </list-item>
                    <list-item>
                        <p>In the file &#x201c;survey responses.zip&#x201d; of data sources, there are many responses that are empty. Is it right or did something wrong happen?</p>
                    </list-item>
                    <list-item>
                        <p>The conclusion section should be more descriptive about your results, provide more details about the future plans and if there are any plans to deploy this project in Kenya.</p>
                    </list-item>
                    <list-item>
                        <p>Please provide definitions to all the acronyms that appeared in the text (e.g. SOPs, JSI, CHAI, etc.)</p>
                    </list-item>
                </list>
            </p>
            <p>Is the work clearly and accurately presented and does it cite the current literature?</p>
            <p>Partly</p>
            <p>If applicable, is the statistical analysis and its interpretation appropriate?</p>
            <p>Not applicable</p>
            <p>Are all the source data underlying the results available to ensure full reproducibility?</p>
            <p>Partly</p>
            <p>Is the study design appropriate and is the work technically sound?</p>
            <p>Partly</p>
            <p>Are the conclusions drawn adequately supported by the results?</p>
            <p>Partly</p>
            <p>Are sufficient details of methods and analysis provided to allow replication by others?</p>
            <p>Partly</p>
            <p>Reviewer Expertise:</p>
            <p>Data Privacy and Security, Blockchain Technology, Health Informatics.</p>
            <p>I confirm that I have read this submission and believe that I have an appropriate level of expertise to confirm that it is of an acceptable scientific standard, however I have significant reservations, as outlined above.</p>
        </body>
        <back>
            <ref-list>
                <title>References</title>
                <ref id="rep-ref-27875-1">
                    <label>1</label>
                    <mixed-citation publication-type="journal">
                        <person-group person-group-type="author"/>:
                        <article-title>Blockchain Applications in the Biomedical Domain: A Scoping Review.</article-title>
                        <source>
                            <italic>Comput Struct Biotechnol J</italic>
                        </source>.<year>2019</year>;<volume>17</volume>:
                        <elocation-id>10.1016/j.csbj.2019.01.010</elocation-id>
                        <fpage>229</fpage>-<lpage>240</lpage>
                        <pub-id pub-id-type="pmid">30847041</pub-id>
                        <pub-id pub-id-type="doi">10.1016/j.csbj.2019.01.010</pub-id>
                    </mixed-citation>
                </ref>
            </ref-list>
        </back>
    </sub-article>
    <sub-article article-type="reviewer-report" id="report27878">
        <front-stub>
            <article-id pub-id-type="doi">10.21956/gatesopenres.13968.r27878</article-id>
            <title-group>
                <article-title>Reviewer response for version 1</article-title>
            </title-group>
            <contrib-group>
                <contrib contrib-type="author">
                    <name>
                        <surname>Fern&#x00e1;ndez-Caram&#x00e9;s</surname>
                        <given-names>Tiago M.</given-names>
                    </name>
                    <xref ref-type="aff" rid="r27878a1">1</xref>
                    <role>Referee</role>
                    <uri content-type="orcid">https://orcid.org/0000-0003-2179-5917</uri>
                </contrib>
                <aff id="r27878a1">
                    <label>1</label>Department of Computer Engineering, Faculty of Computer Science, Universidade da Coru&#x00f1;a, Coru&#x00f1;a, Spain</aff>
            </contrib-group>
            <author-notes>
                <fn fn-type="conflict">
                    <p>
                        <bold>Competing interests: </bold>No competing interests were disclosed.</p>
                </fn>
            </author-notes>
            <pub-date pub-type="epub">
                <day>7</day>
                <month>10</month>
                <year>2019</year>
            </pub-date>
            <permissions>
                <copyright-statement>Copyright: &#x00a9; 2019 Fern&#x00e1;ndez-Caram&#x00e9;s TM</copyright-statement>
                <copyright-year>2019</copyright-year>
                <license xlink:href="https://creativecommons.org/licenses/by/4.0/">
                    <license-p>This is an open access peer review report distributed under the terms of the Creative Commons Attribution Licence, which permits unrestricted use, distribution, and reproduction in any medium, provided the original work is properly cited.</license-p>
                </license>
            </permissions>
            <related-article ext-link-type="doi" id="relatedArticleReport27878" related-article-type="peer-reviewed-article" xlink:href="10.12688/gatesopenres.12877.1"/>
            <custom-meta-group>
                <custom-meta>
                    <meta-name>recommendation</meta-name>
                    <meta-value>approve-with-reservations</meta-value>
                </custom-meta>
            </custom-meta-group>
        </front-stub>
        <body>
            <p>The article describes the multiple processes involved in the analysis and design of a vaccine supply chain tracking and management system for a low and medium-income country. The manuscript is properly structured, well-written and provides useful information on the different involved entities. However, at a technological level, it is necessary to keep on polishing the article regarding several major issues:</p>
            <p> (1) In the Introduction it is stated that vaccine supply chain participants are likely to be altruistic, which is the reason to make use of a permissioned blockchain. Such a statement raises several questions: 
                <list list-type="bullet">
                    <list-item>
                        <p>Actually are there not potential rogue (i.e., not altruistic) entities in the vaccine supply chain system? What about counterfeit vaccine providers? Can you identify such potential rogue agents?</p>
                    </list-item>
                    <list-item>
                        <p>If there is trustworthiness among the altruistic participants, it is difficult to justify the need for a blockchain&#x2026;in such a case a shared database would provide more transaction performance and would be sufficient. Moreover, please note that the requirements of accurate data collection and secure data storage (indicated in Page 8, right column) have been traditionally provided through databases.</p>
                    </list-item>
                    <list-item>
                        <p>The need for a permissioned blockchain instead of other types of blockchain is not clearly justified. In addition, the motivation of the different entities for contributing to the system or the potential consensus mechanisms to be used is not clear. Have you thought about any kind of reward or reputation scheme that would operate depending on the blockchain transactions?</p>
                    </list-item>
                </list> &#x00a0;</p>
            <p> (2) In the Results Section, in Page 7, right column, the use of RFID for cold chain monitoring and tracking is explicitly reviewed. Nonetheless, please note that there are other technologies different than RFID to carry out such tasks (e.g., classical Bluetooth, BLE). In addition, the fact that RFID tags (especially passive tags) may have communications problems when attached to recipients with liquids should be considered.</p>
            <p> (3) Although a detailed description on the collection of the requirements for creating the system is provided, it is not clear whether the information and modeled processes can be extrapolated to other LMICs (please note that certain involved entities are specific for Kenya). Can you provide any insight in such a matter?</p>
            <p> (4) Different figures should be revised: 
                <list list-type="bullet">
                    <list-item>
                        <p>Does Figure 3 actually provide relevant information?</p>
                    </list-item>
                    <list-item>
                        <p>In the UML sequence diagrams, it is not clear whether the entities inside the squares are actually entities, processes or just calls to functions.</p>
                    </list-item>
                    <list-item>
                        <p>Figure 6 seems to be missing some entities on the right side.&#x00a0;&#x00a0;</p>
                    </list-item>
                    <list-item>
                        <p>There is no&#x00a0;information on the bottom of Figure 8, so its size could be reduced. The same occurs in Figure 19.</p>
                    </list-item>
                    <list-item>
                        <p>In Figure 15, "engange" should be corrected.</p>
                    </list-item>
                    <list-item>
                        <p>In Figures 16 and 18 two entities on the right side seem to be unnecessary.</p>
                    </list-item>
                </list> &#x00a0;</p>
            <p> (5) In Page 14, right column, it is indicated that the application will make use of RFID and QR codes. Why use both technologies? Why such specific technologies? Moreover, it is stated that iBeacon and Eddystone will be used, which are actually usually related to BLE instead of RFID.</p>
            <p> (6) In Page 14, right column, it is indicated that temperature readings would be transmitted to the driver's mobile phone. What about potential communications problems? How would you mitigate such problems?</p>
            <p> (7) Regarding the general research process, it seems rare that in the Conclusions (page 16) it is indicated that the next steps will include to review the literature on previous vaccine supply chain models, since this is often the first step to be taken in similar projects. For instance, note that other authors have previously described blockchain-based systems similar to the one proposed but for drug delivery or for cold chain preservation.</p>
            <p>Is the work clearly and accurately presented and does it cite the current literature?</p>
            <p>Partly</p>
            <p>If applicable, is the statistical analysis and its interpretation appropriate?</p>
            <p>Partly</p>
            <p>Are all the source data underlying the results available to ensure full reproducibility?</p>
            <p>Yes</p>
            <p>Is the study design appropriate and is the work technically sound?</p>
            <p>Partly</p>
            <p>Are the conclusions drawn adequately supported by the results?</p>
            <p>Partly</p>
            <p>Are sufficient details of methods and analysis provided to allow replication by others?</p>
            <p>Partly</p>
            <p>Reviewer Expertise:</p>
            <p>Blockchain, RFID, IoT, supply chain tracking systems</p>
            <p>I confirm that I have read this submission and believe that I have an appropriate level of expertise to confirm that it is of an acceptable scientific standard, however I have significant reservations, as outlined above.</p>
        </body>
    </sub-article>
    <sub-article article-type="reviewer-report" id="report27876">
        <front-stub>
            <article-id pub-id-type="doi">10.21956/gatesopenres.13968.r27876</article-id>
            <title-group>
                <article-title>Reviewer response for version 1</article-title>
            </title-group>
            <contrib-group>
                <contrib contrib-type="author">
                    <name>
                        <surname>Bandara</surname>
                        <given-names>Dilum</given-names>
                    </name>
                    <xref ref-type="aff" rid="r27876a1">1</xref>
                    <role>Referee</role>
                    <uri content-type="orcid">https://orcid.org/0000-0002-2927-5628</uri>
                </contrib>
                <aff id="r27876a1">
                    <label>1</label>Data61, Commonwealth Scientific and Industrial Research Organisation&#x00a0;(CSIRO), Sydney, NSW, Australia</aff>
            </contrib-group>
            <author-notes>
                <fn fn-type="conflict">
                    <p>
                        <bold>Competing interests: </bold>No competing interests were disclosed.</p>
                </fn>
            </author-notes>
            <pub-date pub-type="epub">
                <day>4</day>
                <month>10</month>
                <year>2019</year>
            </pub-date>
            <permissions>
                <copyright-statement>Copyright: &#x00a9; 2019 Bandara D</copyright-statement>
                <copyright-year>2019</copyright-year>
                <license xlink:href="https://creativecommons.org/licenses/by/4.0/">
                    <license-p>This is an open access peer review report distributed under the terms of the Creative Commons Attribution Licence, which permits unrestricted use, distribution, and reproduction in any medium, provided the original work is properly cited.</license-p>
                </license>
            </permissions>
            <related-article ext-link-type="doi" id="relatedArticleReport27876" related-article-type="peer-reviewed-article" xlink:href="10.12688/gatesopenres.12877.1"/>
            <custom-meta-group>
                <custom-meta>
                    <meta-name>recommendation</meta-name>
                    <meta-value>reject</meta-value>
                </custom-meta>
            </custom-meta-group>
        </front-stub>
        <body>
            <p>Authors present a detailed requirement analysis of the vaccine supply chain in Kenya with the objective of developing a blockchain-based solution.</p>
            <p> </p>
            <p> Strong aspects: 
                <list list-type="bullet">
                    <list-item>
                        <p>A mixed-methods methodology is applied to enhance the requirement elicitation process</p>
                    </list-item>
                    <list-item>
                        <p>A large number of stakeholders are interviewed or involved in workshops to gain greater visibility of the supply chain operation as well as to get feedback about the potential a blockchain-based solution</p>
                    </list-item>
                    <list-item>
                        <p>Significant details are provided on the requirement identified both within and outside the paper</p>
                    </list-item>
                </list> Weak aspects: 
                <list list-type="bullet">
                    <list-item>
                        <p>I also agree with the 1st reviewers concern on the correctness and completeness of the identified requirements.</p>
                    </list-item>
                    <list-item>
                        <p>It seems authors approached the requirement analysis with a pre-decision that the solution will be based on a blockchain. Ideally, requirements should be analyzed without any biases on the potential solution. That way requirements could govern the shape of the solution and even help identify a suitable software development methodology. Moreover, it seems certain technologies/products were already pre-decided e.g., use of private blockchain, QR codes, iBeacon, and Eddystone. It's not clear whether these technologies are already used within the supply chain or are part of the authors' future solution.&#x00a0;By having workshops with the National Logistics Working Group to discuss the blockchain-based solution authors demonstrate some design thinking methodology too. Furthermore, there seems to be no strong evidence of lack of visibility and trust along the supply chain that warrants a blockchain-based solution over other alternatives such as a well managed/protected databased or distributed ledger technology (DLT) without a blockchain.</p>
                    </list-item>
                    <list-item>
                        <p>Scope seems to be not well-defined before or after the requirement analysis process. For e.g., while maintenance of CCE and transport vehicles are important, would it be practical to consider within the scope (at least for phase 1).</p>
                    </list-item>
                    <list-item>
                        <p>While significant details are provided on the findings in the form of UML diagrams and raw data, no SRS is developed. This would substantially limit the authors' contribution, as anyone else trying to build a solution either for Kenya or another country with similar context doesn't get the full picture of requirements and especially the separation between functional and non-functional requirements. Also, this involves a set of business processes and decisions/rules. These seem to be not captured/presented either.</p>
                    </list-item>
                </list> Suggestions for improvements: 
                <list list-type="bullet">
                    <list-item>
                        <p>Add a discussion to cover completeness of requirements. While it can never be complete a detailed discussion on the effort taken and its pros and cons could help.</p>
                    </list-item>
                    <list-item>
                        <p>Discuss how authors pre-plan to develop a blockchain-based solution may have affected the requirement analysis and conclusions made.</p>
                    </list-item>
                    <list-item>
                        <p>Prepare a formal SRS enabling potential implementation of related systems, as well as a way of documenting the findings in a more formal/usable work. Given that the paper isn't of research nature, this could perhaps be the best contribution that the authors could make.</p>
                    </list-item>
                    <list-item>
                        <p>While IoT aspects of a clod chain is very important, analysis is limited to use of QR codes, iBeacon, and their interaction with a mobile phone. This is an area blockchain solutions are still evolving. Having a more detailed analysis in these requirements could help the blockchain community.</p>
                    </list-item>
                    <list-item>
                        <p>Recommended to capture business processes and decisions/rules using notations like BPMN (Business Process Modeling Language 2.0) that could speed-up blockchain-based application development. For e.g., refer to BPM and blockchain work at the Business Process Management conference over the last ~3 years.</p>
                    </list-item>
                    <list-item>
                        <p>Critically evaluate whether blockchain is the most fitting technology to solve the problem given the findings of the requirement analysis, not the other way around.</p>
                    </list-item>
                    <list-item>
                        <p>Suggest using the term "blockchain-based vaccine supply chain management application" as authors are building an application around blockchain than planning to build another blockchain. If that's the cases then incentives, etc., need to be evaluated too.</p>
                    </list-item>
                    <list-item>
                        <p>It would be better to start the introduction with the need to have an effective solution for vaccine supply chain than requirement analysis or blockchain.</p>
                    </list-item>
                    <list-item>
                        <p>Fig. 1 is an oversimplification of the supply chain. It would be useful to have a more comprehensive diagram.</p>
                    </list-item>
                </list> Address many language errors and typos, e.g.: 
                <list list-type="bullet">
                    <list-item>
                        <p>develop/gather the complete and right requirements --&gt; gather the complete and right requirements</p>
                    </list-item>
                    <list-item>
                        <p>If this activity is not given the importance it requires it becomes one of the reasons why a project could fail and not fulfil the customers&#x2019; needs. --&gt; Lack of sufficient attention to requirement elicitation is a key reason that a project could fail and not fulfil the customers&#x2019; needs.</p>
                    </list-item>
                    <list-item>
                        <p>Vaccine in the right time. --&gt; vaccine at the right time.</p>
                    </list-item>
                    <list-item>
                        <p>Define abbreviations such as "SOPs" and "PROV" on 1st use</p>
                    </list-item>
                    <list-item>
                        <p>Sub county level Vaccines --&gt; Sub-county-level vaccines</p>
                    </list-item>
                    <list-item>
                        <p>lower level propagates --&gt; lower-level propagates</p>
                    </list-item>
                    <list-item>
                        <p>Use quotes when referring to feedback given by interviewees</p>
                    </list-item>
                    <list-item>
                        <p>Para on "Merkle tree is an ..." is not essential for the given discussion. Similarly, names of software such as "draw.io"</p>
                    </list-item>
                    <list-item>
                        <p>Remove unnecessary vertical lines at bottom of Fig. 8, 14, and 19</p>
                    </list-item>
                </list>
            </p>
            <p>Is the work clearly and accurately presented and does it cite the current literature?</p>
            <p>Yes</p>
            <p>If applicable, is the statistical analysis and its interpretation appropriate?</p>
            <p>Not applicable</p>
            <p>Are all the source data underlying the results available to ensure full reproducibility?</p>
            <p>Partly</p>
            <p>Is the study design appropriate and is the work technically sound?</p>
            <p>Partly</p>
            <p>Are the conclusions drawn adequately supported by the results?</p>
            <p>Partly</p>
            <p>Are sufficient details of methods and analysis provided to allow replication by others?</p>
            <p>Yes</p>
            <p>Reviewer Expertise:</p>
            <p>Distributed Systems (Blockchain, Cloud, P2P), IoT, Data Engineering, and Computer Security</p>
            <p>I confirm that I have read this submission and believe that I have an appropriate level of expertise to state that I do not consider it to be of an acceptable scientific standard, for reasons outlined above.</p>
        </body>
    </sub-article>
    <sub-article article-type="reviewer-report" id="report27087">
        <front-stub>
            <article-id pub-id-type="doi">10.21956/gatesopenres.13968.r27087</article-id>
            <title-group>
                <article-title>Reviewer response for version 1</article-title>
            </title-group>
            <contrib-group>
                <contrib contrib-type="author">
                    <name>
                        <surname>Koyamada</surname>
                        <given-names>Koji</given-names>
                    </name>
                    <xref ref-type="aff" rid="r27087a1">1</xref>
                    <role>Referee</role>
                    <uri content-type="orcid">https://orcid.org/0000-0002-4283-3954</uri>
                </contrib>
                <aff id="r27087a1">
                    <label>1</label>Academic Center for Computing and Media Studies, Kyoto University, Kyoto, Japan</aff>
            </contrib-group>
            <author-notes>
                <fn fn-type="conflict">
                    <p>
                        <bold>Competing interests: </bold>No competing interests were disclosed.</p>
                </fn>
            </author-notes>
            <pub-date pub-type="epub">
                <day>31</day>
                <month>5</month>
                <year>2019</year>
            </pub-date>
            <permissions>
                <copyright-statement>Copyright: &#x00a9; 2019 Koyamada K</copyright-statement>
                <copyright-year>2019</copyright-year>
                <license xlink:href="https://creativecommons.org/licenses/by/4.0/">
                    <license-p>This is an open access peer review report distributed under the terms of the Creative Commons Attribution Licence, which permits unrestricted use, distribution, and reproduction in any medium, provided the original work is properly cited.</license-p>
                </license>
            </permissions>
            <related-article ext-link-type="doi" id="relatedArticleReport27087" related-article-type="peer-reviewed-article" xlink:href="10.12688/gatesopenres.12877.1"/>
            <custom-meta-group>
                <custom-meta>
                    <meta-name>recommendation</meta-name>
                    <meta-value>approve-with-reservations</meta-value>
                </custom-meta>
            </custom-meta-group>
        </front-stub>
        <body>
            <p>The authors&#x00a0;present&#x00a0;the&#x00a0;processes&#x00a0;carried&#x00a0;out&#x00a0;to&#x00a0;be&#x00a0;able&#x00a0;to develop&#x00a0;the&#x00a0;complete&#x00a0;and right requirements to&#x00a0;developing&#x00a0;a&#x00a0;secure and&#x00a0;effective&#x00a0;block chain&#x00a0;system&#x00a0;for&#x00a0;the vaccine&#x00a0;supply chain. Actually, using a&#x00a0;mixed methods&#x00a0;methodology, the authors captured the&#x00a0;requirements for&#x00a0;the&#x00a0;development&#x00a0;of&#x00a0;the&#x00a0;desired vaccine&#x00a0;supply&#x00a0;chain&#x00a0;web/mobile&#x00a0;application.</p>
            <p> </p>
            <p> Although the reviewer thinks that the authors present meaningful results, the authors fail to evaluate the results with respect to the completeness and correctness. The reviewer understands that there is a noticeable problem with the completeness of the the System Requirement Speci&#xfb01;cation (SRS) with its measurement and event with its de&#xfb01;nition. As it is a common&#x00a0;situation, the reviewer feels that the authors can not determine the absolute completeness. The authors need to de&#xfb01;ne a&#x00a0;metrics of a relative completeness in the vaccine&#x00a0;supply&#x00a0;chain&#x00a0;web/mobile&#x00a0;application [references 1-4]. &#x00a0;</p>
            <p> </p>
            <p> Correctness from software engineering perspective can be defined as the adherence to the specifications that determine how users can interact with the software and how the software should behave when it is used correctly.The authors also need to de&#xfb01;ne a&#x00a0;metrics of the correctness in the vaccine&#x00a0;supply&#x00a0;chain&#x00a0;web/mobile&#x00a0;application.</p>
            <p>Is the work clearly and accurately presented and does it cite the current literature?</p>
            <p>Yes</p>
            <p>If applicable, is the statistical analysis and its interpretation appropriate?</p>
            <p>Partly</p>
            <p>Are all the source data underlying the results available to ensure full reproducibility?</p>
            <p>Partly</p>
            <p>Is the study design appropriate and is the work technically sound?</p>
            <p>Partly</p>
            <p>Are the conclusions drawn adequately supported by the results?</p>
            <p>Partly</p>
            <p>Are sufficient details of methods and analysis provided to allow replication by others?</p>
            <p>Partly</p>
            <p>Reviewer Expertise:</p>
            <p>Scientific visualization</p>
            <p>I confirm that I have read this submission and believe that I have an appropriate level of expertise to confirm that it is of an acceptable scientific standard, however I have significant reservations, as outlined above.</p>
        </body>
        <back>
            <ref-list>
                <title>References</title>
                <ref id="rep-ref-27087-1">
                    <label>1</label>
                    <mixed-citation>
                        <person-group person-group-type="author"/>:
                        <article-title>IEEE guide for software requirements speci&#xfb01;cations</article-title>.<year>1984</year>;
                        <elocation-id>10.1109/IEEESTD.1984.119205</elocation-id>
                        <fpage>1</fpage>-<lpage>26</lpage>
                        <pub-id pub-id-type="doi">10.1109/IEEESTD.1984.119205</pub-id>
                    </mixed-citation>
                </ref>
                <ref id="rep-ref-27087-2">
                    <label>2</label>
                    <mixed-citation>
                        <person-group person-group-type="author"/>:
                        <article-title>Identifying and measuring quality in a software requirements speci&#xfb01;cation</article-title>.
                        <source>
                            <italic>Proceedings First International Software Metrics Symposium</italic>
                        </source>.<year>1993</year>;
                        <elocation-id>10.1109/METRIC.1993.263792 pp. 141&#x2013; 152</elocation-id>
                        <fpage>141</fpage>-<lpage>152</lpage>
                        <pub-id pub-id-type="doi">10.1109/METRIC.1993.263792 pp. 141&#x2013; 152</pub-id>
                    </mixed-citation>
                </ref>
                <ref id="rep-ref-27087-3">
                    <label>3</label>
                    <mixed-citation publication-type="journal">
                        <person-group person-group-type="author"/>:
                        <article-title>System function implementation and behavioral modeling: A systems theoretic approach</article-title>.
                        <source>
                            <italic>Systems Engineering</italic>
                        </source>.<year>2001</year>;<volume>4</volume>(<issue>1</issue>) :
                        <elocation-id>10.1002/1520-6858(2001)4:1&lt;58::AID-SYS6&gt;3.0.CO;2-Z</elocation-id>
                        <fpage>58</fpage>-<lpage>75</lpage>
                        <pub-id pub-id-type="doi">10.1002/1520-6858(2001)4:1&lt;58::AID-SYS6&gt;3.0.CO;2-Z</pub-id>
                    </mixed-citation>
                </ref>
                <ref id="rep-ref-27087-4">
                    <label>4</label>
                    <mixed-citation publication-type="journal">
                        <person-group person-group-type="author"/>:
                        <article-title>Requirements completeness: Absolute or relative? Comments on</article-title>.
                        <source>
                            <italic>Systems Engineering</italic>
                        </source>.<year>2001</year>;<volume>4</volume>(<issue>3</issue>) :
                        <elocation-id>10.1002/sys.1019</elocation-id>
                        <fpage>230</fpage>-<lpage>231</lpage>
                        <pub-id pub-id-type="doi">10.1002/sys.1019</pub-id>
                    </mixed-citation>
                </ref>
            </ref-list>
        </back>
    </sub-article>
</article>
