Deployed b2c76c7 with MkDocs version: 1.3.1

sitemap.xml

@@ -2,137 +2,137 @@
 <urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
     <url>
          <loc>None</loc>
-         <lastmod>2022-08-21</lastmod>
+         <lastmod>2022-08-23</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>None</loc>
-         <lastmod>2022-08-21</lastmod>
+         <lastmod>2022-08-23</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>None</loc>
-         <lastmod>2022-08-21</lastmod>
+         <lastmod>2022-08-23</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>None</loc>
-         <lastmod>2022-08-21</lastmod>
+         <lastmod>2022-08-23</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>None</loc>
-         <lastmod>2022-08-21</lastmod>
+         <lastmod>2022-08-23</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>None</loc>
-         <lastmod>2022-08-21</lastmod>
+         <lastmod>2022-08-23</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>None</loc>
-         <lastmod>2022-08-21</lastmod>
+         <lastmod>2022-08-23</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>None</loc>
-         <lastmod>2022-08-21</lastmod>
+         <lastmod>2022-08-23</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>None</loc>
-         <lastmod>2022-08-21</lastmod>
+         <lastmod>2022-08-23</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>None</loc>
-         <lastmod>2022-08-21</lastmod>
+         <lastmod>2022-08-23</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>None</loc>
-         <lastmod>2022-08-21</lastmod>
+         <lastmod>2022-08-23</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>None</loc>
-         <lastmod>2022-08-21</lastmod>
+         <lastmod>2022-08-23</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>None</loc>
-         <lastmod>2022-08-21</lastmod>
+         <lastmod>2022-08-23</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>None</loc>
-         <lastmod>2022-08-21</lastmod>
+         <lastmod>2022-08-23</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>None</loc>
-         <lastmod>2022-08-21</lastmod>
+         <lastmod>2022-08-23</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>None</loc>
-         <lastmod>2022-08-21</lastmod>
+         <lastmod>2022-08-23</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>None</loc>
-         <lastmod>2022-08-21</lastmod>
+         <lastmod>2022-08-23</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>None</loc>
-         <lastmod>2022-08-21</lastmod>
+         <lastmod>2022-08-23</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>None</loc>
-         <lastmod>2022-08-21</lastmod>
+         <lastmod>2022-08-23</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>None</loc>
-         <lastmod>2022-08-21</lastmod>
+         <lastmod>2022-08-23</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>None</loc>
-         <lastmod>2022-08-21</lastmod>
+         <lastmod>2022-08-23</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>None</loc>
-         <lastmod>2022-08-21</lastmod>
+         <lastmod>2022-08-23</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>None</loc>
-         <lastmod>2022-08-21</lastmod>
+         <lastmod>2022-08-23</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>None</loc>
-         <lastmod>2022-08-21</lastmod>
+         <lastmod>2022-08-23</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>None</loc>
-         <lastmod>2022-08-21</lastmod>
+         <lastmod>2022-08-23</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>None</loc>
-         <lastmod>2022-08-21</lastmod>
+         <lastmod>2022-08-23</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>None</loc>
-         <lastmod>2022-08-21</lastmod>
+         <lastmod>2022-08-23</lastmod>
          <changefreq>daily</changefreq>
     </url>
 </urlset>

thesis/chapters/0_abstract.tex

@@ -6,8 +6,8 @@ Despite its long-standing history, artificial intelligence (AI) has only recentl
 \absdiv{Objective}
 This thesis investigates the causes of and a possible resolution to the asymmetry between the adoption of libraries for applying and deploying AI. The potential solution is validated through designing a software framework, called \textit{GreatAI}, which aims to facilitate \underline{G}eneral \underline{R}obust \underline{E}nd-to-end \underline{A}utomated \underline{T}rustworthy deployments while attempting to overcome the practical drawbacks of its predecessors.
 
-\absdiv{Method}
+\absdiv{Methods}
-The utility of \textit{GreatAI}'s design is validated by applying the principles of design science methodology through iteratively shaping it in a case study of a commercial text mining pipeline. Subsequently, interviews are conducted with ten practitioners to assess its generalisability.
+\textit{GreatAI} serves as a proxy for exploring the proposed design decisions, moreover, its initial focus is limited to the domain of natural language processing (NLP). Its design is validated by applying the principles of design science methodology through iteratively shaping it in two case studies of a commercial NLP pipeline. Subsequently, interviews are conducted with ten practitioners to assess its generalisability.
 
 \absdiv{Results}
 \textit{GreatAI} successfully helps implement 33 best practices through an accessible interface. These target the transition between the prototype and production phases of the AI development lifecycle. The feedback from professional data scientists and software engineers showed that ease of use and functionality are equally important in deciding to adopt deployment technologies, and the proposed framework was rated overwhelmingly positively in both dimensions.

thesis/chapters/1_introduction.tex

@@ -1,6 +1,6 @@
 \chapter{Introduction}
 
-Artificial intelligence techniques have recently started enjoying widespread industry awareness and adoption; the use of AI is increasingly prevalent in all sectors \cite{wirtz2019artificial,bosch2021engineering}. The reasons behind this are manifold \cite{jordan2015machine}, to name a few: recent breakthroughs in deep learning (DL), increased public awareness, an abundance of available data, access to powerful low-cost commodity hardware, education, but most interestingly, the rise of high-level libraries making ready-to-use state-of-the-art (SOTA) models easily available. The latter practically abolishes the barrier of entry for applying AI --- and with that --- can help use cases in various areas. 
+Artificial intelligence (AI) techniques have recently started enjoying widespread industry awareness and adoption; the use of AI is increasingly prevalent in all sectors \cite{wirtz2019artificial,bosch2021engineering}. The reasons behind this are manifold \cite{jordan2015machine}, to name a few: recent breakthroughs in deep learning (DL), increased public awareness, an abundance of available data, access to powerful low-cost commodity hardware, education, but most interestingly, the rise of high-level libraries making ready-to-use state-of-the-art (SOTA) models easily available. The latter practically abolishes the barrier of entry for applying AI --- and with that --- can help use cases in various areas. 
 
 However, to achieve robust deployments, the successful integration of AI components into production-ready applications demands strong engineering methods \cite{serban2020adoption}. That is why it is as essential as ever to also focus on the quality and robustness of deployed models and software. For instance, the lack of a proper overview of data transformation steps may lead to suboptimal performance and to introducing unintended biases, which might contribute to the ever-increasing negative externality of misused AI \cite{o2016weapons}.
 
@@ -8,16 +8,18 @@ Concerningly, a peculiar tendency seems to be unfolding: even though industry pr
 
 This thesis sets out to investigate the reasons behind the apparent asymmetry between industry adoption of accessible AI-libraries and existing reusable solutions for robust AI deployments. It is hypothesised that the primary reason for the underwhelming adoption rate of best practices is the short supply of professionals equally proficient in the domains of both data science and software engineering. Nevertheless, even without their presence, practitioners could rely on frameworks to achieve some level of automation and maturity in their deployment processes. However, the barrier of entry for using such existing libraries is too high, especially when compared with the simplicity of AI-libraries.
 
-Therefore, a software framework --- called \href{https://github.com/schmelczer/great-ai}{\textit{GreatAI}} --- is designed and its design is presented in this thesis. The principal motivation behind the construction of \textit{GreatAI} is to facilitate the responsible and robust deployment of algorithms and models by designing a more accessible API in an attempt to overcome the practical drawbacks of other similar frameworks. Its name stands for its main aim: to assist easily creating \underline{G}eneral \underline{R}obust \underline{E}nd-to-end \underline{A}utomated, and \underline{T}rustworthy AI deployments.
+Therefore, a software framework --- called \href{https://github.com/schmelczer/great-ai}{\textit{GreatAI}} --- is designed, and its design is presented in this thesis. The principal motivation behind the construction of \textit{GreatAI} is to facilitate the responsible and robust deployment of algorithms and models by designing a more accessible API in an attempt to overcome the practical drawbacks of other similar frameworks. Its name stands for its main aim: to assist easily creating \underline{G}eneral \underline{R}obust \underline{E}nd-to-end \underline{A}utomated, and \underline{T}rustworthy AI deployments.
 
-The utility of \textit{GreatAI} is validated using the principles of design science methodology \cite{wieringa2014design} through iteratively designing its API and implementation in a case study concerning the text mining pipeline for a commercial product in collaboration with \href{https://scoutinscience.com/}{ScoutinScience B.V.} The goal of the aforementioned software suite is to evaluate technology transfer opportunities in scientific publications. Subsequently, interviews are conducted with practitioners to validate the generalisability of the design.
+The utility of \textit{GreatAI} is validated using the principles of design science methodology \cite{wieringa2014design} through iteratively designing its API and implementation in a case study concerning the natural language processing (NLP) pipeline for a commercial product in collaboration with \href{https://scoutinscience.com/}{ScoutinScience B.V.} The goal of the aforementioned software suite is to evaluate technology transfer opportunities in scientific publications. Subsequently, interviews are conducted with practitioners to validate the generalisability of the design.
+
+The choice of case study subject is no coincidence; while working on the ScoutinScience platform for the last two years, my colleagues and I have increasingly noticed the same recurring challenges in deploying and operating AI/ML pipelines. This has motivated me to pursue a general solution. Considering that the company's predominant field is NLP, the case studies, and hence, the prototype of \textit{GreatAI} will also focus primarily on deploying NLP models. Nonetheless, the motivation for creating a general solution for all AI/ML contexts remains and will be taken into account every step of the way.
 
 \section{Research questions}
 
 I hypothesise that facilitating the adoption of AI deployment best practices is viable by finding less complex framework designs that are easier to adopt in order to decrease the negative externality of misused AI. This paper investigates the hypothesis by answering the following research questions. 
 
 \begin{rqlist}
-  \item To what extent does the complexity of deploying AI hinders industrial applications?
+  \item To what extent does the complexity of deploying AI hinder industrial applications?
   \item What API design techniques can be effectively applied in order to decrease the complexity of correctly deploying AI services?
   \item To what extent can \textit{GreatAI} automatically implement AI deployment best practices?
   \item How suitable is the design of \textit{GreatAI} for helping to apply best practices in other contexts?

thesis/chapters/2_background.tex

@@ -6,7 +6,7 @@ In the following, the context of the problem is presented from three perspective
 
 \section{Accessible AI} \label{section:accessible-ai}
 
-Most companies prefer not to develop new models but instead reuse prior ones \cite{bosch2021engineering}, and they are able to do so increasingly easily. In recent years, there has been a proliferation of highly accessible AI-libraries. For example, let us consider the domain of natural language processing (NLP). There are various options for finding AI solutions that work out of the box: FLAIR \cite{akbik2019flair} and Hugging Face's transformers \cite{wolf2019huggingface} let developers access the state-of-the-art models and methods in only a couple of lines of code (in many cases 2 or 3). Using transfer-learning, Hugging Face enables developers to leverage vast amounts of knowledge learned by pretrained models (such as BERT \cite{devlin2018bert} and its many improved variations) and fine-tune them for their specific use case. The API exposing this is also extremely accessible.
+Most companies prefer not to develop new models but instead reuse prior ones \cite{bosch2021engineering}, and they are able to do so increasingly easily. In recent years, there has been a proliferation of highly accessible AI-libraries. For example, let us consider the domain of natural language processing. There are various options for finding AI solutions that work out of the box: FLAIR \cite{akbik2019flair} and Hugging Face's transformers \cite{wolf2019huggingface} let developers access the state-of-the-art models and methods in only a couple of lines of code (in many cases 2 or 3). Using transfer-learning, Hugging Face enables developers to leverage vast amounts of knowledge learned by pretrained models (such as BERT \cite{devlin2018bert} and its many improved variations) and fine-tune them for their specific use case. The API exposing this is also extremely accessible.
 
 It is not just these two packages, the list of readily available tools is vast: SpaCy \cite{srinivasa2018natural}, Gensim \cite{vrehuuvrek2011gensim}, and scikit-learn \cite{pedregosa2011scikit}, XGBoost \cite{Chen_2016} are other great examples. The situation is similar in all subdomains of artificial intelligence: some domain expertise is --- admittedly --- beneficial but not a hard-requirement. This, combined with the exponentially increasing computing power affordably available to consumers and businesses alike \cite{sun2019summarizing}, results in AI that is accessible by many.
 

thesis/chapters/3_methods.tex

@@ -1,10 +1,10 @@
 \chapter{Methods} \label{chapter:methods}
 
-The chosen methodology for this study is Design Science which emphasises the need to design and investigate artifacts in their context \cite{wieringa2014design}. It consists of a design and an empirical cycle. The purpose of the former is to improve a problem context with a new or redesigned artifact, while in the latter, the problem is investigated, and its potential treatment is validated concurrently. This procedure seems fitting for our problem in consequence of its practical nature.
+The chosen methodology for this study is Design Science which emphasises the need to design and investigate artifacts in their context \cite{wieringa2014design}. It consists of a design and an empirical cycle. The purpose of the former is to improve a problem context with a new or redesigned artifact, while in the latter, the problem is investigated, and its potential treatment is validated concurrently. This strategy seems fitting for our problem in consequence of its practical nature.
 
 The design cycle shares similarities with Action Research \cite{davison2004principles} in which researchers attempt to solve a real-world problem while simultaneously studying the experience of solving said problem. As for the empirical cycle, the pragmatist approach is taken since the value of this research lies in its utility. Moreover, pragmatism adopts an engineering approach to research \cite{shull2007guide}, which happens to be in line with the philosophy of design science. Additionally, as no research method is without flaws, it is imperative to try to compensate for their weaknesses by applying multiple methods. Hence, the study also relies on interviews with professionals to validate the design decisions and determine the generalisability of \textit{GreatAI}.
 
-\section{Design \& empirical cycles}
+\section{Design cycle}
 
 The aim of \textit{GreatAI} can be summarised using the terminology of design science in the following way: 
 \textit{Facilitate the easy adoption of AI deployment best practices
@@ -18,11 +18,21 @@ The practical cases used for the evaluation are further elaborated in Chapter \r
 
 Overall, this problem context carries the properties of typical industry use cases: it utilises a wide range of natural language processing methods, contains complex interactions between the services, benefits from the integration of end-to-end feedback, and has to provide the clients with a platform that they can rely on within their organisation's core processes. Since the final ranking affects real people, explainability and robustness are also central questions.
 
-Before generalising, the framework's design is iteratively refined using the feedback acquired from applying it in practical contexts, which in this case is the research and development of a smaller and a more complex AI component using the work-in-progress framework. The treatment is finding a simple, less cognitively-straining-to-use design that still leads to high-quality deployments, the means of which will be defined in Section \ref{section:requirements}.
+\begin{figure}
+    \centering
+    \includegraphics[width=.75\linewidth]{figures/design-cycle.drawio.png}
+    \captionsetup{width=.9\linewidth}
+    \caption{Implementation of the Design Cycle of design science \cite{wieringa2014design} for our problem context of AI/ML deployments. The thinner arrows denote smaller but more frequent iterations.}
+    \label{fig:design-cycle}
+\end{figure}
+
+The goal is to find a simpler, less cognitively-straining-to-use design that still leads to high-quality deployments, the definition of which will be described in Section \ref{section:requirements}. Before generalising, the framework's design is iteratively refined using the feedback acquired from applying it in practical contexts, which in this case are the research and development of a smaller and a more complex AI component using the work-in-progress framework. 
+
+The design cycle summarising the research approach is shown in Figure \ref{fig:design-cycle} indicating the role of the case studies. The concerns arisen in the \textit{Treatment validation} iterations and their short discussions are highlighted in the form of \textit{Design notes}. Afterwards, they are addressed in the following \textit{Treatment design} iteration. This way, the issues are immediately addressed and the proposed solutions can be traced back to the problems prompting their introduction.
 
 \section{Applicability \& generalisability} \label{section:interview-setup}
 
-To conclusively answer \textbf{RQ3} and \textbf{RQ4}, interviews are conducted with a population of software engineers and data scientists with varying levels of professional background. Since my colleagues and I are likely to have a bias for (or against) the proposed design, the first step of checking its applicability in other practical contexts is to ask the opinion of non-affiliated practitioners.
+To conclusively answer \textbf{RQ3} and \textbf{RQ4}, interviews are conducted with a population of software engineers and data scientists with varying levels of professional background. The interview candidates were recruited from the recommendations of my acquaintances, who were kindly asked to seek out people from their professional networks with any connection to AI/ML. After the first few interviews, participants were also asked to suggest other candidates, preferably from different subfields. After two iterations of reaching out to potential interviewees personally, ten engineers and researchers eventually responded positively and participated in the study.
 
 First, before their interview, participants are requested to complete a questionnaire (shown in Appendix \ref{appendix:practices}) about their last completed AI project; the questions refer to the best practices implemented by \textit{GreatAI}. They are also advised to take a quick look at the tutorial page of the documentation. The interviews are divided into two halves. In the first part, after a brief introduction, interviewees are asked to solve a real-world task by finishing a partially completed example application using \textit{GreatAI}. They are also encouraged to think aloud so their feedback can be noted. Successfully completing the task creates a system implementing a known number of best practices. This way, the added value --- in terms of a larger number of implemented best practices --- can be quantitatively analysed by comparing the qualities of the finished implementation with the previously given answers. 
 

thesis/chapters/4_design.tex

@@ -38,7 +38,7 @@ The programming language (PL) of the library should be its only non-general prop
 
 The requirements were chosen stemming from their general importance and potential to be mostly handled (implemented) by a software framework. That is why these provide an ideal initial direction for tackling the issue. Of course, these do not cover all best practices; for instance, the ones relating to organisational processes fall outside the realm of computer science.
 
-\section{Design principles}
+\section{Design principles} \label{section:principles}
 
 Before diving into the concrete issues solved, let us detail the principles that should be used for implementing them in the scope of this framework. As implied in Section \ref{section:scope}, the Unix philosophy \cite{ritchie1978unix,salus1994quarter} of software design is followed. Most notably, the design goal that encourages to \textit{write programs that do one thing and do it well.}\footnote{Of course, \textit{write programs to work together} is also very much applicable since allowing interoperability is one of the core requirements for \textit{GreatAI}.}. Apart from providing a clear and simple picture of the intended use cases for the library, this is also in line with the main notion of \textit{A Philosophy of Software Design} \cite{ousterhout2018philosophy}: APIs should be narrow and deep. A narrow width refers to having a small exposed surface area, i.e. having a small number of functions and classes in the public API. In contrast, depth implies that each accomplishes an involved, complex goal. 
 
@@ -81,3 +81,31 @@ Subjectively, a key component of good DX is \textit{Progressive evaluation} thro
 At the same time, Python codebases are rarely strictly object-oriented (OO). They are a mix of the functional, data-driven, and OO paradigms. Consequently, relying on classes for grouping related functions is not always desirable; therefore, it is even more imperative to name similar functions similarly. This helps discoverability and chunking \cite{hermans2021programmer}, which amounts to quicker comprehension.
 
 There is one more reason to prefer consistency: humans have limited short-term memory (STM) \cite{miller1956magical}. Even though flags as function parameters are frowned upon by some \cite{martin2009clean}, they are useful, especially when configuring libraries. However, if there is no convention for the default value of a flag, clients have to remember the flag's name and initial value simultaneously, quickly overloading their STM. Thus, in the codebase, all defaults must be the same, let us say, \texttt{False}. Sometimes, it can result in a \textit{disable} prefix, which may turn into a double negation. Nevertheless, users should never encounter this since the doubly-negated version is the default; thus, it is only singly negated when overriding it. This approach also implies that something may be recommended to be turned on by default.
+
+\section{Architecture} \label{section:architecture}
+
+Although API design has been the central subject so far, it is worth remembering that APIs are usually expected to have corresponding implementations. \textit{GreatAI} is no exception. As laid out in Section \ref{section:principles}, we strive for narrow and deep interfaces; thus, it is time to address the \textit{depth} component.
+
+\textit{GreatAI} stands on the shoulders of numerous open-source packages and integrates them to provide its various features. The most fundamental dependencies and the entire library in context are shown in Figure \ref{fig:technologies}. Given a Python script or a Jupyter notebook, \textit{GreatAI} transforms the specified prediction functions into a production-ready deployment, deployable either as a Docker image, WSGI-server, or an executable relying on \texttt{uvicorn}. The complete list of dependencies can be found in the repository\footnote{\href{https://github.com/schmelczer/great-ai/blob/main/pyproject.toml}{github.com/schmelczer/great-ai/blob/main/pyproject.toml}}.
+
+\begin{figure}
+    \centering
+    \includegraphics[width=0.65\linewidth]{figures/technologies.png}
+    \captionsetup{width=.9\linewidth}
+    \caption{A very high-level overview of \texttt{GreatAI} in its context. The main dependencies are also highlighted.}
+    \label{fig:technologies}
+\end{figure}
+
+The general theme in the implementation is that each explicit best practice should have its distinct, loosely-coupled functions or classes. When collaboration opportunities arise, such as persisting the model versions (\nth{1} component) into prediction traces (\nth{2} component), there are three primary conduits for realising them. These are the \texttt{context} object responsible for the global configuration per process, the \texttt{FunctionMetadataStore} specifying the expected behaviour of each prediction function, and finally the \texttt{TracingContext} that is created anew for each prediction input (session).
+
+After refining the framework with feedback gathered from case studies and users, we will end up with the core architecture presented in Figure \ref{fig:architecture}. The implementation is mixed-paradigm, combining the expressiveness of functional and the design patterns of object-oriented programming (OOP) in order to maintain an overall low complexity. Reflection is also utilised, especially for run-time type-checking and generating the API definitions and Dashboard components. Regardless, the architecture is still presented with a syntax similar to the class diagrams of UML2 \cite{Rumbaugh2004} because it provides the freedom to express even the non-OOP design aspects.
+
+For the sake of brevity, Figure \ref{fig:architecture} does not show all fields, and some related entities have been combined, e.g. the \textit{GroundTruthAPI} box represents the \texttt{add\_ground\_truth}, \texttt{query\_ground\_truth}, and \texttt{delete\_ground\_truth} functions. The client project can also access most of the presented entities, but these optional dependency arrows are not shown in the diagram. The \texttt{utilities} submodule is also left unexpanded; almost all of its functions are orthogonal with the exception of \texttt{parallel\_map}. The latter follows a textbook producer-consumer model facilitated by queues and event signals \cite{wang2020producer}.
+
+\begin{figure}
+    \centering
+    \includegraphics[width=\linewidth]{figures/architecture.png}
+    \captionsetup{width=.9\linewidth}
+    \caption{The core architecture of \textit{GreatAI} illustrated with syntax loosely-based on UML2 \cite{Rumbaugh2004}. Given its framework nature, the expected client project and the actor integrating it are highlighted; the associations between the framework and the client project are achieved through the use of decorators.}
+    \label{fig:architecture}
+\end{figure}

thesis/chapters/5_cases/main.tex

@@ -1,6 +1,6 @@
 \chapter{The ScoutinScience platform} \label{chapter:case}
 
-The core product of \href{https://scoutinscience.com/}{ScoutinScience B.V.} is its platform\footnote{\href{https://dashboard.scoutinscience.com/}{dashboard.scoutinscience.com}}. The clients are technology-transfer offices of Dutch and German universities, government organisations (e.g. Wetsus), and corporates (e.g. Heraeus Group and Ruma Rubber B.V.) who wish to extend the scope of their R\&D activities. ScoutinScience connects to multiple data sources of academic publications and integrates them into a single database. Each new publication is evaluated with a suite of AI components that ultimately determine its technology transfer potential. Other markers are also extracted that help the users get a quick overview of the authors, topics, and contributions of a given piece of research.
+The core product of ScoutinScience B.V.\footnote{\href{https://scoutinscience.com/}{scoutinscience.com}} is its platform\footnote{\href{https://dashboard.scoutinscience.com/}{dashboard.scoutinscience.com}}. The clients are technology-transfer offices of Dutch and German universities, government organisations (e.g. Wetsus), and corporates (e.g. Heraeus Group and Ruma Rubber B.V.) who wish to extend the scope of their R\&D activities. ScoutinScience connects to multiple data sources of academic publications and integrates them into a single database. Each new publication is evaluated with a suite of AI components that ultimately determine its technology transfer potential. Other markers are also extracted that help the users get a quick overview of the authors, topics, and contributions of a given piece of research.
 
 Each client organisation gets to see a different filtered view of this database ranked by the predicted probability of technology transfer opportunities being present. The main motivation is to make these business developers' and other professionals' work more efficient by showing them which papers have the highest chance of being considered interesting by them. 
 

thesis/chapters/5_cases/naive-bayes.tex

@@ -82,7 +82,7 @@ When this model is applied to the \textit{test} split of MAG, we get the confusi
 
 It is, of course, not entirely surprising that the sophisticated transformer architecture of SciBERT is not necessary for a straightforward task like this. Apart from phrases, the relations between separate words of a sentence do not carry nearly as much discriminative power as the identity of the terms \cite{hand2001idiot}; hence, there is little reason for using an attention mechanism. The fact that SciBERT even works in any way on this task is already a testament to its general applicability. Nevertheless, this short experiment has proved that we can safely opt for using MNB for production.
 
-Since Multinomial Naïve Bayes is best at returning a single label and SSC has multiple labels per datapoint: for evaluation purposes, it is checked whether the returned label is contained in the labels of the ground truth. On this dataset, MNB achieves a lower macro-average F1-score of 0.59.\footnote{The code for this is available at \href{https://great-ai.scoutinscience.com/examples/simple/deploy}{great-ai.scoutinscience.com/examples/simple/deploy}.} The weighted-average F1 is 0.70, and the overall accuracy is also 70\%. The substantial difference between the macro and weighted averages comes from the unbalanced distribution of the labels.
+Since Multinomial Naïve Bayes is best at returning a single label and SSC has multiple labels per datapoint: for evaluation purposes, it is checked whether the returned label is contained in the labels of the ground truth. On this dataset, MNB achieves a lower macro-average than on MAG, with F1-score of 0.59.\footnote{The code for this is available at \href{https://great-ai.scoutinscience.com/examples/simple/deploy}{great-ai.scoutinscience.com/examples/simple/deploy}.} The weighted-average F1 is 0.70, and the overall accuracy is also 70\%. The substantial difference between the macro and weighted averages comes from the unbalanced distribution of the labels.
 
 The lower F1-score is not surprising because this dataset has more than twice as many classes. Additionally, the mistakes made are defensible when we look at Figure \ref{fig:ss-confusion}: most of them are between close or related classes.
 

thesis/chapters/5_cases/scibert.tex

@@ -1,3 +1,4 @@
+\newpage
 \section{Text summarisation with SciBERT} \label{section:complex-case}
 
 The ScoutinScience Dashboard contains a full-page evaluation view for academic publications. On this, the known metadata, historical trends about the paper's topics, social media mentions, a PDF viewer showing the document, and other augmentation tools are displayed. One of these is the \textit{Highlights} section, which aims to summarise the paper from a technology-transfer perspective.

thesis/chapters/6_results.tex

@@ -74,8 +74,8 @@ Allow experimentation with the inference code\textsuperscript{3}
 Keep the model's API and documentation together\textsuperscript{3}                        & Dashboard and Swagger                          & \checkmark\checkmark   \\\hline
 Parallelise feature extraction\textsuperscript{3}                                         & \texttt{parallel\_map}                         & \checkmark\checkmark   \\\hline
 Cache predictions\textsuperscript{3}                                                      & \texttt{@GreatAI.create}                       & \checkmark\checkmark   \\\hline
-Async support for top-down chaining models\textsuperscript{3}                             & All decorators support async                   & \checkmark\checkmark   \\\hline
+Support asynchronous top-down chaining of models\textsuperscript{3}                       & All decorators support async                   & \checkmark\checkmark   \\\hline
-Common schemas for common prediction tasks\textsuperscript{3}                             & \texttt{views}                                 & \checkmark             \\\hline
+Implement standard schemas for common prediction tasks\textsuperscript{3}                 & \texttt{views}                                 & \checkmark             \\\hline
 
 \end{tabular}}
 \begin{tablenotes}
@@ -115,9 +115,7 @@ Because the survey's 15 questions were compiled from the \textit{Fully automated
 
 \subsection{Technology acceptance}
 
-Participants filled out a form (shown in Appendix \ref{appendix:questions}) after finishing their first deployment with \textit{GreatAI} to provide data for creating the technology acceptance model of the problem context. The survey contained 12 questions from 3 categories, which could be rated on a 7-point Likert scale. Following the methodology of \cite{cruz2019catalog}, the connections between the Perceived Utility (PU), Perceived Ease Of Use (PEOU), and Intention To Use (ITU) dimensions of TAM were analysed. Two statistically significant ($P \leq 0.05$) correlations were uncovered: between PU and ITU ($r_{Pearson} = 0.81$ with $p = 0.0048$); and PEOU and ITU ($r_{Pearson} = 0.80$ with $p = 0.0068$). Learning from the findings of prior case studies, it is reasonable to believe that both the \textit{perceived utility} and the \textit{perceived ease of use} play an equally important role in influencing professionals' \textit{intention to use} the deployment framework.
+\begin{table}[H]
-
-\begin{table}
 \centering
 \captionsetup{width=.9\linewidth}
 \caption{Aggregated results of the TAM survey (sample size = 10) presented in Appendix \ref{appendix:questions}. The input values range from 1 to 7.}
@@ -131,6 +129,8 @@ Participants filled out a form (shown in Appendix \ref{appendix:questions}) afte
 \end{tabular}}
 \end{table}
 
+Participants filled out a form (shown in Appendix \ref{appendix:questions}) after finishing their first deployment with \textit{GreatAI} to provide data for creating the technology acceptance model of the problem context. The survey contained 12 questions from 3 categories, which could be rated on a 7-point Likert scale. Following the methodology of \cite{cruz2019catalog}, the connections between the Perceived Utility (PU), Perceived Ease Of Use (PEOU), and Intention To Use (ITU) dimensions of TAM were analysed. Two statistically significant ($P \leq 0.05$) correlations were uncovered: between PU and ITU ($r_{Pearson} = 0.81$ with $p = 0.0048$); and PEOU and ITU ($r_{Pearson} = 0.80$ with $p = 0.0068$). Learning from the findings of prior case studies, it is reasonable to believe that both the \textit{perceived utility} and the \textit{perceived ease of use} play an equally important role in influencing professionals' \textit{intention to use} the deployment framework.
+
 The summary of the answers is presented in Table \ref{table:tam}. The assessment of \textit{ease of use} lags behind the rest, but it is still quite high. It may be possible that PEOU would go up with further use. Nevertheless, the high \textit{perceived utility} implies that \textit{GreatAI} shows its value early on. This, combined with the correlations uncovered within the context's technology acceptance model, validates the hypothesis that focusing on good API design is just as necessary as providing practical features.
 
 \subsection{Task solving \& exit interviews}
@@ -181,11 +181,13 @@ Secondly, the survey answers and, in general, the interviewees may be subject to
 
 \section{Future work} 
 
-The primary purpose of the library was to serve as a proxy through which its design decisions could be tested and evaluated in their practical context. For this reason, its design aimed to be a proof-of-principle for validating hypotheses and answering research questions. After successfully doing that, it has been turned into a practical software library suitable for production-use\footnote{\href{https://pypi.org/project/great-ai/}{pypi.org/project/great-ai}}. Although it has already proved its utility, it has also shown that extending its functionality would be worthwhile. Therefore, many potential improvements to \textit{GreatAI} are presented below primarily from the needs arisen during the exit interviews.
+The primary purpose of the library was to serve as a proxy through which its design decisions could be tested and evaluated in their practical context. For this reason, its design aimed to be a proof-of-principle for validating hypotheses and answering research questions. After successfully doing that, it has been turned into a practical software library suitable for production-use\footnote{\href{https://pypi.org/project/great-ai/}{pypi.org/project/great-ai}}. 
+ 
+The library's main limitations come from its bias toward NLP deployments. This is not unreasonable given the design's explorative nature and the context of the case studies. Nevertheless, future work must focus on introducing and balancing support for many more fields' deployments. Although \textit{GreatAI} has already proved its utility, it has also shown that generalising and extending its functionality would be worthwhile. Therefore, many potential improvements are presented below, primarily from the needs arisen during the exit interviews.
 
 \subsection{More AI/ML fields}
 
-The cases presented in Chapter \ref{chapter:case} revolved around NLP. This, of course, heavily influenced the design process. The two most notable effects can be found in the REST API's \texttt{/predict} endpoint and some \texttt{utilities} functions. The former is streamlined to accept JSON-compatible data, while the latter gives robust feature extraction support for only textual input. Supporting the easy, direct upload of larger non-JSON files --- e.g. by saving them to S3 and showing a preview for them on the Dashboard's trace table --- and extending \texttt{utilities} to handle multimedia formats should be sufficient for widely expanding the scope of applicability of \textit{GreatAI}.
+The cases presented in Chapter \ref{chapter:case} revolved around NLP. This, of course, heavily influenced the design process. The two most notable effects can be found in the REST API's \texttt{/predict} endpoint and some \texttt{utilities} functions. The former is streamlined to accept JSON-compatible data, while the latter gives robust feature extraction support for only textual input. Supporting the easy, direct upload of larger non-JSON files --- e.g. by saving them to S3 and showing a preview of them on the Dashboard's trace table --- and extending \texttt{utilities} to handle multimedia formats should be sufficient for counteracting the NLP bias. Hence, widely expanding the scope of applicability of \textit{GreatAI}. As we have seen in Section \ref{section:architecture}, the architecture is otherwise adequately general; therefore, incremental extensions can be applied.
 
 \subsection{More best practices}
 

thesis/chapters/7_conclusion.tex

@@ -12,4 +12,4 @@ The open-ended exit interviews revealed that value can be derived from the libra
 
 \textit{GreatAI} may have the potential to bridge the gap between data science and software engineering. Stemming from the bidirectional nature of bridges, we can look at the framework from two perspectives: for professionals closer to the field of data science, it provides an automatic scaffolding of software facilities that are required for deploying, monitoring, and iterating on their models. For software engineers, it highlights the necessary steps required for robust and improvable deployments. At the same time, it also saves them from the menial work of manually implementing these constructs. While most importantly, it proves that increasing the adoption rate of AI/ML deployment best practices is feasible by designing narrower and deeper APIs.
 
-Good deployments benefit all of us. Accordingly, continued research into the means of good deployments remains crucial. However, next to that --- as the presented results have shown --- better deployments can also be achieved by facilitating the \textit{transition} step of the AI lifecycle with a focus on adoptability. Having automated implementations, even if for just the straightforward best practices, leaves professionals additional time to tackle the more complex deployment challenges and fewer opportunities to miss critical steps. Overall, resulting in more robust, end-to-end automated, and trustworthy AI deployments.
+Good deployments benefit all of us. Accordingly, continued research into the means of good deployments remains crucial. However, next to that --- as the presented results have shown --- better deployments can also be achieved by facilitating the \textit{transition} step of the AI lifecycle with a focus on adoptability. Having automated implementations, even if for just the straightforward best practices, leaves professionals additional time to tackle the more complex deployment challenges and fewer opportunities to miss critical steps. Overall, resulting in more general, robust, end-to-end, automated, and trustworthy AI deployments.

thesis/chapters/appendix.tex

@@ -21,7 +21,7 @@ Similarly to the approach of \cite{serban2020adoption}, participants are asked a
 \item Keep the model's API and documentation together
 \item Parallelise feature extraction
 \item Cache predictions
-\item Async support for top-down chaining models
+\item Support asynchronous top-down chaining of models
 \end{enumerate}
 
 \chapter{Technology acceptance model questionnaire} \label{appendix:questions}

thesis/main.tex

@@ -9,6 +9,7 @@
 \usepackage{enumitem}
 \usepackage{threeparttable}
 \usepackage{multicol}
+\usepackage[super]{nth}
 \usepackage[compact]{titlesec}
 \usepackage{framed}
 \usepackage{quoting}
@@ -144,6 +145,7 @@
 \includepdf[pages=-]{frontpage/frontpage.pdf}
 \include{chapters/0_abstract}
 
+\setcounter{page}{3}
 \tableofcontents
 
 \chapter*{Acknowledgements}

thesis/ref.bib

@@ -868,3 +868,21 @@ numpages = {3}
   year={2019},
   publisher={Springer}
 }
+
+@book{Rumbaugh2004,
+  title = {Unified Modeling Language Reference Manual, The (2nd Edition)},
+  publisher = {Pearson Higher Education},
+  year = {2004},
+  author = {Rumbaugh, James and Jacobson, Ivar and Booch, Grady},
+  isbn = {0321245628}
+}
+
+@inproceedings{wang2020producer,
+  title={Producer-consumer model based thread pool design},
+  author={Wang, Liangzhou and Wang, Chaobin},
+  booktitle={Journal of Physics: Conference Series},
+  volume={1616},
+  pages={012073},
+  year={2020},
+  organization={IOP Publishing}
+}