Deployed 4653ef3 with MkDocs version: 1.3.1
This commit is contained in:
parent
8fbe75890e
commit
292f62958b
43 changed files with 165 additions and 118 deletions
2
404.html
2
404.html
|
|
@ -13,7 +13,7 @@
|
|||
|
||||
|
||||
<link rel="icon" href="/media/favicon.ico">
|
||||
<meta name="generator" content="mkdocs-1.3.1, mkdocs-material-8.4.1">
|
||||
<meta name="generator" content="mkdocs-1.3.1, mkdocs-material-8.4.2">
|
||||
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -13,7 +13,7 @@
|
|||
|
||||
|
||||
<link rel="icon" href="../../../media/favicon.ico">
|
||||
<meta name="generator" content="mkdocs-1.3.1, mkdocs-material-8.4.1">
|
||||
<meta name="generator" content="mkdocs-1.3.1, mkdocs-material-8.4.2">
|
||||
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -13,7 +13,7 @@
|
|||
|
||||
|
||||
<link rel="icon" href="../../../media/favicon.ico">
|
||||
<meta name="generator" content="mkdocs-1.3.1, mkdocs-material-8.4.1">
|
||||
<meta name="generator" content="mkdocs-1.3.1, mkdocs-material-8.4.2">
|
||||
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -13,7 +13,7 @@
|
|||
|
||||
|
||||
<link rel="icon" href="../../../media/favicon.ico">
|
||||
<meta name="generator" content="mkdocs-1.3.1, mkdocs-material-8.4.1">
|
||||
<meta name="generator" content="mkdocs-1.3.1, mkdocs-material-8.4.2">
|
||||
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -13,7 +13,7 @@
|
|||
|
||||
|
||||
<link rel="icon" href="../../media/favicon.ico">
|
||||
<meta name="generator" content="mkdocs-1.3.1, mkdocs-material-8.4.1">
|
||||
<meta name="generator" content="mkdocs-1.3.1, mkdocs-material-8.4.2">
|
||||
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -13,7 +13,7 @@
|
|||
|
||||
|
||||
<link rel="icon" href="../../../media/favicon.ico">
|
||||
<meta name="generator" content="mkdocs-1.3.1, mkdocs-material-8.4.1">
|
||||
<meta name="generator" content="mkdocs-1.3.1, mkdocs-material-8.4.2">
|
||||
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -13,7 +13,7 @@
|
|||
|
||||
|
||||
<link rel="icon" href="../../../media/favicon.ico">
|
||||
<meta name="generator" content="mkdocs-1.3.1, mkdocs-material-8.4.1">
|
||||
<meta name="generator" content="mkdocs-1.3.1, mkdocs-material-8.4.2">
|
||||
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -13,7 +13,7 @@
|
|||
|
||||
|
||||
<link rel="icon" href="../../../media/favicon.ico">
|
||||
<meta name="generator" content="mkdocs-1.3.1, mkdocs-material-8.4.1">
|
||||
<meta name="generator" content="mkdocs-1.3.1, mkdocs-material-8.4.2">
|
||||
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -13,7 +13,7 @@
|
|||
|
||||
|
||||
<link rel="icon" href="../../../media/favicon.ico">
|
||||
<meta name="generator" content="mkdocs-1.3.1, mkdocs-material-8.4.1">
|
||||
<meta name="generator" content="mkdocs-1.3.1, mkdocs-material-8.4.2">
|
||||
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -13,7 +13,7 @@
|
|||
|
||||
|
||||
<link rel="icon" href="../../../media/favicon.ico">
|
||||
<meta name="generator" content="mkdocs-1.3.1, mkdocs-material-8.4.1">
|
||||
<meta name="generator" content="mkdocs-1.3.1, mkdocs-material-8.4.2">
|
||||
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -13,7 +13,7 @@
|
|||
|
||||
|
||||
<link rel="icon" href="../../../media/favicon.ico">
|
||||
<meta name="generator" content="mkdocs-1.3.1, mkdocs-material-8.4.1">
|
||||
<meta name="generator" content="mkdocs-1.3.1, mkdocs-material-8.4.2">
|
||||
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -13,7 +13,7 @@
|
|||
|
||||
|
||||
<link rel="icon" href="../media/favicon.ico">
|
||||
<meta name="generator" content="mkdocs-1.3.1, mkdocs-material-8.4.1">
|
||||
<meta name="generator" content="mkdocs-1.3.1, mkdocs-material-8.4.2">
|
||||
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -13,7 +13,7 @@
|
|||
|
||||
|
||||
<link rel="icon" href="../../media/favicon.ico">
|
||||
<meta name="generator" content="mkdocs-1.3.1, mkdocs-material-8.4.1">
|
||||
<meta name="generator" content="mkdocs-1.3.1, mkdocs-material-8.4.2">
|
||||
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -13,7 +13,7 @@
|
|||
|
||||
|
||||
<link rel="icon" href="../../media/favicon.ico">
|
||||
<meta name="generator" content="mkdocs-1.3.1, mkdocs-material-8.4.1">
|
||||
<meta name="generator" content="mkdocs-1.3.1, mkdocs-material-8.4.2">
|
||||
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -13,7 +13,7 @@
|
|||
|
||||
|
||||
<link rel="icon" href="../../media/favicon.ico">
|
||||
<meta name="generator" content="mkdocs-1.3.1, mkdocs-material-8.4.1">
|
||||
<meta name="generator" content="mkdocs-1.3.1, mkdocs-material-8.4.2">
|
||||
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -13,7 +13,7 @@
|
|||
|
||||
|
||||
<link rel="icon" href="../../media/favicon.ico">
|
||||
<meta name="generator" content="mkdocs-1.3.1, mkdocs-material-8.4.1">
|
||||
<meta name="generator" content="mkdocs-1.3.1, mkdocs-material-8.4.2">
|
||||
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -13,7 +13,7 @@
|
|||
|
||||
|
||||
<link rel="icon" href="../../media/favicon.ico">
|
||||
<meta name="generator" content="mkdocs-1.3.1, mkdocs-material-8.4.1">
|
||||
<meta name="generator" content="mkdocs-1.3.1, mkdocs-material-8.4.2">
|
||||
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -13,7 +13,7 @@
|
|||
|
||||
|
||||
<link rel="icon" href="../../media/favicon.ico">
|
||||
<meta name="generator" content="mkdocs-1.3.1, mkdocs-material-8.4.1">
|
||||
<meta name="generator" content="mkdocs-1.3.1, mkdocs-material-8.4.2">
|
||||
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -13,7 +13,7 @@
|
|||
|
||||
|
||||
<link rel="icon" href="../../media/favicon.ico">
|
||||
<meta name="generator" content="mkdocs-1.3.1, mkdocs-material-8.4.1">
|
||||
<meta name="generator" content="mkdocs-1.3.1, mkdocs-material-8.4.2">
|
||||
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -13,7 +13,7 @@
|
|||
|
||||
|
||||
<link rel="icon" href="media/favicon.ico">
|
||||
<meta name="generator" content="mkdocs-1.3.1, mkdocs-material-8.4.1">
|
||||
<meta name="generator" content="mkdocs-1.3.1, mkdocs-material-8.4.2">
|
||||
|
||||
|
||||
|
||||
|
|
@ -947,7 +947,7 @@
|
|||
|
||||
<p><a href="https://badge.fury.io/py/great-ai"><img alt="PyPI version" src="https://badge.fury.io/py/great-ai.svg" /></a>
|
||||
<a href="https://pepy.tech/project/great-ai"><img alt="Downloads" src="https://pepy.tech/badge/great-ai/month" /></a>
|
||||
<img alt="Docker Pulls" src="https://img.shields.io/docker/pulls/schmelczera/great-ai" />
|
||||
<a href="https://hub.docker.com/repository/docker/schmelczera/great-ai"><img alt="Docker Pulls" src="https://img.shields.io/docker/pulls/schmelczera/great-ai" /></a>
|
||||
<a href="https://github.com/schmelczer/great-ai/actions/workflows/test.yml"><img alt="Test" src="https://github.com/schmelczer/great-ai/actions/workflows/test.yml/badge.svg" /></a>
|
||||
<a href="https://sonar.scoutinscience.com/dashboard?id=great-ai"><img alt="Sonar line coverage" src="https://sonar.scoutinscience.com/api/project_badges/measure?project=great-ai&metric=coverage" /></a>
|
||||
<a href="https://sonar.scoutinscience.com/dashboard?id=great-ai"><img alt="Sonar LoC" src="https://sonar.scoutinscience.com/api/project_badges/measure?project=great-ai&metric=ncloc" /></a></p>
|
||||
|
|
@ -1052,7 +1052,7 @@
|
|||
<small>
|
||||
|
||||
Last update:
|
||||
<span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-date">August 20, 2022</span>
|
||||
<span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-date">September 4, 2022</span>
|
||||
|
||||
|
||||
</small>
|
||||
|
|
|
|||
2
index.md
2
index.md
|
|
@ -5,7 +5,7 @@
|
|||
|
||||
[](https://badge.fury.io/py/great-ai)
|
||||
[](https://pepy.tech/project/great-ai)
|
||||
|
||||
[](https://hub.docker.com/repository/docker/schmelczera/great-ai)
|
||||
[](https://github.com/schmelczer/great-ai/actions/workflows/test.yml)
|
||||
[](https://sonar.scoutinscience.com/dashboard?id=great-ai)
|
||||
[](https://sonar.scoutinscience.com/dashboard?id=great-ai)
|
||||
|
|
|
|||
BIN
media/scibert-confusion.png
Normal file
BIN
media/scibert-confusion.png
Normal file
Binary file not shown.
|
After Width: | Height: | Size: 41 KiB |
|
|
@ -13,7 +13,7 @@
|
|||
|
||||
|
||||
<link rel="icon" href="../media/favicon.ico">
|
||||
<meta name="generator" content="mkdocs-1.3.1, mkdocs-material-8.4.1">
|
||||
<meta name="generator" content="mkdocs-1.3.1, mkdocs-material-8.4.2">
|
||||
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -13,7 +13,7 @@
|
|||
|
||||
|
||||
<link rel="icon" href="../../media/favicon.ico">
|
||||
<meta name="generator" content="mkdocs-1.3.1, mkdocs-material-8.4.1">
|
||||
<meta name="generator" content="mkdocs-1.3.1, mkdocs-material-8.4.2">
|
||||
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -13,7 +13,7 @@
|
|||
|
||||
|
||||
<link rel="icon" href="../../media/favicon.ico">
|
||||
<meta name="generator" content="mkdocs-1.3.1, mkdocs-material-8.4.1">
|
||||
<meta name="generator" content="mkdocs-1.3.1, mkdocs-material-8.4.2">
|
||||
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -13,7 +13,7 @@
|
|||
|
||||
|
||||
<link rel="icon" href="../../media/favicon.ico">
|
||||
<meta name="generator" content="mkdocs-1.3.1, mkdocs-material-8.4.1">
|
||||
<meta name="generator" content="mkdocs-1.3.1, mkdocs-material-8.4.2">
|
||||
|
||||
|
||||
|
||||
|
|
|
|||
54
sitemap.xml
54
sitemap.xml
|
|
@ -2,137 +2,137 @@
|
|||
<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
|
||||
<url>
|
||||
<loc>None</loc>
|
||||
<lastmod>2022-08-23</lastmod>
|
||||
<lastmod>2022-09-04</lastmod>
|
||||
<changefreq>daily</changefreq>
|
||||
</url>
|
||||
<url>
|
||||
<loc>None</loc>
|
||||
<lastmod>2022-08-23</lastmod>
|
||||
<lastmod>2022-09-04</lastmod>
|
||||
<changefreq>daily</changefreq>
|
||||
</url>
|
||||
<url>
|
||||
<loc>None</loc>
|
||||
<lastmod>2022-08-23</lastmod>
|
||||
<lastmod>2022-09-04</lastmod>
|
||||
<changefreq>daily</changefreq>
|
||||
</url>
|
||||
<url>
|
||||
<loc>None</loc>
|
||||
<lastmod>2022-08-23</lastmod>
|
||||
<lastmod>2022-09-04</lastmod>
|
||||
<changefreq>daily</changefreq>
|
||||
</url>
|
||||
<url>
|
||||
<loc>None</loc>
|
||||
<lastmod>2022-08-23</lastmod>
|
||||
<lastmod>2022-09-04</lastmod>
|
||||
<changefreq>daily</changefreq>
|
||||
</url>
|
||||
<url>
|
||||
<loc>None</loc>
|
||||
<lastmod>2022-08-23</lastmod>
|
||||
<lastmod>2022-09-04</lastmod>
|
||||
<changefreq>daily</changefreq>
|
||||
</url>
|
||||
<url>
|
||||
<loc>None</loc>
|
||||
<lastmod>2022-08-23</lastmod>
|
||||
<lastmod>2022-09-04</lastmod>
|
||||
<changefreq>daily</changefreq>
|
||||
</url>
|
||||
<url>
|
||||
<loc>None</loc>
|
||||
<lastmod>2022-08-23</lastmod>
|
||||
<lastmod>2022-09-04</lastmod>
|
||||
<changefreq>daily</changefreq>
|
||||
</url>
|
||||
<url>
|
||||
<loc>None</loc>
|
||||
<lastmod>2022-08-23</lastmod>
|
||||
<lastmod>2022-09-04</lastmod>
|
||||
<changefreq>daily</changefreq>
|
||||
</url>
|
||||
<url>
|
||||
<loc>None</loc>
|
||||
<lastmod>2022-08-23</lastmod>
|
||||
<lastmod>2022-09-04</lastmod>
|
||||
<changefreq>daily</changefreq>
|
||||
</url>
|
||||
<url>
|
||||
<loc>None</loc>
|
||||
<lastmod>2022-08-23</lastmod>
|
||||
<lastmod>2022-09-04</lastmod>
|
||||
<changefreq>daily</changefreq>
|
||||
</url>
|
||||
<url>
|
||||
<loc>None</loc>
|
||||
<lastmod>2022-08-23</lastmod>
|
||||
<lastmod>2022-09-04</lastmod>
|
||||
<changefreq>daily</changefreq>
|
||||
</url>
|
||||
<url>
|
||||
<loc>None</loc>
|
||||
<lastmod>2022-08-23</lastmod>
|
||||
<lastmod>2022-09-04</lastmod>
|
||||
<changefreq>daily</changefreq>
|
||||
</url>
|
||||
<url>
|
||||
<loc>None</loc>
|
||||
<lastmod>2022-08-23</lastmod>
|
||||
<lastmod>2022-09-04</lastmod>
|
||||
<changefreq>daily</changefreq>
|
||||
</url>
|
||||
<url>
|
||||
<loc>None</loc>
|
||||
<lastmod>2022-08-23</lastmod>
|
||||
<lastmod>2022-09-04</lastmod>
|
||||
<changefreq>daily</changefreq>
|
||||
</url>
|
||||
<url>
|
||||
<loc>None</loc>
|
||||
<lastmod>2022-08-23</lastmod>
|
||||
<lastmod>2022-09-04</lastmod>
|
||||
<changefreq>daily</changefreq>
|
||||
</url>
|
||||
<url>
|
||||
<loc>None</loc>
|
||||
<lastmod>2022-08-23</lastmod>
|
||||
<lastmod>2022-09-04</lastmod>
|
||||
<changefreq>daily</changefreq>
|
||||
</url>
|
||||
<url>
|
||||
<loc>None</loc>
|
||||
<lastmod>2022-08-23</lastmod>
|
||||
<lastmod>2022-09-04</lastmod>
|
||||
<changefreq>daily</changefreq>
|
||||
</url>
|
||||
<url>
|
||||
<loc>None</loc>
|
||||
<lastmod>2022-08-23</lastmod>
|
||||
<lastmod>2022-09-04</lastmod>
|
||||
<changefreq>daily</changefreq>
|
||||
</url>
|
||||
<url>
|
||||
<loc>None</loc>
|
||||
<lastmod>2022-08-23</lastmod>
|
||||
<lastmod>2022-09-04</lastmod>
|
||||
<changefreq>daily</changefreq>
|
||||
</url>
|
||||
<url>
|
||||
<loc>None</loc>
|
||||
<lastmod>2022-08-23</lastmod>
|
||||
<lastmod>2022-09-04</lastmod>
|
||||
<changefreq>daily</changefreq>
|
||||
</url>
|
||||
<url>
|
||||
<loc>None</loc>
|
||||
<lastmod>2022-08-23</lastmod>
|
||||
<lastmod>2022-09-04</lastmod>
|
||||
<changefreq>daily</changefreq>
|
||||
</url>
|
||||
<url>
|
||||
<loc>None</loc>
|
||||
<lastmod>2022-08-23</lastmod>
|
||||
<lastmod>2022-09-04</lastmod>
|
||||
<changefreq>daily</changefreq>
|
||||
</url>
|
||||
<url>
|
||||
<loc>None</loc>
|
||||
<lastmod>2022-08-23</lastmod>
|
||||
<lastmod>2022-09-04</lastmod>
|
||||
<changefreq>daily</changefreq>
|
||||
</url>
|
||||
<url>
|
||||
<loc>None</loc>
|
||||
<lastmod>2022-08-23</lastmod>
|
||||
<lastmod>2022-09-04</lastmod>
|
||||
<changefreq>daily</changefreq>
|
||||
</url>
|
||||
<url>
|
||||
<loc>None</loc>
|
||||
<lastmod>2022-08-23</lastmod>
|
||||
<lastmod>2022-09-04</lastmod>
|
||||
<changefreq>daily</changefreq>
|
||||
</url>
|
||||
<url>
|
||||
<loc>None</loc>
|
||||
<lastmod>2022-08-23</lastmod>
|
||||
<lastmod>2022-09-04</lastmod>
|
||||
<changefreq>daily</changefreq>
|
||||
</url>
|
||||
</urlset>
|
||||
BIN
sitemap.xml.gz
BIN
sitemap.xml.gz
Binary file not shown.
|
|
@ -13,7 +13,7 @@
|
|||
|
||||
|
||||
<link rel="icon" href="../../media/favicon.ico">
|
||||
<meta name="generator" content="mkdocs-1.3.1, mkdocs-material-8.4.1">
|
||||
<meta name="generator" content="mkdocs-1.3.1, mkdocs-material-8.4.2">
|
||||
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -7,7 +7,7 @@ Despite its long-standing history, artificial intelligence (AI) has only recentl
|
|||
This thesis investigates the causes of and a possible resolution to the asymmetry between the adoption of libraries for applying and deploying AI. The potential solution is validated through designing a software framework, called \textit{GreatAI}, which aims to facilitate \underline{G}eneral \underline{R}obust \underline{E}nd-to-end \underline{A}utomated \underline{T}rustworthy deployments while attempting to overcome the practical drawbacks of its predecessors.
|
||||
|
||||
\absdiv{Methods}
|
||||
\textit{GreatAI} serves as a proxy for exploring the proposed design decisions, moreover, its initial focus is limited to the domain of natural language processing (NLP). Its design is validated by applying the principles of design science methodology through iteratively shaping it in two case studies of a commercial NLP pipeline. Subsequently, interviews are conducted with ten practitioners to assess its generalisability.
|
||||
\textit{GreatAI} serves as a proxy for exploring the proposed design decisions; moreover, its initial focus is limited to the domain of natural language processing (NLP). Its design is validated by applying the principles of design science methodology through iteratively shaping it in two case studies of a commercial NLP pipeline. Subsequently, interviews are conducted with ten practitioners to assess its applicability and generalisability.
|
||||
|
||||
\absdiv{Results}
|
||||
\textit{GreatAI} successfully helps implement 33 best practices through an accessible interface. These target the transition between the prototype and production phases of the AI development lifecycle. The feedback from professional data scientists and software engineers showed that ease of use and functionality are equally important in deciding to adopt deployment technologies, and the proposed framework was rated overwhelmingly positively in both dimensions.
|
||||
|
|
|
|||
|
|
@ -20,7 +20,7 @@ Overall, this problem context carries the properties of typical industry use cas
|
|||
|
||||
\begin{figure}
|
||||
\centering
|
||||
\includegraphics[width=.75\linewidth]{figures/design-cycle.drawio.png}
|
||||
\includegraphics[width=.85\linewidth]{figures/design-cycle.drawio.png}
|
||||
\captionsetup{width=.9\linewidth}
|
||||
\caption{Implementation of the Design Cycle of design science \cite{wieringa2014design} for our problem context of AI/ML deployments. The thinner arrows denote smaller but more frequent iterations.}
|
||||
\label{fig:design-cycle}
|
||||
|
|
@ -32,12 +32,16 @@ The design cycle summarising the research approach is shown in Figure \ref{fig:d
|
|||
|
||||
\section{Applicability \& generalisability} \label{section:interview-setup}
|
||||
|
||||
To conclusively answer \textbf{RQ3} and \textbf{RQ4}, interviews are conducted with a population of software engineers and data scientists with varying levels of professional background. The interview candidates were recruited from the recommendations of my acquaintances, who were kindly asked to seek out people from their professional networks with any connection to AI/ML. After the first few interviews, participants were also asked to suggest other candidates, preferably from different subfields. After two iterations of reaching out to potential interviewees personally, ten engineers and researchers eventually responded positively and participated in the study.
|
||||
To conclusively answer \textbf{RQ3} and \textbf{RQ4}, interviews are conducted with a population of software engineers and data scientists with varying levels of professional background. The interview candidates were recruited from the recommendations of my acquaintances, who were kindly asked to seek out people from their professional networks with any connection to AI/ML. After the first few interviews, participants were also asked to suggest other candidates, preferably from different subfields. After two iterations of reaching out to potential interviewees personally, ten engineers and researchers eventually responded positively and participated in the study. Albeit the sample size is small, it still represents a wide range of organisation types: experts were included from startups, consultancies, government organisations, and research companies.
|
||||
|
||||
First, before their interview, participants are requested to complete a questionnaire (shown in Appendix \ref{appendix:practices}) about their last completed AI project; the questions refer to the best practices implemented by \textit{GreatAI}. They are also advised to take a quick look at the tutorial page of the documentation. The interviews are divided into two halves. In the first part, after a brief introduction, interviewees are asked to solve a real-world task by finishing a partially completed example application using \textit{GreatAI}. They are also encouraged to think aloud so their feedback can be noted. Successfully completing the task creates a system implementing a known number of best practices. This way, the added value --- in terms of a larger number of implemented best practices --- can be quantitatively analysed by comparing the qualities of the finished implementation with the previously given answers.
|
||||
First, before their interview, participants are requested to complete a questionnaire (shown in Appendix \ref{appendix:practices}) about their last completed AI project; the questions refer to the best practices implemented by \textit{GreatAI}. They are also advised to take a quick look at the tutorial page of the documentation.
|
||||
|
||||
Notes are taken throughout the interviews and subsequently extended using reflective journaling \cite{halcomb2006verbatim} combined with thematic coding. After which, the insights from the interviewed professionals are distilled using the techniques of thematic analysis \cite{fereday2006demonstrating} following the methodologies of \cite{cruz2019catalog} and \cite{haakman2021ai}. These insights can then be combined with the numerical results to explain and elaborate on them.
|
||||
The interviews are divided into two halves. In the first part, after a brief introduction, interviewees are asked to solve a real-world deployment task by finishing a partially completed example project\footnote{Available at \href{https://github.com/schmelczer/great-ai-interview-task}{github.com/schmelczer/great-ai-interview-task}. The training part of the task has already been done, and the participants only have to deploy a trained classifier.} using \textit{GreatAI}. This is a more straightforward instance of the AI development lifecycle presented in the \textit{GreatAI} tutorials. The interviews took approximately one and a half hours each.
|
||||
|
||||
The second half consists of a short survey allowing to create the Technology Acceptance Model (TAM) \cite{davis1989perceived} of the problem context. The ultimate goal of the presented library is to help increase the adoption rate of best practices. In order to reach that goal, first, the library itself has to gain adoption. TAM and its numerous variations provide means of measuring users' willingness of adopting new technologies. TAM has been widely applied in literature \cite{marangunic2015technology}, and due to its general psychological origins, it proves to be effective in other areas of technology, not just software \cite{riemenschneider2002explaining}.
|
||||
They are also encouraged to think aloud so their feedback can be noted. Successfully completing the task creates a system implementing a known number of best practices. This way, the added value --- in terms of a larger number of implemented best practices --- can be quantitatively analysed by comparing the qualities of the finished implementation with the previously given answers.
|
||||
|
||||
The guidelines proposed by Halcomb et al. \cite{halcomb2006verbatim} are followed for collecting information from interviews and reporting it. This reflexive, iterative process starts by recording participants (with their permission) and concurrent note-taking. Reflective journaling is immediately done post-interview, which is subsequently extended and revised by listening to the recordings. Afterwards, the gathered information is interpreted by applying the methodology of thematic analysis \cite{alhojailan2012thematic}. Thematic analysis is an iterative qualitative investigation technique consisting of labelling, correlating, and structuring the central recurring topics raised during discussions. It has been successfully used in previous software engineering studies for extracting emergent patterns \cite{haakman2021ai,cruz2019catalog}.
|
||||
|
||||
The parsimonious version of TAM will be employed, which was measured to have similar predictive power to that of the original TAM while having fewer variables \cite{wu2011user}. Parsimonious TAM observes three interconnected human aspects that influence the actual behaviour (adoption): perceived usefulness, perceived ease of use, and intention to use. Participants are asked ten questions corresponding to these aspects of their experience using \textit{GreatAI}. The questionnaire is shown in Appendix \ref{appendix:questions}. The internal consistency of the answers is calculated using Chronbach's Alpha \cite{bland1997statistics}, after which the responses are reflected upon.
|
||||
The second half of the one-on-one sessions consists of a short survey allowing us to create the Technology Acceptance Model (TAM) \cite{davis1989perceived} of the problem context. The ultimate goal of the presented library is to help increase the adoption rate of best practices. In order to reach that goal, first, the library itself has to gain adoption. TAM and its numerous variations provide means of measuring users' willingness of adopting new technologies. TAM has been widely applied in literature \cite{marangunic2015technology}, and due to its general psychological origins, it proves to be effective in other areas of technology, not just software \cite{riemenschneider2002explaining}.
|
||||
|
||||
The parsimonious version of TAM is employed, which has been measured to have similar predictive power to that of the original TAM while having fewer variables \cite{wu2011user}. Parsimonious TAM observes three interconnected human aspects that influence the actual behaviour (adoption): perceived usefulness, perceived ease of use, and intention to use. Participants are asked ten questions corresponding to these aspects of their experience using \textit{GreatAI}. The questionnaire is shown in Appendix \ref{appendix:questions}. The internal consistency of the answers is calculated using Cronbach's Alpha \cite{bland1997statistics}, after which the responses are reflected upon.
|
||||
|
|
|
|||
|
|
@ -40,7 +40,7 @@ The requirements were chosen stemming from their general importance and potentia
|
|||
|
||||
\section{Design principles} \label{section:principles}
|
||||
|
||||
Before diving into the concrete issues solved, let us detail the principles that should be used for implementing them in the scope of this framework. As implied in Section \ref{section:scope}, the Unix philosophy \cite{ritchie1978unix,salus1994quarter} of software design is followed. Most notably, the design goal that encourages to \textit{write programs that do one thing and do it well.}\footnote{Of course, \textit{write programs to work together} is also very much applicable since allowing interoperability is one of the core requirements for \textit{GreatAI}.}. Apart from providing a clear and simple picture of the intended use cases for the library, this is also in line with the main notion of \textit{A Philosophy of Software Design} \cite{ousterhout2018philosophy}: APIs should be narrow and deep. A narrow width refers to having a small exposed surface area, i.e. having a small number of functions and classes in the public API. In contrast, depth implies that each accomplishes an involved, complex goal.
|
||||
Before diving into the concrete issues solved, let us detail the principles that should be used for implementing them in the scope of this framework. As implied in Section \ref{section:scope}, the Unix philosophy \cite{ritchie1978unix,salus1994quarter} of software design is followed. Most notably, the design goal that encourages to \textit{write programs that do one thing and do it well.}\footnote{\textit{Write programs to work together} is also applicable since allowing interoperability is part of the core requirements.}. Apart from providing a clear and simple picture of the intended use cases for the library, this is also in line with the main notion of \textit{A Philosophy of Software Design} \cite{ousterhout2018philosophy}: APIs should be narrow and deep. A narrow width refers to having a small exposed surface area, i.e. having a small number of functions and classes in the public API. In contrast, depth implies that each accomplishes an involved, complex goal.
|
||||
|
||||
In a way, the width of an API is the price users have to pay (the effort required for learning it) to use it, while the depth is analogous to the return they get from it. Having to learn little and being provided with a lot of functionality maximises return on investment (ROI), hence, developer experience (DX). The theoretical frameworks presented in \textit{The Programmer's Brain} \cite{hermans2021programmer} provides us with explanations and vocabulary from psychology for arguing about the cognitive aspects of API design. In the following, two of them will be used for detailing the design principles: cognitive dimensions of code bases (CDCB) which is an extension of the cognitive dimensions of notation (CDN) framework \cite{blackwell2001cognitive}, and linguistic anti-patterns \cite{arnaoudova2016linguistic}. The former comes with a set of dimensions describing different (often competing) cognitive aspects of code that influence one's ability to perform specific tasks.
|
||||
|
||||
|
|
@ -50,7 +50,7 @@ Nonetheless, simple APIs come at a high technical cost. The library has to imple
|
|||
|
||||
\subsection{Default configuration}
|
||||
|
||||
\href{https://grugbrain.dev/#grug-on-apis}{Existing frameworks frequently suffer from the entanglement of numerous levels of abstractions.} Instead of exposing each implementation detail and encouraging users to interact with most of them, many of these could be abstracted away in a more high-level layer. Even where configuration may be helpful for advanced users, default values can still be chosen automatically while providing an override option where necessary.
|
||||
Existing frameworks frequently suffer from the entanglement of numerous levels of abstractions.\footnote{\href{https://grugbrain.dev/\#grug-on-apis}{grugbrain.dev}} Instead of exposing each implementation detail and encouraging users to interact with most of them, many of these could be abstracted away in a more high-level layer. Even where configuration may be helpful for advanced users, default values can still be chosen automatically while providing an override option where necessary.
|
||||
|
||||
For example, tracing the evaluations and the model versions used in a distributed fashion is very much expected of a trustworthy system. Hence, turning this feature on by default but allowing opting-out from it can result in less scaffolding required from the library's users. It also decreases their up-front cognitive load, which by definition flattens the learning-curve \cite{hermans2021programmer}. Similar features can be imagined for providing a service API for the algorithms, giving feedback, marking outliers, and more.
|
||||
|
||||
|
|
|
|||
|
|
@ -1,3 +1,4 @@
|
|||
\newpage
|
||||
\section{Domain classification with Naïve Bayes} \label{section:simple-case}
|
||||
|
||||
Using different models for slight variations of the same problem is commonplace in the industry. For instance, UberEats has a vast hierarchical set of models for every country, region, and city for calculating the estimated time of delivery \cite{li2017scaling}. We have also found that in order to best process an academic publication, knowing its domain is essential. One of the reasons for this can be the wildly different vocabularies of different domains. For example, the term \textit{framework} in computer science almost always refers to a software artifact (usually implying high tech-transfer potential). In contrast, in most other domains, \textit{framework} is used to describe theoretical models that are less central to practical applications. Of course, it is not merely the meaning of the terms but, more importantly, their distribution that varies significantly. Therefore, the topic of this section is to design and develop a domain prediction classifier for academic papers.
|
||||
|
|
@ -14,18 +15,18 @@ Prior work evaluated SciBERT \cite{beltagy2019scibert} --- a BERT \cite{devlin20
|
|||
|
||||
\subsection{Data}
|
||||
|
||||
Two datasets are considered for the experiments: SciBERT's MAG and the SSC. The former is used to compare the results with SciBERT's, while the latter is utilised for training a model for production purposes because it has 19 labels compared to MAG's seven, and it also contains abstracts instead of just sentences; thus, it is more fitting for our practical use case.
|
||||
|
||||
SciBERT's version of the MAG dataset has 84 thousand and 22.3 thousand sentences in its train and test splits, respectively. These are mostly in English and have all punctuation and casing removed. Each sentence is classified as belonging to one of seven fields. Figure \ref{fig:mag-distribtion} shows that the classes have a uniform distribution.
|
||||
|
||||
\begin{figure}
|
||||
\centering
|
||||
\includegraphics[width=0.5\linewidth]{figures/mag-distribution.png}
|
||||
\includegraphics[width=0.45\linewidth]{figures/mag-distribution.png}
|
||||
\captionsetup{width=.9\linewidth}
|
||||
\caption{Class distribution of the MAG \cite{wang2019review} dataset's 84000 sentences in its \textit{train} split.}
|
||||
\label{fig:mag-distribtion}
|
||||
\end{figure}
|
||||
|
||||
Two datasets are considered for the experiments: SciBERT's MAG and the SSC. The former is used to compare the results with SciBERT's, while the latter is utilised for training a model for production purposes because it has 19 labels compared to MAG's seven, and it also contains abstracts instead of just sentences; thus, it is more fitting for our practical use case.
|
||||
|
||||
SciBERT's version of the MAG dataset has 84,000 and 22,300 sentences in its train and test splits, respectively. These are mostly in English and have all punctuation and casing removed. Each sentence is classified as belonging to one of seven fields. Figure \ref{fig:mag-distribtion} shows that the classes have a uniform distribution.
|
||||
|
||||
SSC is much larger: it contains over 80 million abstracts. Having more data certainly helps in sampling the term distribution more accurately; nonetheless, the law of diminishing returns applies, especially when using simple models. Therefore, the data are randomly downsampled to give us a more manageable couple of hundreds of megabytes of abstracts. We can see the distribution of class labels in Figure \ref{fig:ss-distribution}. The dataset is considerably less balanced: \textit{medicine} is by far the most voluminous field.
|
||||
|
||||
\begin{figure}
|
||||
|
|
@ -52,7 +53,7 @@ Our aims are twofold: (1) to evaluate a sentence classification model on MAG and
|
|||
|
||||
It seems reasonable that only considering the distribution (frequencies) of individual terms may be sufficient. For testing this hypothesis, a unigram language model (Multinomial Naïve Bayes) is constructed, and its accuracy is compared with SciBERT's. The former definitely aligns with the advice to \textit{Use The Most Efficient Models}.
|
||||
|
||||
Using the MNB implementation of scikit-learn \cite{pedregosa2011scikit}, it only took a couple of lines to create, hyperparameter-optimise, and test a text classifier. Including data loading and visualisations, it takes 71 lines of code (LOC) to be more precise. \footnote{The code is available at \href{https://great-ai.scoutinscience.com/tutorial/}{great-ai.scoutinscience.com/tutorial}.} This further proves how simple it is to use standard packages. The code can be considered for satisfying the \textit{Automate Hyper-Parameter Optimisation} best practice since it also implements an automated hyperparameter sweep.
|
||||
Using the MNB implementation of scikit-learn \cite{pedregosa2011scikit}, it only took a couple of lines to create, hyperparameter optimise, and test a text classifier. Including data loading and visualisations, it takes 71 lines of code (LOC) to be more precise. \footnote{The code is available at \href{https://great-ai.scoutinscience.com/tutorial/}{great-ai.scoutinscience.com/tutorial}.} This further proves how simple it is to use standard packages. The code can be considered for satisfying the \textit{Automate Hyper-Parameter Optimisation} best practice since it also implements an automated hyperparameter sweep.
|
||||
|
||||
The sentences are tokenised into words and vectorised with TF-IDF (with logarithmic term frequency) \cite{buckley1985implementation}, the hyperparameters found via 10-fold cross-validation on the \textit{train} split lead to filtering out tokens which occur in fewer than five documents or more than 5\% of the documents.
|
||||
|
||||
|
|
@ -114,7 +115,7 @@ This section briefly explores how the problems raised can be solved using \texti
|
|||
|
||||
The obstacles coming from the intertwined nature of different models are widely recognised \cite{haakman2021ai,amershi2019software,sculley2015hidden}. This can lead to non-monotonic error propagation, meaning that improvements in one part of the system might decrease the overall system quality \cite{amershi2019software}. The importance of schema versioning in an environment of rapidly changing models and transformations is highlighted for a specific use case in \cite{van2017versioning} and more generally by the \textit{Use Versioning for Data, Model, Configurations and Training Scripts} best practice. These emphasise the requirement for versioning models and, in general, data.
|
||||
|
||||
We must address two data storage needs: training data and trained models. Because our code is probably already tracked under Git (and \href{https://octoverse.github.com/#lets-look-back-at-the-code-and-communities-built-on-git-hub-this-year}{likely synchronised with GitHub}), using Git Large File Storage (LFS)\footnote{\href{https://git-lfs.github.com/}{git-lfs.github.com}} might seem intriguing. However, it is a paid (and surprisingly expensive) service of GitHub, especially when we factor in the expected sizes of the models and training data with the fact that the only way to remove files counting towards our quota is to \href{https://docs.github.com/en/repositories/working-with-files/managing-large-files/removing-files-from-git-large-file-storage#git-lfs-objects-in-your-repository}{delete the entire repository}.
|
||||
We must address two data storage needs: training data and trained models. Proper version control is one of the most basic expectations for commercial codebases. Based on developer surveys, it is likely that our code is already tracked under Git and synchronised with GitHub\footnote{\href{https://octoverse.github.com/\#lets-look-back-at-the-code-and-communities-built-on-git-hub-this-year}{octoverse.github.com}}. Therefore, using Git Large File Storage (LFS) might seem intriguing. However, it is a paid (and surprisingly expensive) service of GitHub, especially when we factor in the expected sizes of the models and training data with the fact that the only way to remove files counting towards our quota is to delete the entire repository\footnote{\href{https://docs.github.com/en/repositories/working-with-files/managing-large-files/removing-files-from-git-large-file-storage\#git-lfs-objects-in-your-repository}{docs.github.com/en/repositories/working-with-files/managing-large-files/removing-files-from-git-large-file-storage}}.
|
||||
|
||||
An open-source tool, the Data Version Control (DVC)\footnote{\href{https://dvc.org/}{dvc.org}} provides a nearly perfect alternative. It comes with a command-line interface (CLI) inspired by Git's and can be integrated with several backend storage servers. Its only downside is, of course, that it is one more tool that increases the complexity of the project and the initial setup time. If this is an acceptable price to pay, then I personally recommend opting for DVC. Nevertheless, if this may prohibit a team\footnote{As was the case with MLFlow tracking in an ING team described in Section \ref{section:industry}.} from properly handling data according to the best practices, I present a simpler solution.
|
||||
|
||||
|
|
@ -137,7 +138,7 @@ A function called \texttt{parallel\_map()} is implemented which closely mimics t
|
|||
Some of the expectations one might have for data-intensive (such as AI) software are similar to that for software in general. These are also captured by the best practices: \textit{Use Continuous Integration}, \textit{Automate Model Deployment}, and \textit{Enable Automatic Roll Backs for Production Model} to name a few. It is important to notice that these have already been solved by software engineering, more specifically, by the DevOps paradigm \cite{leite2019survey}.
|
||||
In line with the findings of John et al. \cite{john2020architecting} on the SOTA of AI deployments, I suggest we wrap the applications in a format more compatible with existing DevOps toolkits. Instead of reinventing the wheel, we should rely on more established DevOps best practices for implementing the SE4ML deployment best practices. Besides, organisations are expected to have their deployment processes for classical applications; thus, allowing them to reuse those for AI applications seems to be the most convenient approach.
|
||||
|
||||
Based on personal empirical evidence, three types of software artifacts are identified (in the context of Python) for which a wide range of established practices exist. WSGI server\footnote{\href{https://peps.python.org/pep-3333/}{peps.python.org/pep-3333}} compatible applications, executable scripts, and Docker Images\footnote{\href{https://docs.docker.com/registry/spec/manifest-v2-2/}{docs.docker.com/registry/spec/manifest-v2-2}}. To achieve this, \textit{GreatAI} provides a compatibility layer between simple Python inference functions and all the above common artifacts. Taking functions as input for the first step also satisfies the requirement to be \textbf{General}. Nevertheless, to also allow customisation, additional configuration, metadata, and behavioural specification can be given as well.
|
||||
Based on personal experiences, three types of software artifacts are identified (in the context of Python) for which a wide range of established practices exist. WSGI server\footnote{\href{https://peps.python.org/pep-3333/}{peps.python.org/pep-3333}} compatible applications, executable scripts, and Docker Images\footnote{\href{https://docs.docker.com/registry/spec/manifest-v2-2/}{docs.docker.com/registry/spec/manifest-v2-2}}. To achieve this, \textit{GreatAI} provides a compatibility layer between simple Python inference functions and all the above common artifacts. Taking functions as input for the first step also satisfies the requirement to be \textbf{General}. Nevertheless, to also allow customisation, additional configuration, metadata, and behavioural specification can be given as well.
|
||||
|
||||
\begin{listing}[!ht]
|
||||
\begin{minted}[
|
||||
|
|
@ -160,24 +161,26 @@ def greeter(name: str) -> str:
|
|||
The main advantage of the wrapping approach is that it does not require any input from the clients (by default). I opted for a decorator \cite{gamma1995design}, which lets users wrap their function by adding a single additional line of code as shown in Listing \ref{listing:hello-world}. After which, the created WSGI application can be accessed through the \texttt{greeter.app} property where \texttt{greeter} is the identifier of the user-defined function. A CLI script (\texttt{great-ai}), along with a \texttt{Dockerfile} are also provided to cover the other two deployment artifacts.
|
||||
|
||||
|
||||
\begin{listing}[!ht]
|
||||
\begin{listing}
|
||||
\begin{minted}[
|
||||
frame=lines,
|
||||
framesep=2mm,
|
||||
baselinestretch=1,
|
||||
linenos
|
||||
]{python}
|
||||
from great_ai import GreatAI, parameter, use_model, log_metric
|
||||
from great_ai import save_model, GreatAI, parameter, use_model, log_metric
|
||||
|
||||
save_model('special_number', 405) # this could have been called in another script
|
||||
|
||||
@GreatAI.create
|
||||
@parameter('positive_number', validate=lambda n: n > 0)
|
||||
@use_model('secret-number', version='latest')
|
||||
def add_to_secret_number(positive_number: int, model: int) -> int:
|
||||
@use_model('special_number', version='latest', model_kwarg_name='special')
|
||||
def add_to_special_number(positive_number: int, special: int) -> int:
|
||||
"""This docstring will be exported as documentation."""
|
||||
log_metric('log directly into the Trace', positive_number * 2)
|
||||
return secret + positive_number
|
||||
log_metric('log directly into the Trace', positive_number ** 2)
|
||||
return special + positive_number
|
||||
|
||||
assert add_number(1).output == 5
|
||||
assert add_number(12).output == 417
|
||||
\end{minted}
|
||||
\captionsetup{width=.9\linewidth,position=top,skip=-20pt}
|
||||
\caption{A simple \textit{GreatAI} service with behavioural customisations.}
|
||||
|
|
@ -188,14 +191,14 @@ Coincidentally, deployment best practices can be easily implemented in this wrap
|
|||
|
||||
To allow customising the service's behaviour to fit different use cases, the default configurations can be overridden by calling some library functions. An example of this can be seen in Listing \ref{listing:complex}, while more details of the semantics can be found in the documentation\footnote{\href{https://great-ai.scoutinscience.com/how-to-guides/create-service/}{great-ai.scoutinscience.com/how-to-guides/create-service}}.
|
||||
|
||||
\subsection{Summary}
|
||||
|
||||
After implementing some features of the library, it can already be used for deploying the previously discussed domain prediction model. In this case, online prediction is expected; hence, the REST API-based deployment is chosen, which is created by \texttt{GreatAI.create} and packaged in a Docker image. This image can be instantiated by the company's existing DevOps pipeline and cloud infrastructure. In the end, users can see one more tag in the header section of publication evaluations, where they can also see the explanation behind the model's decision as demonstrated in Figure \ref{fig:dashboard-domains}. Let us now explore how the franework fares in a more complex case.
|
||||
|
||||
\begin{figure}[H]
|
||||
\centering
|
||||
\includegraphics[width=0.7\linewidth]{figures/dashboard-domains.png}
|
||||
\includegraphics[width=0.85\linewidth]{figures/dashboard-domains.png}
|
||||
\captionsetup{width=.9\linewidth}
|
||||
\caption{Screenshot of the domain prediction integrated into the ScoutinScience platform, where it is used as a filtering option.}
|
||||
\label{fig:dashboard-domains}
|
||||
\end{figure}
|
||||
|
||||
\subsection{Summary}
|
||||
|
||||
After implementing some features of the library, it can already be used for deploying the previously discussed domain prediction model. In this case, online prediction is expected; hence, the REST API-based deployment is chosen, which is created by \texttt{GreatAI.create} and packaged in a Docker image. This image can be instantiated by the company's existing DevOps pipeline and cloud infrastructure. In the end, users can see one more tag in the header section of publication evaluations, where they can also see the explanation behind the model's decision as demonstrated in Figure \ref{fig:dashboard-domains}. Let us now explore how the franework fares in a more complex case.
|
||||
|
|
|
|||
|
|
@ -23,11 +23,11 @@ A formulaic expression is a phrase with zero or more ``slots'' which, when fille
|
|||
|
||||
In order to compile a new dataset, experts are asked to judge sentences that passed an \textit{intention check}. This pooling approach is commonly used in information retrieval \cite{schutze2008introduction}. The filtering is expected to sieve out sentences that are probably not relevant from a technology-transfer perspective using Iwatsuki's formulaic expression intention classes. Subsequently, relevance judgements --- in the form of \textit{interesting} or \textit{not interesting} labels --- are gathered for the remaining sentences. This method turns the extractive summarisation into a binary classification task for which a SciBERT model \cite{beltagy2019scibert} can be fine-tuned. Ultimately, the summaries are derived from sentences selected by the classifier trained on the experts' annotations.
|
||||
|
||||
We have to note two possible shortcomings of this setup: firstly, the FE intentions are assumed to be strongly correlated with the sought-after aspect. This may or may not be true. Secondly, only the individual relevance of the sentences is considered instead of the overall relevance (utility) of the summary. Nonetheless, it is expected that stemming from the length of the documents and the sparseness of the selected sentences, that any combination of them is likely to have low redundancy.
|
||||
We have to note two possible shortcomings of this setup: firstly, the FE intentions are assumed to be strongly correlated with the sought-after \textit{tech-transfer opportunities} aspect. This may or may not be true. Secondly, only the individual relevance of the sentences is considered instead of the overall relevance (utility) of the summary. Nonetheless, it is expected that stemming from the length of the documents, and the sparseness of the selected sentences, any combination of them is likely to have low redundancy.
|
||||
|
||||
\subsection{Results}
|
||||
|
||||
For the first iteration, 1500 sentences were selected for two experts to annotate in a binary fashion according to strict guidelines. An example is shown in Figure \ref{fig:annotator}. Afterwards, for measuring the interrater agreement, Cohen's kappa \cite{cohen1960coefficient} is calculated as shown in Equation \ref{equation:kappa}, which turns out to be \textbf{0.4310} for the two annotators. This happens to be just above the lower end of \textit{moderate agreement}. However, we have to note that the original quality ranges are often criticised for being too relaxed \cite{mchugh2012interrater}. Regardless, in the case of summarisation, Verberne et al. \cite{verberne2018creating} argue that reasonable end-quality can be reached even when the interrater agreement is relatively low. The ground truth is determined by taking the logical disjunction of the annotations.
|
||||
For the first iteration, 1500 sentences were selected for two experts to annotate in a binary fashion according to strict guidelines. An example is shown in Figure \ref{fig:annotator}. Afterwards, for measuring the interrater agreement, Cohen's kappa \cite{cohen1960coefficient} is calculated as shown in Equation \ref{equation:kappa}, which turns out to be \textbf{0.43} for the two annotators. This happens to be just above the lower end of \textit{moderate agreement}. Even though the original quality ranges are sometimes criticised for being too relaxed for the medical domain \cite{mchugh2012interrater}, some leniency is acceptable for many NLP tasks due to their subjectiveness. Regardless, in the case of summarisation, Verberne et al. \cite{verberne2018creating} argue that reasonable end-quality can be reached even when the interrater agreement is relatively low. The ground truth is determined by taking the logical disjunction of the annotations.
|
||||
|
||||
\begin{figure}
|
||||
\centering
|
||||
|
|
@ -51,17 +51,33 @@ The next step is fine-tuning SciBERT with the help of Hugging Face \texttt{trans
|
|||
\textbf{Utility of LargeFiles} For the purposes of the documentation, the fine-tuning was conducted in the Google Colab online environment, which is excellent for providing anyone with GPU time for free. However, notebook environments are ephemeral, resulting in the need to manually upload and download all relevant data whenever a new virtual machine (VM) instance is granted. The \texttt{LargeFile} implementation alleviated this problem by automatically handling the uploads and downloads. Of course, first, backwards compatibility had to be solved for Python 3.7, the only available environment in Colab.
|
||||
\end{displayquote}
|
||||
|
||||
The best validation results were achieved after eight epochs which is slightly more than expected but is presumably due to the weight decay. The confusion matrix on the test split can be seen in Figure \ref{fig:scibert-confusion}: regardless of the task's subjective definition, SciBERT achieves good quality indicated by an F1-score of \textbf{0.89}.
|
||||
The best validation results were achieved after eight epochs which is slightly more than expected but is presumably due to the weight decay. The confusion matrix on the test split can be seen in Figure \ref{fig:scibert-confusion}, and the per class accuracy metrics in Table \ref{table:scibert-pr}. Despite the task's subjective definition, SciBERT achieves good quality, indicated by an F1-score of \textbf{0.80}.
|
||||
|
||||
\begin{figure}
|
||||
\centering
|
||||
\includegraphics[width=0.4\linewidth]{figures/scibert-confusion.png}
|
||||
\includegraphics[width=0.55\linewidth]{figures/scibert-confusion.png}
|
||||
\captionsetup{width=.9\linewidth}
|
||||
\caption{Confusion matrix of the fine-tuned SciBERT model on the \textit{summary candidate sentences} dataset. The values are globally normalised and represent percentages.}
|
||||
\caption{Confusion matrix of the fine-tuned SciBERT model on the \textit{summary candidate sentences} dataset.}
|
||||
\label{fig:scibert-confusion}
|
||||
\end{figure}
|
||||
|
||||
Let us check how well the selected sentences correspond with the tech-transfer potential. Users and in-house experts can rate publications (from a tech-transfer perspective) by assigning them to one of four categories: \texttt{A}, \texttt{B}, \texttt{C}, and \texttt{D} with \texttt{A} being the most and \texttt{D} the least promising. This feedback is stored and used for analytic and training purposes. Since both the feedback grade and the ``highlights'' are supposed to reflect the same aspect of papers, therefore, we can reasonably expect some correlation between them.
|
||||
\begin{table}
|
||||
\centering
|
||||
\begin{threeparttable}
|
||||
\caption{Accuracty metrics of the fine-tuned SciBERT model on the \textit{summary candidate sentences} dataset.}
|
||||
\label{table:scibert-pr}
|
||||
\setlength{\tabcolsep}{0.75em} % for the horizontal padding
|
||||
{\renewcommand{\arraystretch}{1.2} % for the vertical padding
|
||||
\begin{tabular}{|l|r|r|r|}
|
||||
\hline
|
||||
{} & \textbf{Precision} & \textbf{Recall} & \textbf{Support} \\\hline
|
||||
\textsc{non-relevant} & 0.93 & 0.83 & 191 \\\hline
|
||||
\textsc{relevant} & 0.73 & 0.88 & 109 \\\hline
|
||||
\end{tabular}}
|
||||
\end{threeparttable}
|
||||
\end{table}
|
||||
|
||||
Let us check how well the selected sentences correspond with the tech-transfer potential. Users and in-house experts can rate publications (from a tech-transfer perspective) by assigning them to one of four categories: \texttt{A}, \texttt{B}, \texttt{C}, and \texttt{D} with \texttt{A} being the most and \texttt{D} the least promising. This feedback is stored and used for analytic and training purposes. Since both the feedback grade and the relevant (summary candidate) sentences are supposed to reflect the same aspect of papers, we can reasonably expect some correlation between the grades and relevant sentence counts.
|
||||
|
||||
Figure \ref{fig:histograms} shows the ratio of summary candidate sentences as predicted by the fine-tuned model in 4 categories (grades) of papers. This dataset does not overlap with the training data; hence, the results come solely from the model's ability to generalise. It is interesting to see that the Spearman's rank correlation coefficient \cite{spearman1961proof} between the normalised ``highlights'' counts and the ratings of papers is \textbf{0.4784} and is statistically significant ($P = 5.4 \times 10^{-74}$). This proves the presence of a monotonic association. For context, the correlation between the grades and the number of sentences chosen by the baseline approach is 0.06597 ($P = 0.03$). We can conclude that the classifier's output is indicative of publications' tech-transfer potential.
|
||||
|
||||
|
|
@ -69,7 +85,7 @@ Figure \ref{fig:histograms} shows the ratio of summary candidate sentences as pr
|
|||
\centering
|
||||
\includegraphics[width=0.85\linewidth]{figures/highlights-histograms.png}
|
||||
\captionsetup{width=.9\linewidth}
|
||||
\caption{Distribution of mean predicted summary candidate sentence counts in 4 categories. Category \texttt{A} corresponds to the most, while \texttt{D} to the least interesting papers based on mean user feedback. The sample size is 1406 (\texttt{D}=715, \texttt{C}=309, \texttt{B}=198, \texttt{A}=184). The histograms are on the same scale.}
|
||||
\caption{Distribution of mean predicted summary candidate sentence counts (refered to as \textit{highlights}) in 4 categories. Category \texttt{A} corresponds to the most, while \texttt{D} to the least interesting papers based on mean user feedback. The sample size is 1406 (\texttt{D}=715, \texttt{C}=309, \texttt{B}=198, \texttt{A}=184). The histograms are on the same scale.}
|
||||
\label{fig:histograms}
|
||||
\end{figure}
|
||||
|
||||
|
|
|
|||
|
|
@ -17,7 +17,7 @@ This is not the case for \textit{Log production predictions with the model's ver
|
|||
|
||||
\label{table:best-practices-1}
|
||||
{\renewcommand{\arraystretch}{1.2} % for the vertical padding
|
||||
\begin{tabular}{p{7cm}@{\hskip 0.5cm}l@{\hskip 0cm}c} \hline
|
||||
\begin{tabular}{P{7cm}@{\hskip 0.5cm}l@{\hskip 0cm}c} \hline
|
||||
|
||||
\textbf{Best practice} & \textbf{Implementation} & \textbf{Support} \\\hline
|
||||
Use sanity checks for all external data sources\textsuperscript{1} & \texttt{@parameter} & \checkmark \\\hline
|
||||
|
|
@ -52,7 +52,7 @@ Log production predictions with the model's version and input data\textsuperscri
|
|||
|
||||
\label{table:best-practices-2}
|
||||
{\renewcommand{\arraystretch}{1.2} % for the vertical padding
|
||||
\begin{tabular}{p{7cm}@{\hskip 0.5cm}l@{\hskip 0cm}c} \hline
|
||||
\begin{tabular}{P{7cm}@{\hskip 0.5cm}l@{\hskip 0cm}c} \hline
|
||||
|
||||
\textbf{Best practice} & \textbf{Implementation} & \textbf{Support} \\\hline
|
||||
Execute validation techniques: error rates and cross-validation\textsuperscript{2} & \texttt{*\_ground\_truth} & \checkmark \\\hline
|
||||
|
|
@ -103,13 +103,13 @@ The practitioners were first asked to fill out a questionnaire about their lates
|
|||
|
||||
The y-axis of Figure \ref{fig:adoption} is calculated by discarding the \textit{Not applicable} answers and projecting the 5-point Likert scale to a range from 0 to 1, which is subsequently averaged over all questions. The overall mean adoption rate/extent is just above 0.5, which equates to the \textit{Neither agree nor disagree} label. These data are in line with the findings of Serban et al. \cite{serban2020adoption}.
|
||||
|
||||
Because the survey's 15 questions were compiled from the \textit{Fully automated} rows of Tables \ref{table:best-practices-1} and \ref{table:best-practices-2}, that means that when using \textit{GreatAI}, they are all implemented automatically. Consequently, the adoption rate/extent is doubled immediately: this is the added value of \textit{GreatAI}\footnote{As explained earlier, measuring quality as a function of best practice count would be dubious. Thus, the achieved magnitude of the doubling is irrelevant; however, the direction of change is.}. Moreover, this provides further evidence for answering \textbf{RQ3} showing the extent of automatically implemented practices over non-\textit{GreatAI} deployments.
|
||||
Because the survey's 15 questions were compiled from the \textit{Fully automated} rows of Tables \ref{table:best-practices-1} and \ref{table:best-practices-2}, that means that when using \textit{GreatAI}, they are all implemented automatically. Consequently, the adoption rate/extent is doubled immediately just by wrapping the inference function with \texttt{@GreatAI.create}: this is the added value of \textit{GreatAI}\footnote{As explained earlier, measuring quality as a function of best practice count would be dubious. Thus, the achieved magnitude of the doubling is irrelevant; however, the direction of change is not.}. Moreover, this provides further evidence for answering \textbf{RQ3} showing the extent of automatically implemented practices over non-\textit{GreatAI} deployments.
|
||||
|
||||
\begin{figure}
|
||||
\centering
|
||||
\includegraphics[width=0.6\linewidth]{figures/best-practices.png}
|
||||
\captionsetup{width=.9\linewidth}
|
||||
\caption{Best practices adoption rate as a function of software engineering experience. The point sizes denote the practitioners' experience in Data Science (DS). The correlation between the axes is significant ($r_{Pearson} = 0.67$ with $p = 0.0033$).}
|
||||
\caption{Best practices adoption rate as a function of software engineering (SE) and Data Science (DS) experience. SE experience is shown on the horizontal axis, while the point sizes denote the practitioners' experience in DS. The correlation between the axes is significant ($r_{Pearson} = 0.67$ with $p = 0.0033$).}
|
||||
\label{fig:adoption}
|
||||
\end{figure}
|
||||
|
||||
|
|
@ -118,30 +118,31 @@ Because the survey's 15 questions were compiled from the \textit{Fully automated
|
|||
\begin{table}[H]
|
||||
\centering
|
||||
\captionsetup{width=.9\linewidth}
|
||||
\caption{Aggregated results of the TAM survey (sample size = 10) presented in Appendix \ref{appendix:questions}. The input values range from 1 to 7.}
|
||||
\caption{TAM survey (presented in Appendix \ref{appendix:questions}, sample size = 10) results per variable. The input values range from 1 to 7.}
|
||||
\label{table:tam}
|
||||
{\renewcommand{\arraystretch}{1.1} % for the vertical padding
|
||||
\begin{tabular}{|c|r|r|r|} \hline
|
||||
\begin{tabular}{|r|l|l|l|} \hline
|
||||
& \textbf{Perceived ease of use} & \textbf{Perceived utility} & \textbf{Intention to use} \\\hline
|
||||
\textbf{Median} & 5.750 & 6.375 & 6.250 \\\hline
|
||||
\textbf{Mean} & 5.450 & 6.125 & 5.950 \\\hline
|
||||
\textbf{Standard deviation} & 1.039 & 0.850 & 1.322 \\\hline
|
||||
\textbf{Median} & 5.8 & 6.4 & 6.3 \\\hline
|
||||
\textbf{Mean} & 5.5 & 6.1 & 6.0 \\\hline
|
||||
\textbf{Standard deviation} & 1.0 & 0.9 & 1.3 \\\hline
|
||||
\textbf{Cronbach's alpha} & 0.77 & 0.88 & 0.95 \\\hline
|
||||
\end{tabular}}
|
||||
\end{table}
|
||||
|
||||
Participants filled out a form (shown in Appendix \ref{appendix:questions}) after finishing their first deployment with \textit{GreatAI} to provide data for creating the technology acceptance model of the problem context. The survey contained 12 questions from 3 categories, which could be rated on a 7-point Likert scale. Following the methodology of \cite{cruz2019catalog}, the connections between the Perceived Utility (PU), Perceived Ease Of Use (PEOU), and Intention To Use (ITU) dimensions of TAM were analysed. Two statistically significant ($P \leq 0.05$) correlations were uncovered: between PU and ITU ($r_{Pearson} = 0.81$ with $p = 0.0048$); and PEOU and ITU ($r_{Pearson} = 0.80$ with $p = 0.0068$). Learning from the findings of prior case studies, it is reasonable to believe that both the \textit{perceived utility} and the \textit{perceived ease of use} play an equally important role in influencing professionals' \textit{intention to use} the deployment framework.
|
||||
Participants filled out a form (shown in Appendix \ref{appendix:questions}) after finishing their first deployment with \textit{GreatAI} to provide data for creating the technology acceptance model of the problem context. The survey contained ten questions from three categories, which could be rated on a 7-point Likert scale. The summary of the answers is presented in Table \ref{table:tam}. The high Cronbach's alpha values indicate strong internal consistency \cite{nunnally1994psychometric} for each TAM dimension; thus, averaging the responses per category is semantically meaningful.
|
||||
|
||||
The summary of the answers is presented in Table \ref{table:tam}. The assessment of \textit{ease of use} lags behind the rest, but it is still quite high. It may be possible that PEOU would go up with further use. Nevertheless, the high \textit{perceived utility} implies that \textit{GreatAI} shows its value early on. This, combined with the correlations uncovered within the context's technology acceptance model, validates the hypothesis that focusing on good API design is just as necessary as providing practical features.
|
||||
Following the methodology of \cite{cruz2019catalog}, the connections between the Perceived Utility (PU), Perceived Ease Of Use (PEOU), and Intention To Use (ITU) dimensions of TAM were analysed. Two statistically significant ($P \leq 0.05$) correlations were uncovered: between PU and ITU ($r_{Pearson} = 0.81$ with $p = 0.0048$); and PEOU and ITU ($r_{Pearson} = 0.80$ with $p = 0.0068$). Learning from the findings of prior case studies, it is reasonable to believe that both the \textit{perceived utility} and the \textit{perceived ease of use} play an equally important role in influencing professionals' \textit{intention to use} the deployment framework.
|
||||
|
||||
The assessment of \textit{ease of use} lags behind the rest, but it is still quite high. It may be possible that PEOU would go up with further use. Nevertheless, the high \textit{perceived utility} implies that \textit{GreatAI} shows its value early on. This, combined with the correlations uncovered within the context's technology acceptance model, validates the hypothesis that focusing on good API design is just as necessary as providing practical features.
|
||||
|
||||
\subsection{Task solving \& exit interviews}
|
||||
|
||||
In order to give qualitative depth to the previously presented quantitative results, it is time to discuss the main segment of the interviews. The participants' backgrounds cover a vast and fascinating cross-section of industrial AI/ML: one of them researched market prediction models for the Hungarian State Treasury, but building an upcoming digital bank's core services, investigating companies' AI use as part of due diligence processes, intrusion detection from network packet traces, creating pose-recognition for people with disabilities, and predicting Sun activity at the European Space Agency are just some of the core activities they had been doing recently. Stemming from this diversity, these semi-structured interviews could be expected to provide valuable insights into the generalisability of \textit{GreatAI}.
|
||||
In order to give qualitative depth to the previously presented quantitative results, it is time to discuss the main segment of the interviews. The participants' backgrounds cover a vast and fascinating cross-section of industrial AI/ML.
|
||||
|
||||
First, the volunteers were asked to skim through the library's documentation beforehand, and they were also given a short verbal overview during the one-on-one sessions. This was followed by having them solve a prepared deployment task\footnote{Available at \href{https://github.com/schmelczer/great-ai-interview-task}{github.com/schmelczer/great-ai-interview-task}.}, which is a more straightforward instance of the AI development lifecycle presented in the \textit{GreatAI} tutorials. The training part of the task had already been done, and the participants only had to deploy a trained classifier. The interviews took approximately one and a half hours each.
|
||||
The financial sector was represented by a researcher working on market prediction models for the Hungarian State Treasury and two people building an upcoming digital bank's core services. Image processing contexts were illustrated by professionals predicting Sun activity at the European Space Agency and different ones creating pose-recognition at a startup for people with disabilities using 3D cameras. Moreover, investigating companies' AI use as part of due diligence processes and intrusion detection from network packet traces are just some of the other core activities the interviewees had been doing recently. Stemming from this diversity, these semi-structured interviews could be expected to provide valuable insights into the generalisability of \textit{GreatAI}.
|
||||
|
||||
The guidelines proposed by Halcomb et al. \cite{halcomb2006verbatim} are followed for collecting information from interviews and reporting it. It is a reflexive, iterative process which starts by recording participants (with their permission) and concurrent note-taking. Reflective journaling is immediately done post-interview, which is subsequently extended and revised by listening to the recordings. Afterwards, the gathered information is interpreted by applying the methodology of thematic analysis \cite{alhojailan2012thematic}.
|
||||
|
||||
Thematic analysis is an iterative qualitative investigation technique consisting of labelling, correlating, and structuring the central recurring topics raised during discussions. It has been successfully used in previous software engineering studies for extracting emergent patterns \cite{haakman2021ai,cruz2019catalog}. After labelling each aspect of the feedback, and two iterations of merging redundant or related topics, we end up with three overarching themes: \textit{Functionality}, \textit{API}, and \textit{Responsibility to adopt}. As we will soon see, these correspond to the \textit{perceived utility}, \textit{perceived ease of use}, and \textit{intetion to use} components of TAM moderately well.
|
||||
The methodology of Section \ref{section:interview-setup} was followed by applying reflective journaling and thematic analysis. After labelling each aspect of the feedback, and two iterations of merging redundant or related topics, we end up with three overarching themes: \textit{Functionality}, \textit{API}, and \textit{Responsibility to adopt}. As we will soon see, these correspond to the \textit{perceived utility}, \textit{perceived ease of use}, and \textit{intetion to use} components of TAM moderately well.
|
||||
|
||||
\paragraph{Functionality} The library's feature-set was complimented during most interviews, with one participant noting that, although the overall number of features is relatively small, most of them are utilised in most cases. Similarly, the \texttt{utilities} submodule was appreciated for helping greatly in the interview task, but non-NLP researchers noted its likely inadequacy for their area. Still, they would like to see ``bundle'' or ``toolbox''-style modules for their fields because it would save them from a lot of copy-pasting.
|
||||
|
||||
|
|
@ -181,13 +182,15 @@ Secondly, the survey answers and, in general, the interviewees may be subject to
|
|||
|
||||
\section{Future work}
|
||||
|
||||
The primary purpose of the library was to serve as a proxy through which its design decisions could be tested and evaluated in their practical context. For this reason, its design aimed to be a proof-of-principle for validating hypotheses and answering research questions. After successfully doing that, it has been turned into a practical software library suitable for production-use\footnote{\href{https://pypi.org/project/great-ai/}{pypi.org/project/great-ai}}.
|
||||
The primary purpose of the library was to serve as a proxy through which its design decisions could be tested and evaluated in their practical context. For this reason, its design aimed to be a proof-of-principle for validating hypotheses and answering research questions. After successfully doing that, it has been turned into a practical software library suitable for production-use\footnote{Available at \href{https://pypi.org/project/great-ai/}{pypi.org/project/great-ai} and \href{https://hub.docker.com/repository/docker/schmelczera/great-ai}{hub.docker.com/repository/docker/schmelczera/great-ai}.}.
|
||||
|
||||
The library's main limitations come from its bias toward NLP deployments. This is not unreasonable given the design's explorative nature and the context of the case studies. Nevertheless, future work must focus on introducing and balancing support for many more fields' deployments. Although \textit{GreatAI} has already proved its utility, it has also shown that generalising and extending its functionality would be worthwhile. Therefore, many potential improvements are presented below, primarily from the needs arisen during the exit interviews.
|
||||
|
||||
\subsection{More AI/ML fields}
|
||||
\subsection{More ML fields}
|
||||
|
||||
The cases presented in Chapter \ref{chapter:case} revolved around NLP. This, of course, heavily influenced the design process. The two most notable effects can be found in the REST API's \texttt{/predict} endpoint and some \texttt{utilities} functions. The former is streamlined to accept JSON-compatible data, while the latter gives robust feature extraction support for only textual input. Supporting the easy, direct upload of larger non-JSON files --- e.g. by saving them to S3 and showing a preview of them on the Dashboard's trace table --- and extending \texttt{utilities} to handle multimedia formats should be sufficient for counteracting the NLP bias. Hence, widely expanding the scope of applicability of \textit{GreatAI}. As we have seen in Section \ref{section:architecture}, the architecture is otherwise adequately general; therefore, incremental extensions can be applied.
|
||||
The cases presented in Chapter \ref{chapter:case} revolved around NLP. This, of course, heavily influenced the design process. The two most notable effects can be found in the REST API's \texttt{/predict} endpoint and some \texttt{utilities} functions. The former is streamlined to accept JSON-compatible data (which caters to textual and tabular data), while the latter gives robust feature extraction support only for textual input. However, sound, image, and video are also widely taken as input. Furthermore, with the rise of multimodal models \cite{gao2020survey}, even different combinations of them may be simultaneously taken as input.
|
||||
|
||||
Supporting the easy, direct upload of larger non-JSON files --- e.g. by saving them to S3 and showing a preview of them on the Dashboard's traces table --- and extending \texttt{utilities} to handle multimedia formats should be sufficient for counteracting the NLP bias. Hence, widely expanding the scope of applicability of \textit{GreatAI}. As we have seen in Section \ref{section:architecture}, the architecture is otherwise adequately general; therefore, incremental extensions can be applied.
|
||||
|
||||
\subsection{More best practices}
|
||||
|
||||
|
|
|
|||
|
|
@ -24,9 +24,9 @@ Similarly to the approach of \cite{serban2020adoption}, participants are asked a
|
|||
\item Support asynchronous top-down chaining of models
|
||||
\end{enumerate}
|
||||
|
||||
\chapter{Technology acceptance model questionnaire} \label{appendix:questions}
|
||||
\chapter{TAM questionnaire} \label{appendix:questions}
|
||||
|
||||
Following the methodology for parsimonious TAM of Wu et al. \cite{wu2011user}, each statement can be rated on a 7-point Likert scale.
|
||||
Following the methodology for the parsimonious technology acceptance model of Wu et al. \cite{wu2011user}, each statement can be rated on a 7-point Likert scale.
|
||||
|
||||
\paragraph{Perceived usefulness (PU)}
|
||||
\begin{enumerate}
|
||||
|
|
|
|||
Binary file not shown.
|
Before Width: | Height: | Size: 29 KiB After Width: | Height: | Size: 42 KiB |
BIN
thesis/main.pdf
BIN
thesis/main.pdf
Binary file not shown.
|
|
@ -22,7 +22,12 @@
|
|||
left=2.5cm,
|
||||
right=2.5cm,
|
||||
top=2.5cm,
|
||||
bottom=3cm]{geometry}
|
||||
bottom=2.65cm]{geometry}
|
||||
\usepackage{array}
|
||||
\usepackage{ragged2e}
|
||||
|
||||
% Left-aligned fixed-width table column
|
||||
\newcolumntype{P}[1]{>{\RaggedRight\hspace{0pt}}p{#1}}
|
||||
|
||||
% Header & footer
|
||||
\pagestyle{fancy}
|
||||
|
|
|
|||
|
|
@ -530,7 +530,6 @@
|
|||
|
||||
@inproceedings{Chen_2016,
|
||||
doi = {10.1145/2939672.2939785},
|
||||
url = {https://doi.org/10.1145\%2F2939672.2939785},
|
||||
year = 2016,
|
||||
month = {aug},
|
||||
publisher = {{ACM}},
|
||||
|
|
@ -786,7 +785,6 @@ address = {New York, NY, USA},
|
|||
volume = {51},
|
||||
number = {10},
|
||||
issn = {0001-0782},
|
||||
url = {https://doi.org/10.1145/1400181.1400186},
|
||||
doi = {10.1145/1400181.1400186},
|
||||
abstract = {Are you ready for a personal energy meter?},
|
||||
journal = {Commun. ACM},
|
||||
|
|
@ -886,3 +884,21 @@ numpages = {3}
|
|||
year={2020},
|
||||
organization={IOP Publishing}
|
||||
}
|
||||
|
||||
@book{nunnally1994psychometric,
|
||||
title={Psychometric theory 3E},
|
||||
author={Nunnally, Jum C},
|
||||
year={1994},
|
||||
publisher={Tata McGraw-hill education}
|
||||
}
|
||||
|
||||
@article{gao2020survey,
|
||||
title={A survey on deep learning for multimodal data fusion},
|
||||
author={Gao, Jing and Li, Peng and Chen, Zhikui and Zhang, Jianing},
|
||||
journal={Neural Computation},
|
||||
volume={32},
|
||||
number={5},
|
||||
pages={829--864},
|
||||
year={2020},
|
||||
publisher={MIT Press One Rogers Street, Cambridge, MA 02142-1209, USA journals}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -13,7 +13,7 @@
|
|||
|
||||
|
||||
<link rel="icon" href="../../media/favicon.ico">
|
||||
<meta name="generator" content="mkdocs-1.3.1, mkdocs-material-8.4.1">
|
||||
<meta name="generator" content="mkdocs-1.3.1, mkdocs-material-8.4.2">
|
||||
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -13,7 +13,7 @@
|
|||
|
||||
|
||||
<link rel="icon" href="../media/favicon.ico">
|
||||
<meta name="generator" content="mkdocs-1.3.1, mkdocs-material-8.4.1">
|
||||
<meta name="generator" content="mkdocs-1.3.1, mkdocs-material-8.4.2">
|
||||
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -13,7 +13,7 @@
|
|||
|
||||
|
||||
<link rel="icon" href="../../media/favicon.ico">
|
||||
<meta name="generator" content="mkdocs-1.3.1, mkdocs-material-8.4.1">
|
||||
<meta name="generator" content="mkdocs-1.3.1, mkdocs-material-8.4.2">
|
||||
|
||||
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue