Deployed ea540e8 with MkDocs version: 1.3.0
This commit is contained in:
parent
2c0fafb925
commit
54400a8829
52 changed files with 14526 additions and 502 deletions
104
404.html
104
404.html
|
|
@ -9,7 +9,7 @@
|
||||||
|
|
||||||
|
|
||||||
<link rel="icon" href="/assets/images/favicon.png">
|
<link rel="icon" href="/assets/images/favicon.png">
|
||||||
<meta name="generator" content="mkdocs-1.3.0, mkdocs-material-8.3.8">
|
<meta name="generator" content="mkdocs-1.3.0, mkdocs-material-8.3.9">
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
@ -49,13 +49,17 @@
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
<body dir="ltr" data-md-color-scheme="" data-md-color-primary="none" data-md-color-accent="none">
|
|
||||||
|
<body dir="ltr" data-md-color-scheme="default" data-md-color-primary="" data-md-color-accent="">
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
<script>var palette=__md_get("__palette");if(palette&&"object"==typeof palette.color)for(var key of Object.keys(palette.color))document.body.setAttribute("data-md-color-"+key,palette.color[key])</script>
|
||||||
|
|
||||||
<input class="md-toggle" data-md-toggle="drawer" type="checkbox" id="__drawer" autocomplete="off">
|
<input class="md-toggle" data-md-toggle="drawer" type="checkbox" id="__drawer" autocomplete="off">
|
||||||
<input class="md-toggle" data-md-toggle="search" type="checkbox" id="__search" autocomplete="off">
|
<input class="md-toggle" data-md-toggle="search" type="checkbox" id="__search" autocomplete="off">
|
||||||
<label class="md-overlay" for="__drawer"></label>
|
<label class="md-overlay" for="__drawer"></label>
|
||||||
|
|
@ -97,9 +101,76 @@
|
||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
|
<form class="md-header__option" data-md-component="palette">
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
<input class="md-option" data-md-color-media="" data-md-color-scheme="default" data-md-color-primary="" data-md-color-accent="" aria-label="Switch to dark mode" type="radio" name="__palette" id="__palette_1">
|
||||||
|
|
||||||
|
<label class="md-header__button md-icon" title="Switch to dark mode" for="__palette_2" hidden>
|
||||||
|
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M12 8a4 4 0 0 0-4 4 4 4 0 0 0 4 4 4 4 0 0 0 4-4 4 4 0 0 0-4-4m0 10a6 6 0 0 1-6-6 6 6 0 0 1 6-6 6 6 0 0 1 6 6 6 6 0 0 1-6 6m8-9.31V4h-4.69L12 .69 8.69 4H4v4.69L.69 12 4 15.31V20h4.69L12 23.31 15.31 20H20v-4.69L23.31 12 20 8.69Z"/></svg>
|
||||||
|
</label>
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
<input class="md-option" data-md-color-media="" data-md-color-scheme="slate" data-md-color-primary="" data-md-color-accent="" aria-label="Switch to light mode" type="radio" name="__palette" id="__palette_2">
|
||||||
|
|
||||||
|
<label class="md-header__button md-icon" title="Switch to light mode" for="__palette_1" hidden>
|
||||||
|
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M12 18c-.89 0-1.74-.2-2.5-.55C11.56 16.5 13 14.42 13 12c0-2.42-1.44-4.5-3.5-5.45C10.26 6.2 11.11 6 12 6a6 6 0 0 1 6 6 6 6 0 0 1-6 6m8-9.31V4h-4.69L12 .69 8.69 4H4v4.69L.69 12 4 15.31V20h4.69L12 23.31 15.31 20H20v-4.69L23.31 12 20 8.69Z"/></svg>
|
||||||
|
</label>
|
||||||
|
|
||||||
|
|
||||||
|
</form>
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
<label class="md-header__button md-icon" for="__search">
|
||||||
|
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M9.5 3A6.5 6.5 0 0 1 16 9.5c0 1.61-.59 3.09-1.56 4.23l.27.27h.79l5 5-1.5 1.5-5-5v-.79l-.27-.27A6.516 6.516 0 0 1 9.5 16 6.5 6.5 0 0 1 3 9.5 6.5 6.5 0 0 1 9.5 3m0 2C7 5 5 7 5 9.5S7 14 9.5 14 14 12 14 9.5 12 5 9.5 5Z"/></svg>
|
||||||
|
</label>
|
||||||
|
<div class="md-search" data-md-component="search" role="dialog">
|
||||||
|
<label class="md-search__overlay" for="__search"></label>
|
||||||
|
<div class="md-search__inner" role="search">
|
||||||
|
<form class="md-search__form" name="search">
|
||||||
|
<input type="text" class="md-search__input" name="query" aria-label="Search" placeholder="Search" autocapitalize="off" autocorrect="off" autocomplete="off" spellcheck="false" data-md-component="search-query" required>
|
||||||
|
<label class="md-search__icon md-icon" for="__search">
|
||||||
|
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M9.5 3A6.5 6.5 0 0 1 16 9.5c0 1.61-.59 3.09-1.56 4.23l.27.27h.79l5 5-1.5 1.5-5-5v-.79l-.27-.27A6.516 6.516 0 0 1 9.5 16 6.5 6.5 0 0 1 3 9.5 6.5 6.5 0 0 1 9.5 3m0 2C7 5 5 7 5 9.5S7 14 9.5 14 14 12 14 9.5 12 5 9.5 5Z"/></svg>
|
||||||
|
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M20 11v2H8l5.5 5.5-1.42 1.42L4.16 12l7.92-7.92L13.5 5.5 8 11h12Z"/></svg>
|
||||||
|
</label>
|
||||||
|
<nav class="md-search__options" aria-label="Search">
|
||||||
|
|
||||||
|
<button type="reset" class="md-search__icon md-icon" aria-label="Clear" tabindex="-1">
|
||||||
|
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M19 6.41 17.59 5 12 10.59 6.41 5 5 6.41 10.59 12 5 17.59 6.41 19 12 13.41 17.59 19 19 17.59 13.41 12 19 6.41Z"/></svg>
|
||||||
|
</button>
|
||||||
|
</nav>
|
||||||
|
|
||||||
|
</form>
|
||||||
|
<div class="md-search__output">
|
||||||
|
<div class="md-search__scrollwrap" data-md-scrollfix>
|
||||||
|
<div class="md-search-result" data-md-component="search-result">
|
||||||
|
<div class="md-search-result__meta">
|
||||||
|
Initializing search
|
||||||
|
</div>
|
||||||
|
<ol class="md-search-result__list"></ol>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
|
||||||
|
<div class="md-header__source">
|
||||||
|
<a href="https://github.com/schmelczer/great-ai" title="Go to repository" class="md-source" data-md-component="source">
|
||||||
|
<div class="md-source__icon md-icon">
|
||||||
|
|
||||||
|
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 448 512"><!--! Font Awesome Free 6.1.1 by @fontawesome - https://fontawesome.com License - https://fontawesome.com/license/free (Icons: CC BY 4.0, Fonts: SIL OFL 1.1, Code: MIT License) Copyright 2022 Fonticons, Inc.--><path d="M439.55 236.05 244 40.45a28.87 28.87 0 0 0-40.81 0l-40.66 40.63 51.52 51.52c27.06-9.14 52.68 16.77 43.39 43.68l49.66 49.66c34.23-11.8 61.18 31 35.47 56.69-26.49 26.49-70.21-2.87-56-37.34L240.22 199v121.85c25.3 12.54 22.26 41.85 9.08 55a34.34 34.34 0 0 1-48.55 0c-17.57-17.6-11.07-46.91 11.25-56v-123c-20.8-8.51-24.6-30.74-18.64-45L142.57 101 8.45 235.14a28.86 28.86 0 0 0 0 40.81l195.61 195.6a28.86 28.86 0 0 0 40.8 0l194.69-194.69a28.86 28.86 0 0 0 0-40.81z"/></svg>
|
||||||
|
</div>
|
||||||
|
<div class="md-source__repository">
|
||||||
|
GitHub
|
||||||
|
</div>
|
||||||
|
</a>
|
||||||
|
</div>
|
||||||
|
|
||||||
</nav>
|
</nav>
|
||||||
|
|
||||||
</header>
|
</header>
|
||||||
|
|
@ -133,6 +204,18 @@
|
||||||
GreatAI documentation
|
GreatAI documentation
|
||||||
</label>
|
</label>
|
||||||
|
|
||||||
|
<div class="md-nav__source">
|
||||||
|
<a href="https://github.com/schmelczer/great-ai" title="Go to repository" class="md-source" data-md-component="source">
|
||||||
|
<div class="md-source__icon md-icon">
|
||||||
|
|
||||||
|
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 448 512"><!--! Font Awesome Free 6.1.1 by @fontawesome - https://fontawesome.com License - https://fontawesome.com/license/free (Icons: CC BY 4.0, Fonts: SIL OFL 1.1, Code: MIT License) Copyright 2022 Fonticons, Inc.--><path d="M439.55 236.05 244 40.45a28.87 28.87 0 0 0-40.81 0l-40.66 40.63 51.52 51.52c27.06-9.14 52.68 16.77 43.39 43.68l49.66 49.66c34.23-11.8 61.18 31 35.47 56.69-26.49 26.49-70.21-2.87-56-37.34L240.22 199v121.85c25.3 12.54 22.26 41.85 9.08 55a34.34 34.34 0 0 1-48.55 0c-17.57-17.6-11.07-46.91 11.25-56v-123c-20.8-8.51-24.6-30.74-18.64-45L142.57 101 8.45 235.14a28.86 28.86 0 0 0 0 40.81l195.61 195.6a28.86 28.86 0 0 0 40.8 0l194.69-194.69a28.86 28.86 0 0 0 0-40.81z"/></svg>
|
||||||
|
</div>
|
||||||
|
<div class="md-source__repository">
|
||||||
|
GitHub
|
||||||
|
</div>
|
||||||
|
</a>
|
||||||
|
</div>
|
||||||
|
|
||||||
<ul class="md-nav__list" data-md-scrollfix>
|
<ul class="md-nav__list" data-md-scrollfix>
|
||||||
|
|
||||||
|
|
||||||
|
|
@ -210,6 +293,21 @@
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
<li class="md-nav__item">
|
||||||
|
<a href="/simple-mag/train/" class="md-nav__link">
|
||||||
|
Train a field of study (domain) classification of sentences on the MAG dataset
|
||||||
|
</a>
|
||||||
|
</li>
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
</ul>
|
</ul>
|
||||||
</nav>
|
</nav>
|
||||||
</div>
|
</div>
|
||||||
|
|
@ -271,7 +369,7 @@
|
||||||
<script id="__config" type="application/json">{"base": "/", "features": [], "search": "/assets/javascripts/workers/search.b97dbffb.min.js", "translations": {"clipboard.copied": "Copied to clipboard", "clipboard.copy": "Copy to clipboard", "search.config.lang": "en", "search.config.pipeline": "trimmer, stopWordFilter", "search.config.separator": "[\\s\\-]+", "search.placeholder": "Search", "search.result.more.one": "1 more on this page", "search.result.more.other": "# more on this page", "search.result.none": "No matching documents", "search.result.one": "1 matching document", "search.result.other": "# matching documents", "search.result.placeholder": "Type to start searching", "search.result.term.missing": "Missing", "select.version.title": "Select version"}}</script>
|
<script id="__config" type="application/json">{"base": "/", "features": [], "search": "/assets/javascripts/workers/search.b97dbffb.min.js", "translations": {"clipboard.copied": "Copied to clipboard", "clipboard.copy": "Copy to clipboard", "search.config.lang": "en", "search.config.pipeline": "trimmer, stopWordFilter", "search.config.separator": "[\\s\\-]+", "search.placeholder": "Search", "search.result.more.one": "1 more on this page", "search.result.more.other": "# more on this page", "search.result.none": "No matching documents", "search.result.one": "1 matching document", "search.result.other": "# matching documents", "search.result.placeholder": "Type to start searching", "search.result.term.missing": "Missing", "select.version.title": "Select version"}}</script>
|
||||||
|
|
||||||
|
|
||||||
<script src="/assets/javascripts/bundle.0238f547.min.js"></script>
|
<script src="/assets/javascripts/bundle.6c7ad80a.min.js"></script>
|
||||||
|
|
||||||
|
|
||||||
</body>
|
</body>
|
||||||
|
|
|
||||||
Binary file not shown.
Binary file not shown.
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
|
|
@ -9,7 +9,7 @@
|
||||||
|
|
||||||
|
|
||||||
<link rel="icon" href="../assets/images/favicon.png">
|
<link rel="icon" href="../assets/images/favicon.png">
|
||||||
<meta name="generator" content="mkdocs-1.3.0, mkdocs-material-8.3.8">
|
<meta name="generator" content="mkdocs-1.3.0, mkdocs-material-8.3.9">
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
@ -49,13 +49,17 @@
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
<body dir="ltr" data-md-color-scheme="" data-md-color-primary="none" data-md-color-accent="none">
|
|
||||||
|
<body dir="ltr" data-md-color-scheme="default" data-md-color-primary="" data-md-color-accent="">
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
<script>var palette=__md_get("__palette");if(palette&&"object"==typeof palette.color)for(var key of Object.keys(palette.color))document.body.setAttribute("data-md-color-"+key,palette.color[key])</script>
|
||||||
|
|
||||||
<input class="md-toggle" data-md-toggle="drawer" type="checkbox" id="__drawer" autocomplete="off">
|
<input class="md-toggle" data-md-toggle="drawer" type="checkbox" id="__drawer" autocomplete="off">
|
||||||
<input class="md-toggle" data-md-toggle="search" type="checkbox" id="__search" autocomplete="off">
|
<input class="md-toggle" data-md-toggle="search" type="checkbox" id="__search" autocomplete="off">
|
||||||
<label class="md-overlay" for="__drawer"></label>
|
<label class="md-overlay" for="__drawer"></label>
|
||||||
|
|
@ -97,9 +101,76 @@
|
||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
|
<form class="md-header__option" data-md-component="palette">
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
<input class="md-option" data-md-color-media="" data-md-color-scheme="default" data-md-color-primary="" data-md-color-accent="" aria-label="Switch to dark mode" type="radio" name="__palette" id="__palette_1">
|
||||||
|
|
||||||
|
<label class="md-header__button md-icon" title="Switch to dark mode" for="__palette_2" hidden>
|
||||||
|
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M12 8a4 4 0 0 0-4 4 4 4 0 0 0 4 4 4 4 0 0 0 4-4 4 4 0 0 0-4-4m0 10a6 6 0 0 1-6-6 6 6 0 0 1 6-6 6 6 0 0 1 6 6 6 6 0 0 1-6 6m8-9.31V4h-4.69L12 .69 8.69 4H4v4.69L.69 12 4 15.31V20h4.69L12 23.31 15.31 20H20v-4.69L23.31 12 20 8.69Z"/></svg>
|
||||||
|
</label>
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
<input class="md-option" data-md-color-media="" data-md-color-scheme="slate" data-md-color-primary="" data-md-color-accent="" aria-label="Switch to light mode" type="radio" name="__palette" id="__palette_2">
|
||||||
|
|
||||||
|
<label class="md-header__button md-icon" title="Switch to light mode" for="__palette_1" hidden>
|
||||||
|
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M12 18c-.89 0-1.74-.2-2.5-.55C11.56 16.5 13 14.42 13 12c0-2.42-1.44-4.5-3.5-5.45C10.26 6.2 11.11 6 12 6a6 6 0 0 1 6 6 6 6 0 0 1-6 6m8-9.31V4h-4.69L12 .69 8.69 4H4v4.69L.69 12 4 15.31V20h4.69L12 23.31 15.31 20H20v-4.69L23.31 12 20 8.69Z"/></svg>
|
||||||
|
</label>
|
||||||
|
|
||||||
|
|
||||||
|
</form>
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
<label class="md-header__button md-icon" for="__search">
|
||||||
|
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M9.5 3A6.5 6.5 0 0 1 16 9.5c0 1.61-.59 3.09-1.56 4.23l.27.27h.79l5 5-1.5 1.5-5-5v-.79l-.27-.27A6.516 6.516 0 0 1 9.5 16 6.5 6.5 0 0 1 3 9.5 6.5 6.5 0 0 1 9.5 3m0 2C7 5 5 7 5 9.5S7 14 9.5 14 14 12 14 9.5 12 5 9.5 5Z"/></svg>
|
||||||
|
</label>
|
||||||
|
<div class="md-search" data-md-component="search" role="dialog">
|
||||||
|
<label class="md-search__overlay" for="__search"></label>
|
||||||
|
<div class="md-search__inner" role="search">
|
||||||
|
<form class="md-search__form" name="search">
|
||||||
|
<input type="text" class="md-search__input" name="query" aria-label="Search" placeholder="Search" autocapitalize="off" autocorrect="off" autocomplete="off" spellcheck="false" data-md-component="search-query" required>
|
||||||
|
<label class="md-search__icon md-icon" for="__search">
|
||||||
|
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M9.5 3A6.5 6.5 0 0 1 16 9.5c0 1.61-.59 3.09-1.56 4.23l.27.27h.79l5 5-1.5 1.5-5-5v-.79l-.27-.27A6.516 6.516 0 0 1 9.5 16 6.5 6.5 0 0 1 3 9.5 6.5 6.5 0 0 1 9.5 3m0 2C7 5 5 7 5 9.5S7 14 9.5 14 14 12 14 9.5 12 5 9.5 5Z"/></svg>
|
||||||
|
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M20 11v2H8l5.5 5.5-1.42 1.42L4.16 12l7.92-7.92L13.5 5.5 8 11h12Z"/></svg>
|
||||||
|
</label>
|
||||||
|
<nav class="md-search__options" aria-label="Search">
|
||||||
|
|
||||||
|
<button type="reset" class="md-search__icon md-icon" aria-label="Clear" tabindex="-1">
|
||||||
|
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M19 6.41 17.59 5 12 10.59 6.41 5 5 6.41 10.59 12 5 17.59 6.41 19 12 13.41 17.59 19 19 17.59 13.41 12 19 6.41Z"/></svg>
|
||||||
|
</button>
|
||||||
|
</nav>
|
||||||
|
|
||||||
|
</form>
|
||||||
|
<div class="md-search__output">
|
||||||
|
<div class="md-search__scrollwrap" data-md-scrollfix>
|
||||||
|
<div class="md-search-result" data-md-component="search-result">
|
||||||
|
<div class="md-search-result__meta">
|
||||||
|
Initializing search
|
||||||
|
</div>
|
||||||
|
<ol class="md-search-result__list"></ol>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
|
||||||
|
<div class="md-header__source">
|
||||||
|
<a href="https://github.com/schmelczer/great-ai" title="Go to repository" class="md-source" data-md-component="source">
|
||||||
|
<div class="md-source__icon md-icon">
|
||||||
|
|
||||||
|
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 448 512"><!--! Font Awesome Free 6.1.1 by @fontawesome - https://fontawesome.com License - https://fontawesome.com/license/free (Icons: CC BY 4.0, Fonts: SIL OFL 1.1, Code: MIT License) Copyright 2022 Fonticons, Inc.--><path d="M439.55 236.05 244 40.45a28.87 28.87 0 0 0-40.81 0l-40.66 40.63 51.52 51.52c27.06-9.14 52.68 16.77 43.39 43.68l49.66 49.66c34.23-11.8 61.18 31 35.47 56.69-26.49 26.49-70.21-2.87-56-37.34L240.22 199v121.85c25.3 12.54 22.26 41.85 9.08 55a34.34 34.34 0 0 1-48.55 0c-17.57-17.6-11.07-46.91 11.25-56v-123c-20.8-8.51-24.6-30.74-18.64-45L142.57 101 8.45 235.14a28.86 28.86 0 0 0 0 40.81l195.61 195.6a28.86 28.86 0 0 0 40.8 0l194.69-194.69a28.86 28.86 0 0 0 0-40.81z"/></svg>
|
||||||
|
</div>
|
||||||
|
<div class="md-source__repository">
|
||||||
|
GitHub
|
||||||
|
</div>
|
||||||
|
</a>
|
||||||
|
</div>
|
||||||
|
|
||||||
</nav>
|
</nav>
|
||||||
|
|
||||||
</header>
|
</header>
|
||||||
|
|
@ -133,6 +204,18 @@
|
||||||
GreatAI documentation
|
GreatAI documentation
|
||||||
</label>
|
</label>
|
||||||
|
|
||||||
|
<div class="md-nav__source">
|
||||||
|
<a href="https://github.com/schmelczer/great-ai" title="Go to repository" class="md-source" data-md-component="source">
|
||||||
|
<div class="md-source__icon md-icon">
|
||||||
|
|
||||||
|
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 448 512"><!--! Font Awesome Free 6.1.1 by @fontawesome - https://fontawesome.com License - https://fontawesome.com/license/free (Icons: CC BY 4.0, Fonts: SIL OFL 1.1, Code: MIT License) Copyright 2022 Fonticons, Inc.--><path d="M439.55 236.05 244 40.45a28.87 28.87 0 0 0-40.81 0l-40.66 40.63 51.52 51.52c27.06-9.14 52.68 16.77 43.39 43.68l49.66 49.66c34.23-11.8 61.18 31 35.47 56.69-26.49 26.49-70.21-2.87-56-37.34L240.22 199v121.85c25.3 12.54 22.26 41.85 9.08 55a34.34 34.34 0 0 1-48.55 0c-17.57-17.6-11.07-46.91 11.25-56v-123c-20.8-8.51-24.6-30.74-18.64-45L142.57 101 8.45 235.14a28.86 28.86 0 0 0 0 40.81l195.61 195.6a28.86 28.86 0 0 0 40.8 0l194.69-194.69a28.86 28.86 0 0 0 0-40.81z"/></svg>
|
||||||
|
</div>
|
||||||
|
<div class="md-source__repository">
|
||||||
|
GitHub
|
||||||
|
</div>
|
||||||
|
</a>
|
||||||
|
</div>
|
||||||
|
|
||||||
<ul class="md-nav__list" data-md-scrollfix>
|
<ul class="md-nav__list" data-md-scrollfix>
|
||||||
|
|
||||||
|
|
||||||
|
|
@ -218,6 +301,21 @@
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
<li class="md-nav__item">
|
||||||
|
<a href="../simple-mag/train/" class="md-nav__link">
|
||||||
|
Train a field of study (domain) classification of sentences on the MAG dataset
|
||||||
|
</a>
|
||||||
|
</li>
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
</ul>
|
</ul>
|
||||||
</nav>
|
</nav>
|
||||||
</div>
|
</div>
|
||||||
|
|
@ -245,14 +343,33 @@
|
||||||
<div class="md-content" data-md-component="content">
|
<div class="md-content" data-md-component="content">
|
||||||
<article class="md-content__inner md-typeset">
|
<article class="md-content__inner md-typeset">
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
<a href="https://github.com/schmelczer/great-ai/edit/main/docs/explanation.md" title="Edit this page" class="md-content__button md-icon">
|
||||||
|
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M20.71 7.04c.39-.39.39-1.04 0-1.41l-2.34-2.34c-.37-.39-1.02-.39-1.41 0l-1.84 1.83 3.75 3.75M3 17.25V21h3.75L17.81 9.93l-3.75-3.75L3 17.25Z"/></svg>
|
||||||
|
</a>
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
<h1>Explanation</h1>
|
<h1>Explanation</h1>
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
<hr>
|
||||||
|
<div class="md-source-file">
|
||||||
|
<small>
|
||||||
|
|
||||||
|
Last update:
|
||||||
|
<span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-date">July 8, 2022</span>
|
||||||
|
|
||||||
|
|
||||||
|
</small>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
</article>
|
</article>
|
||||||
|
|
||||||
</div>
|
</div>
|
||||||
|
|
@ -281,6 +398,21 @@
|
||||||
</a>
|
</a>
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
<a href="../simple-mag/train/" class="md-footer__link md-footer__link--next" aria-label="Next: Train a field of study (domain) classification of sentences on the MAG dataset" rel="next">
|
||||||
|
<div class="md-footer__title">
|
||||||
|
<div class="md-ellipsis">
|
||||||
|
<span class="md-footer__direction">
|
||||||
|
Next
|
||||||
|
</span>
|
||||||
|
Train a field of study (domain) classification of sentences on the MAG dataset
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<div class="md-footer__button md-icon">
|
||||||
|
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M4 11v2h12l-5.5 5.5 1.42 1.42L19.84 12l-7.92-7.92L10.5 5.5 16 11H4Z"/></svg>
|
||||||
|
</div>
|
||||||
|
</a>
|
||||||
|
|
||||||
</nav>
|
</nav>
|
||||||
|
|
||||||
<div class="md-footer-meta md-typeset">
|
<div class="md-footer-meta md-typeset">
|
||||||
|
|
@ -306,7 +438,7 @@
|
||||||
<script id="__config" type="application/json">{"base": "..", "features": [], "search": "../assets/javascripts/workers/search.b97dbffb.min.js", "translations": {"clipboard.copied": "Copied to clipboard", "clipboard.copy": "Copy to clipboard", "search.config.lang": "en", "search.config.pipeline": "trimmer, stopWordFilter", "search.config.separator": "[\\s\\-]+", "search.placeholder": "Search", "search.result.more.one": "1 more on this page", "search.result.more.other": "# more on this page", "search.result.none": "No matching documents", "search.result.one": "1 matching document", "search.result.other": "# matching documents", "search.result.placeholder": "Type to start searching", "search.result.term.missing": "Missing", "select.version.title": "Select version"}}</script>
|
<script id="__config" type="application/json">{"base": "..", "features": [], "search": "../assets/javascripts/workers/search.b97dbffb.min.js", "translations": {"clipboard.copied": "Copied to clipboard", "clipboard.copy": "Copy to clipboard", "search.config.lang": "en", "search.config.pipeline": "trimmer, stopWordFilter", "search.config.separator": "[\\s\\-]+", "search.placeholder": "Search", "search.result.more.one": "1 more on this page", "search.result.more.other": "# more on this page", "search.result.none": "No matching documents", "search.result.one": "1 matching document", "search.result.other": "# matching documents", "search.result.placeholder": "Type to start searching", "search.result.term.missing": "Missing", "select.version.title": "Select version"}}</script>
|
||||||
|
|
||||||
|
|
||||||
<script src="../assets/javascripts/bundle.0238f547.min.js"></script>
|
<script src="../assets/javascripts/bundle.6c7ad80a.min.js"></script>
|
||||||
|
|
||||||
|
|
||||||
</body>
|
</body>
|
||||||
|
|
|
||||||
11
great_ai_example-main/Dockerfile
Normal file
11
great_ai_example-main/Dockerfile
Normal file
|
|
@ -0,0 +1,11 @@
|
||||||
|
FROM test
|
||||||
|
|
||||||
|
COPY requirements.txt .
|
||||||
|
RUN pip install --no-cache-dir ./great_ai
|
||||||
|
|
||||||
|
COPY mongo.ini .
|
||||||
|
COPY deploy.ipynb .
|
||||||
|
|
||||||
|
# RUN python3 -m large_file --backend s3 -secrets ~/.aws/credentials --cache my_first_file.json:3 my_second_file my_folder:0
|
||||||
|
|
||||||
|
CMD ["deploy.ipynb"]
|
||||||
23
great_ai_example-main/README.md
Normal file
23
great_ai_example-main/README.md
Normal file
|
|
@ -0,0 +1,23 @@
|
||||||
|
# Train a domain classifier on the [semantic scholar dataset](https://api.semanticscholar.org/corpus)
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
## Install
|
||||||
|
|
||||||
|
### System dependencies
|
||||||
|
|
||||||
|
Make sure you have `python3`, `pip`, and `venv` installed.
|
||||||
|
> On Ubuntu, execute: `sudo apt install -y python3 python3-pip python3-venv`
|
||||||
|
|
||||||
|
### Install dependencies
|
||||||
|
```sh
|
||||||
|
python3 -m venv --copies .env
|
||||||
|
source .env/bin/activate
|
||||||
|
|
||||||
|
pip install -r requirements.txt
|
||||||
|
```
|
||||||
|
## Execute
|
||||||
|
|
||||||
|
- [Part 1](src/data.ipynb)
|
||||||
|
- [Part 2](src/train.ipynb)
|
||||||
|
- [Part 3](src/deploy.ipynb)
|
||||||
1
great_ai_example-main/diagrams/scope-data.svg
Normal file
1
great_ai_example-main/diagrams/scope-data.svg
Normal file
File diff suppressed because one or more lines are too long
|
After Width: | Height: | Size: 14 KiB |
1
great_ai_example-main/diagrams/scope-deploy.svg
Normal file
1
great_ai_example-main/diagrams/scope-deploy.svg
Normal file
File diff suppressed because one or more lines are too long
|
After Width: | Height: | Size: 14 KiB |
1
great_ai_example-main/diagrams/scope-train.svg
Normal file
1
great_ai_example-main/diagrams/scope-train.svg
Normal file
File diff suppressed because one or more lines are too long
|
After Width: | Height: | Size: 14 KiB |
1
great_ai_example-main/diagrams/scope.svg
Normal file
1
great_ai_example-main/diagrams/scope.svg
Normal file
File diff suppressed because one or more lines are too long
|
After Width: | Height: | Size: 14 KiB |
31
great_ai_example-main/format.sh
Normal file
31
great_ai_example-main/format.sh
Normal file
|
|
@ -0,0 +1,31 @@
|
||||||
|
#!/bin/sh
|
||||||
|
|
||||||
|
set -e
|
||||||
|
|
||||||
|
echo "Installing dependencies if necessary"
|
||||||
|
python3 -m pip install --upgrade autoflake isort black[jupyter] mypy flake8
|
||||||
|
|
||||||
|
echo "Formatting and checking $1"
|
||||||
|
|
||||||
|
cd $1
|
||||||
|
|
||||||
|
echo Running autoflake
|
||||||
|
python3 -m autoflake --expand-star-imports --remove-all-unused-imports --ignore-init-module-imports --remove-unused-variables --in-place -r .
|
||||||
|
|
||||||
|
echo Running isort
|
||||||
|
python3 -m isort --profile black --skip .env .
|
||||||
|
|
||||||
|
echo Running black
|
||||||
|
python3 -m black . --exclude .env
|
||||||
|
|
||||||
|
if ls *.py 1> /dev/null 2>&1; then
|
||||||
|
echo Running mypy
|
||||||
|
python3 -m mypy --namespace-packages --ignore-missing-imports --install-types --non-interactive --disallow-untyped-defs --disallow-incomplete-defs --pretty --follow-imports=silent --exclude=external/ --exclude=/build/ .
|
||||||
|
fi
|
||||||
|
|
||||||
|
echo Running Flake8
|
||||||
|
python3 -m flake8 . --count --show-source --statistics --exclude=__init__.py,.env,external --ignore=E501,E722,E402,W503,E203
|
||||||
|
|
||||||
|
cd -
|
||||||
|
|
||||||
|
echo "Finished formatting"
|
||||||
467
great_ai_example-main/index.html
Normal file
467
great_ai_example-main/index.html
Normal file
|
|
@ -0,0 +1,467 @@
|
||||||
|
|
||||||
|
<!doctype html>
|
||||||
|
<html lang="en" class="no-js">
|
||||||
|
<head>
|
||||||
|
|
||||||
|
<meta charset="utf-8">
|
||||||
|
<meta name="viewport" content="width=device-width,initial-scale=1">
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
<link rel="icon" href="../assets/images/favicon.png">
|
||||||
|
<meta name="generator" content="mkdocs-1.3.0, mkdocs-material-8.3.9">
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
<title>Train a domain classifier on the [semantic scholar dataset](https://api.semanticscholar.org/corpus) - GreatAI documentation</title>
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
<link rel="stylesheet" href="../assets/stylesheets/main.1d29e8d0.min.css">
|
||||||
|
|
||||||
|
|
||||||
|
<link rel="stylesheet" href="../assets/stylesheets/palette.cbb835fc.min.css">
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
<link rel="preconnect" href="https://fonts.gstatic.com" crossorigin>
|
||||||
|
<link rel="stylesheet" href="https://fonts.googleapis.com/css?family=Roboto:300,300i,400,400i,700,700i%7CRoboto+Mono:400,400i,700,700i&display=fallback">
|
||||||
|
<style>:root{--md-text-font:"Roboto";--md-code-font:"Roboto Mono"}</style>
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
<link rel="stylesheet" href="../assets/_mkdocstrings.css">
|
||||||
|
|
||||||
|
<script>__md_scope=new URL("..",location),__md_get=(e,_=localStorage,t=__md_scope)=>JSON.parse(_.getItem(t.pathname+"."+e)),__md_set=(e,_,t=localStorage,a=__md_scope)=>{try{t.setItem(a.pathname+"."+e,JSON.stringify(_))}catch(e){}}</script>
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
</head>
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
<body dir="ltr" data-md-color-scheme="default" data-md-color-primary="" data-md-color-accent="">
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
<script>var palette=__md_get("__palette");if(palette&&"object"==typeof palette.color)for(var key of Object.keys(palette.color))document.body.setAttribute("data-md-color-"+key,palette.color[key])</script>
|
||||||
|
|
||||||
|
<input class="md-toggle" data-md-toggle="drawer" type="checkbox" id="__drawer" autocomplete="off">
|
||||||
|
<input class="md-toggle" data-md-toggle="search" type="checkbox" id="__search" autocomplete="off">
|
||||||
|
<label class="md-overlay" for="__drawer"></label>
|
||||||
|
<div data-md-component="skip">
|
||||||
|
|
||||||
|
|
||||||
|
<a href="#train-a-domain-classifier-on-the-semantic-scholar-dataset" class="md-skip">
|
||||||
|
Skip to content
|
||||||
|
</a>
|
||||||
|
|
||||||
|
</div>
|
||||||
|
<div data-md-component="announce">
|
||||||
|
|
||||||
|
</div>
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
<header class="md-header" data-md-component="header">
|
||||||
|
<nav class="md-header__inner md-grid" aria-label="Header">
|
||||||
|
<a href=".." title="GreatAI documentation" class="md-header__button md-logo" aria-label="GreatAI documentation" data-md-component="logo">
|
||||||
|
|
||||||
|
|
||||||
|
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M12 8a3 3 0 0 0 3-3 3 3 0 0 0-3-3 3 3 0 0 0-3 3 3 3 0 0 0 3 3m0 3.54C9.64 9.35 6.5 8 3 8v11c3.5 0 6.64 1.35 9 3.54 2.36-2.19 5.5-3.54 9-3.54V8c-3.5 0-6.64 1.35-9 3.54Z"/></svg>
|
||||||
|
|
||||||
|
</a>
|
||||||
|
<label class="md-header__button md-icon" for="__drawer">
|
||||||
|
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M3 6h18v2H3V6m0 5h18v2H3v-2m0 5h18v2H3v-2Z"/></svg>
|
||||||
|
</label>
|
||||||
|
<div class="md-header__title" data-md-component="header-title">
|
||||||
|
<div class="md-header__ellipsis">
|
||||||
|
<div class="md-header__topic">
|
||||||
|
<span class="md-ellipsis">
|
||||||
|
GreatAI documentation
|
||||||
|
</span>
|
||||||
|
</div>
|
||||||
|
<div class="md-header__topic" data-md-component="header-topic">
|
||||||
|
<span class="md-ellipsis">
|
||||||
|
|
||||||
|
Train a domain classifier on the [semantic scholar dataset](https://api.semanticscholar.org/corpus)
|
||||||
|
|
||||||
|
</span>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<form class="md-header__option" data-md-component="palette">
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
<input class="md-option" data-md-color-media="" data-md-color-scheme="default" data-md-color-primary="" data-md-color-accent="" aria-label="Switch to dark mode" type="radio" name="__palette" id="__palette_1">
|
||||||
|
|
||||||
|
<label class="md-header__button md-icon" title="Switch to dark mode" for="__palette_2" hidden>
|
||||||
|
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M12 8a4 4 0 0 0-4 4 4 4 0 0 0 4 4 4 4 0 0 0 4-4 4 4 0 0 0-4-4m0 10a6 6 0 0 1-6-6 6 6 0 0 1 6-6 6 6 0 0 1 6 6 6 6 0 0 1-6 6m8-9.31V4h-4.69L12 .69 8.69 4H4v4.69L.69 12 4 15.31V20h4.69L12 23.31 15.31 20H20v-4.69L23.31 12 20 8.69Z"/></svg>
|
||||||
|
</label>
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
<input class="md-option" data-md-color-media="" data-md-color-scheme="slate" data-md-color-primary="" data-md-color-accent="" aria-label="Switch to light mode" type="radio" name="__palette" id="__palette_2">
|
||||||
|
|
||||||
|
<label class="md-header__button md-icon" title="Switch to light mode" for="__palette_1" hidden>
|
||||||
|
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M12 18c-.89 0-1.74-.2-2.5-.55C11.56 16.5 13 14.42 13 12c0-2.42-1.44-4.5-3.5-5.45C10.26 6.2 11.11 6 12 6a6 6 0 0 1 6 6 6 6 0 0 1-6 6m8-9.31V4h-4.69L12 .69 8.69 4H4v4.69L.69 12 4 15.31V20h4.69L12 23.31 15.31 20H20v-4.69L23.31 12 20 8.69Z"/></svg>
|
||||||
|
</label>
|
||||||
|
|
||||||
|
|
||||||
|
</form>
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
<label class="md-header__button md-icon" for="__search">
|
||||||
|
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M9.5 3A6.5 6.5 0 0 1 16 9.5c0 1.61-.59 3.09-1.56 4.23l.27.27h.79l5 5-1.5 1.5-5-5v-.79l-.27-.27A6.516 6.516 0 0 1 9.5 16 6.5 6.5 0 0 1 3 9.5 6.5 6.5 0 0 1 9.5 3m0 2C7 5 5 7 5 9.5S7 14 9.5 14 14 12 14 9.5 12 5 9.5 5Z"/></svg>
|
||||||
|
</label>
|
||||||
|
<div class="md-search" data-md-component="search" role="dialog">
|
||||||
|
<label class="md-search__overlay" for="__search"></label>
|
||||||
|
<div class="md-search__inner" role="search">
|
||||||
|
<form class="md-search__form" name="search">
|
||||||
|
<input type="text" class="md-search__input" name="query" aria-label="Search" placeholder="Search" autocapitalize="off" autocorrect="off" autocomplete="off" spellcheck="false" data-md-component="search-query" required>
|
||||||
|
<label class="md-search__icon md-icon" for="__search">
|
||||||
|
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M9.5 3A6.5 6.5 0 0 1 16 9.5c0 1.61-.59 3.09-1.56 4.23l.27.27h.79l5 5-1.5 1.5-5-5v-.79l-.27-.27A6.516 6.516 0 0 1 9.5 16 6.5 6.5 0 0 1 3 9.5 6.5 6.5 0 0 1 9.5 3m0 2C7 5 5 7 5 9.5S7 14 9.5 14 14 12 14 9.5 12 5 9.5 5Z"/></svg>
|
||||||
|
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M20 11v2H8l5.5 5.5-1.42 1.42L4.16 12l7.92-7.92L13.5 5.5 8 11h12Z"/></svg>
|
||||||
|
</label>
|
||||||
|
<nav class="md-search__options" aria-label="Search">
|
||||||
|
|
||||||
|
<button type="reset" class="md-search__icon md-icon" aria-label="Clear" tabindex="-1">
|
||||||
|
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M19 6.41 17.59 5 12 10.59 6.41 5 5 6.41 10.59 12 5 17.59 6.41 19 12 13.41 17.59 19 19 17.59 13.41 12 19 6.41Z"/></svg>
|
||||||
|
</button>
|
||||||
|
</nav>
|
||||||
|
|
||||||
|
</form>
|
||||||
|
<div class="md-search__output">
|
||||||
|
<div class="md-search__scrollwrap" data-md-scrollfix>
|
||||||
|
<div class="md-search-result" data-md-component="search-result">
|
||||||
|
<div class="md-search-result__meta">
|
||||||
|
Initializing search
|
||||||
|
</div>
|
||||||
|
<ol class="md-search-result__list"></ol>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
|
||||||
|
<div class="md-header__source">
|
||||||
|
<a href="https://github.com/schmelczer/great-ai" title="Go to repository" class="md-source" data-md-component="source">
|
||||||
|
<div class="md-source__icon md-icon">
|
||||||
|
|
||||||
|
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 448 512"><!--! Font Awesome Free 6.1.1 by @fontawesome - https://fontawesome.com License - https://fontawesome.com/license/free (Icons: CC BY 4.0, Fonts: SIL OFL 1.1, Code: MIT License) Copyright 2022 Fonticons, Inc.--><path d="M439.55 236.05 244 40.45a28.87 28.87 0 0 0-40.81 0l-40.66 40.63 51.52 51.52c27.06-9.14 52.68 16.77 43.39 43.68l49.66 49.66c34.23-11.8 61.18 31 35.47 56.69-26.49 26.49-70.21-2.87-56-37.34L240.22 199v121.85c25.3 12.54 22.26 41.85 9.08 55a34.34 34.34 0 0 1-48.55 0c-17.57-17.6-11.07-46.91 11.25-56v-123c-20.8-8.51-24.6-30.74-18.64-45L142.57 101 8.45 235.14a28.86 28.86 0 0 0 0 40.81l195.61 195.6a28.86 28.86 0 0 0 40.8 0l194.69-194.69a28.86 28.86 0 0 0 0-40.81z"/></svg>
|
||||||
|
</div>
|
||||||
|
<div class="md-source__repository">
|
||||||
|
GitHub
|
||||||
|
</div>
|
||||||
|
</a>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
</nav>
|
||||||
|
|
||||||
|
</header>
|
||||||
|
|
||||||
|
<div class="md-container" data-md-component="container">
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
<main class="md-main" data-md-component="main">
|
||||||
|
<div class="md-main__inner md-grid">
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
<div class="md-sidebar md-sidebar--primary" data-md-component="sidebar" data-md-type="navigation" >
|
||||||
|
<div class="md-sidebar__scrollwrap">
|
||||||
|
<div class="md-sidebar__inner">
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
<nav class="md-nav md-nav--primary" aria-label="Navigation" data-md-level="0">
|
||||||
|
<label class="md-nav__title" for="__drawer">
|
||||||
|
<a href=".." title="GreatAI documentation" class="md-nav__button md-logo" aria-label="GreatAI documentation" data-md-component="logo">
|
||||||
|
|
||||||
|
|
||||||
|
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M12 8a3 3 0 0 0 3-3 3 3 0 0 0-3-3 3 3 0 0 0-3 3 3 3 0 0 0 3 3m0 3.54C9.64 9.35 6.5 8 3 8v11c3.5 0 6.64 1.35 9 3.54 2.36-2.19 5.5-3.54 9-3.54V8c-3.5 0-6.64 1.35-9 3.54Z"/></svg>
|
||||||
|
|
||||||
|
</a>
|
||||||
|
GreatAI documentation
|
||||||
|
</label>
|
||||||
|
|
||||||
|
<div class="md-nav__source">
|
||||||
|
<a href="https://github.com/schmelczer/great-ai" title="Go to repository" class="md-source" data-md-component="source">
|
||||||
|
<div class="md-source__icon md-icon">
|
||||||
|
|
||||||
|
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 448 512"><!--! Font Awesome Free 6.1.1 by @fontawesome - https://fontawesome.com License - https://fontawesome.com/license/free (Icons: CC BY 4.0, Fonts: SIL OFL 1.1, Code: MIT License) Copyright 2022 Fonticons, Inc.--><path d="M439.55 236.05 244 40.45a28.87 28.87 0 0 0-40.81 0l-40.66 40.63 51.52 51.52c27.06-9.14 52.68 16.77 43.39 43.68l49.66 49.66c34.23-11.8 61.18 31 35.47 56.69-26.49 26.49-70.21-2.87-56-37.34L240.22 199v121.85c25.3 12.54 22.26 41.85 9.08 55a34.34 34.34 0 0 1-48.55 0c-17.57-17.6-11.07-46.91 11.25-56v-123c-20.8-8.51-24.6-30.74-18.64-45L142.57 101 8.45 235.14a28.86 28.86 0 0 0 0 40.81l195.61 195.6a28.86 28.86 0 0 0 40.8 0l194.69-194.69a28.86 28.86 0 0 0 0-40.81z"/></svg>
|
||||||
|
</div>
|
||||||
|
<div class="md-source__repository">
|
||||||
|
GitHub
|
||||||
|
</div>
|
||||||
|
</a>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<ul class="md-nav__list" data-md-scrollfix>
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
<li class="md-nav__item">
|
||||||
|
<a href=".." class="md-nav__link">
|
||||||
|
Home
|
||||||
|
</a>
|
||||||
|
</li>
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
<li class="md-nav__item">
|
||||||
|
<a href="../tutorials/" class="md-nav__link">
|
||||||
|
Tutorials
|
||||||
|
</a>
|
||||||
|
</li>
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
<li class="md-nav__item">
|
||||||
|
<a href="../how-to-guides/" class="md-nav__link">
|
||||||
|
How-To Guides
|
||||||
|
</a>
|
||||||
|
</li>
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
<li class="md-nav__item">
|
||||||
|
<a href="../reference/" class="md-nav__link">
|
||||||
|
Reference
|
||||||
|
</a>
|
||||||
|
</li>
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
<li class="md-nav__item">
|
||||||
|
<a href="../explanation/" class="md-nav__link">
|
||||||
|
Explanation
|
||||||
|
</a>
|
||||||
|
</li>
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
<li class="md-nav__item">
|
||||||
|
<a href="../simple-mag/train/" class="md-nav__link">
|
||||||
|
Train a field of study (domain) classification of sentences on the MAG dataset
|
||||||
|
</a>
|
||||||
|
</li>
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
</ul>
|
||||||
|
</nav>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
<div class="md-sidebar md-sidebar--secondary" data-md-component="sidebar" data-md-type="toc" >
|
||||||
|
<div class="md-sidebar__scrollwrap">
|
||||||
|
<div class="md-sidebar__inner">
|
||||||
|
|
||||||
|
|
||||||
|
<nav class="md-nav md-nav--secondary" aria-label="Table of contents">
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
<label class="md-nav__title" for="__toc">
|
||||||
|
<span class="md-nav__icon md-icon"></span>
|
||||||
|
Table of contents
|
||||||
|
</label>
|
||||||
|
<ul class="md-nav__list" data-md-component="toc" data-md-scrollfix>
|
||||||
|
|
||||||
|
<li class="md-nav__item">
|
||||||
|
<a href="#install" class="md-nav__link">
|
||||||
|
Install
|
||||||
|
</a>
|
||||||
|
|
||||||
|
<nav class="md-nav" aria-label="Install">
|
||||||
|
<ul class="md-nav__list">
|
||||||
|
|
||||||
|
<li class="md-nav__item">
|
||||||
|
<a href="#system-dependencies" class="md-nav__link">
|
||||||
|
System dependencies
|
||||||
|
</a>
|
||||||
|
|
||||||
|
</li>
|
||||||
|
|
||||||
|
<li class="md-nav__item">
|
||||||
|
<a href="#install-dependencies" class="md-nav__link">
|
||||||
|
Install dependencies
|
||||||
|
</a>
|
||||||
|
|
||||||
|
</li>
|
||||||
|
|
||||||
|
</ul>
|
||||||
|
</nav>
|
||||||
|
|
||||||
|
</li>
|
||||||
|
|
||||||
|
<li class="md-nav__item">
|
||||||
|
<a href="#execute" class="md-nav__link">
|
||||||
|
Execute
|
||||||
|
</a>
|
||||||
|
|
||||||
|
</li>
|
||||||
|
|
||||||
|
</ul>
|
||||||
|
|
||||||
|
</nav>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
|
||||||
|
<div class="md-content" data-md-component="content">
|
||||||
|
<article class="md-content__inner md-typeset">
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
<a href="https://github.com/schmelczer/great-ai/edit/main/docs/great_ai_example-main/README.md" title="Edit this page" class="md-content__button md-icon">
|
||||||
|
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M20.71 7.04c.39-.39.39-1.04 0-1.41l-2.34-2.34c-.37-.39-1.02-.39-1.41 0l-1.84 1.83 3.75 3.75M3 17.25V21h3.75L17.81 9.93l-3.75-3.75L3 17.25Z"/></svg>
|
||||||
|
</a>
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
<h1 id="train-a-domain-classifier-on-the-semantic-scholar-dataset">Train a domain classifier on the <a href="https://api.semanticscholar.org/corpus">semantic scholar dataset</a></h1>
|
||||||
|
<p><img alt="steps" src="diagrams/scope.svg" /></p>
|
||||||
|
<h2 id="install">Install</h2>
|
||||||
|
<h3 id="system-dependencies">System dependencies</h3>
|
||||||
|
<p>Make sure you have <code>python3</code>, <code>pip</code>, and <code>venv</code> installed.</p>
|
||||||
|
<blockquote>
|
||||||
|
<p>On Ubuntu, execute: <code>sudo apt install -y python3 python3-pip python3-venv</code></p>
|
||||||
|
</blockquote>
|
||||||
|
<h3 id="install-dependencies">Install dependencies</h3>
|
||||||
|
<pre><code class="language-sh">python3 -m venv --copies .env
|
||||||
|
source .env/bin/activate
|
||||||
|
|
||||||
|
pip install -r requirements.txt
|
||||||
|
</code></pre>
|
||||||
|
<h2 id="execute">Execute</h2>
|
||||||
|
<ul>
|
||||||
|
<li><a href="src/data/">Part 1</a></li>
|
||||||
|
<li><a href="src/train/">Part 2</a></li>
|
||||||
|
<li><a href="src/deploy/">Part 3</a></li>
|
||||||
|
</ul>
|
||||||
|
|
||||||
|
<hr>
|
||||||
|
<div class="md-source-file">
|
||||||
|
<small>
|
||||||
|
|
||||||
|
Last update:
|
||||||
|
<span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-date">July 8, 2022</span>
|
||||||
|
|
||||||
|
|
||||||
|
</small>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
</article>
|
||||||
|
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
</main>
|
||||||
|
|
||||||
|
<footer class="md-footer">
|
||||||
|
|
||||||
|
<div class="md-footer-meta md-typeset">
|
||||||
|
<div class="md-footer-meta__inner md-grid">
|
||||||
|
<div class="md-copyright">
|
||||||
|
|
||||||
|
|
||||||
|
Made with
|
||||||
|
<a href="https://squidfunk.github.io/mkdocs-material/" target="_blank" rel="noopener">
|
||||||
|
Material for MkDocs
|
||||||
|
</a>
|
||||||
|
|
||||||
|
</div>
|
||||||
|
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</footer>
|
||||||
|
|
||||||
|
</div>
|
||||||
|
<div class="md-dialog" data-md-component="dialog">
|
||||||
|
<div class="md-dialog__inner md-typeset"></div>
|
||||||
|
</div>
|
||||||
|
<script id="__config" type="application/json">{"base": "..", "features": [], "search": "../assets/javascripts/workers/search.b97dbffb.min.js", "translations": {"clipboard.copied": "Copied to clipboard", "clipboard.copy": "Copy to clipboard", "search.config.lang": "en", "search.config.pipeline": "trimmer, stopWordFilter", "search.config.separator": "[\\s\\-]+", "search.placeholder": "Search", "search.result.more.one": "1 more on this page", "search.result.more.other": "# more on this page", "search.result.none": "No matching documents", "search.result.one": "1 matching document", "search.result.other": "# matching documents", "search.result.placeholder": "Type to start searching", "search.result.term.missing": "Missing", "select.version.title": "Select version"}}</script>
|
||||||
|
|
||||||
|
|
||||||
|
<script src="../assets/javascripts/bundle.6c7ad80a.min.js"></script>
|
||||||
|
|
||||||
|
|
||||||
|
</body>
|
||||||
|
</html>
|
||||||
4
great_ai_example-main/requirements.txt
Normal file
4
great_ai_example-main/requirements.txt
Normal file
|
|
@ -0,0 +1,4 @@
|
||||||
|
great_ai
|
||||||
|
notebook
|
||||||
|
nbformat
|
||||||
|
nbconvert
|
||||||
265
great_ai_example-main/src/data/data.ipynb
Normal file
265
great_ai_example-main/src/data/data.ipynb
Normal file
|
|
@ -0,0 +1,265 @@
|
||||||
|
{
|
||||||
|
"cells": [
|
||||||
|
{
|
||||||
|
"cell_type": "markdown",
|
||||||
|
"metadata": {},
|
||||||
|
"source": [
|
||||||
|
"# Train a domain classifier on the [semantic scholar dataset](https://api.semanticscholar.org/corpus)\n",
|
||||||
|
"> Part 1: obtain and clean data\n",
|
||||||
|
"\n",
|
||||||
|
"\n",
|
||||||
|
"> The blue boxes show the steps implemented in this notebook.\n",
|
||||||
|
"\n",
|
||||||
|
"### Extract\n",
|
||||||
|
"\n",
|
||||||
|
"This can be achieved by downloading a public dataset (such as in this case), or by having a Data Engineer setup and give us access to the organisation's data.\n",
|
||||||
|
"\n",
|
||||||
|
"In this case, we download the semantic scholar dataset from a public S3 bucket."
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": 1,
|
||||||
|
"metadata": {},
|
||||||
|
"outputs": [],
|
||||||
|
"source": [
|
||||||
|
"MAX_CHUNK_COUNT = 1"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": 2,
|
||||||
|
"metadata": {},
|
||||||
|
"outputs": [
|
||||||
|
{
|
||||||
|
"data": {
|
||||||
|
"text/plain": [
|
||||||
|
"'Processing 1 out of the 6002 available chunks'"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"execution_count": 2,
|
||||||
|
"metadata": {},
|
||||||
|
"output_type": "execute_result"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"source": [
|
||||||
|
"import urllib.request\n",
|
||||||
|
"from random import shuffle\n",
|
||||||
|
"\n",
|
||||||
|
"manifest = (\n",
|
||||||
|
" urllib.request.urlopen(\n",
|
||||||
|
" \"https://s3-us-west-2.amazonaws.com/ai2-s2-research-public/open-corpus/2022-02-01/manifest.txt\"\n",
|
||||||
|
" )\n",
|
||||||
|
" .read()\n",
|
||||||
|
" .decode()\n",
|
||||||
|
") # a list of available chunks separated by '\\n' characters\n",
|
||||||
|
"\n",
|
||||||
|
"lines = manifest.split()\n",
|
||||||
|
"shuffle(lines)\n",
|
||||||
|
"chunks = lines[:MAX_CHUNK_COUNT]\n",
|
||||||
|
"\n",
|
||||||
|
"f\"Processing {len(chunks)} out of the {len(manifest.split())} available chunks\""
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "markdown",
|
||||||
|
"metadata": {},
|
||||||
|
"source": [
|
||||||
|
"### Transform\n",
|
||||||
|
"\n",
|
||||||
|
"- Filter out non-English abstracts using `great_ai.utilities.predict_language`\n",
|
||||||
|
"- Project it to only keep the necessary components (text and labels), clean the textual content using `great_ai.utilities.clean`\n",
|
||||||
|
"- We will speed up processing using `great_ai.utilities.parallel_map`."
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": 3,
|
||||||
|
"metadata": {},
|
||||||
|
"outputs": [
|
||||||
|
{
|
||||||
|
"name": "stdout",
|
||||||
|
"output_type": "stream",
|
||||||
|
"text": [
|
||||||
|
"Spacy model en_core_web_sm not found locally, downloading...\n",
|
||||||
|
"Collecting en-core-web-sm==3.3.0\n",
|
||||||
|
" Downloading https://github.com/explosion/spacy-models/releases/download/en_core_web_sm-3.3.0/en_core_web_sm-3.3.0-py3-none-any.whl (12.8 MB)\n",
|
||||||
|
" ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 12.8/12.8 MB 3.6 MB/s eta 0:00:00\n",
|
||||||
|
"Requirement already satisfied: spacy<3.4.0,>=3.3.0.dev0 in ./.env/lib/python3.10/site-packages (from en-core-web-sm==3.3.0) (3.3.1)\n",
|
||||||
|
"Requirement already satisfied: tqdm<5.0.0,>=4.38.0 in ./.env/lib/python3.10/site-packages (from spacy<3.4.0,>=3.3.0.dev0->en-core-web-sm==3.3.0) (4.64.0)\n",
|
||||||
|
"Requirement already satisfied: wasabi<1.1.0,>=0.9.1 in ./.env/lib/python3.10/site-packages (from spacy<3.4.0,>=3.3.0.dev0->en-core-web-sm==3.3.0) (0.9.1)\n",
|
||||||
|
"Requirement already satisfied: srsly<3.0.0,>=2.4.3 in ./.env/lib/python3.10/site-packages (from spacy<3.4.0,>=3.3.0.dev0->en-core-web-sm==3.3.0) (2.4.3)\n",
|
||||||
|
"Requirement already satisfied: typer<0.5.0,>=0.3.0 in ./.env/lib/python3.10/site-packages (from spacy<3.4.0,>=3.3.0.dev0->en-core-web-sm==3.3.0) (0.4.1)\n",
|
||||||
|
"Requirement already satisfied: setuptools in ./.env/lib/python3.10/site-packages (from spacy<3.4.0,>=3.3.0.dev0->en-core-web-sm==3.3.0) (59.6.0)\n",
|
||||||
|
"Requirement already satisfied: catalogue<2.1.0,>=2.0.6 in ./.env/lib/python3.10/site-packages (from spacy<3.4.0,>=3.3.0.dev0->en-core-web-sm==3.3.0) (2.0.7)\n",
|
||||||
|
"Requirement already satisfied: spacy-loggers<2.0.0,>=1.0.0 in ./.env/lib/python3.10/site-packages (from spacy<3.4.0,>=3.3.0.dev0->en-core-web-sm==3.3.0) (1.0.2)\n",
|
||||||
|
"Requirement already satisfied: blis<0.8.0,>=0.4.0 in ./.env/lib/python3.10/site-packages (from spacy<3.4.0,>=3.3.0.dev0->en-core-web-sm==3.3.0) (0.7.8)\n",
|
||||||
|
"Requirement already satisfied: murmurhash<1.1.0,>=0.28.0 in ./.env/lib/python3.10/site-packages (from spacy<3.4.0,>=3.3.0.dev0->en-core-web-sm==3.3.0) (1.0.7)\n",
|
||||||
|
"Requirement already satisfied: spacy-legacy<3.1.0,>=3.0.9 in ./.env/lib/python3.10/site-packages (from spacy<3.4.0,>=3.3.0.dev0->en-core-web-sm==3.3.0) (3.0.9)\n",
|
||||||
|
"Requirement already satisfied: requests<3.0.0,>=2.13.0 in ./.env/lib/python3.10/site-packages (from spacy<3.4.0,>=3.3.0.dev0->en-core-web-sm==3.3.0) (2.28.0)\n",
|
||||||
|
"Requirement already satisfied: numpy>=1.15.0 in ./.env/lib/python3.10/site-packages (from spacy<3.4.0,>=3.3.0.dev0->en-core-web-sm==3.3.0) (1.23.0)\n",
|
||||||
|
"Requirement already satisfied: cymem<2.1.0,>=2.0.2 in ./.env/lib/python3.10/site-packages (from spacy<3.4.0,>=3.3.0.dev0->en-core-web-sm==3.3.0) (2.0.6)\n",
|
||||||
|
"Requirement already satisfied: pathy>=0.3.5 in ./.env/lib/python3.10/site-packages (from spacy<3.4.0,>=3.3.0.dev0->en-core-web-sm==3.3.0) (0.6.1)\n",
|
||||||
|
"Requirement already satisfied: jinja2 in ./.env/lib/python3.10/site-packages (from spacy<3.4.0,>=3.3.0.dev0->en-core-web-sm==3.3.0) (3.1.2)\n",
|
||||||
|
"Requirement already satisfied: langcodes<4.0.0,>=3.2.0 in ./.env/lib/python3.10/site-packages (from spacy<3.4.0,>=3.3.0.dev0->en-core-web-sm==3.3.0) (3.3.0)\n",
|
||||||
|
"Requirement already satisfied: pydantic!=1.8,!=1.8.1,<1.9.0,>=1.7.4 in ./.env/lib/python3.10/site-packages (from spacy<3.4.0,>=3.3.0.dev0->en-core-web-sm==3.3.0) (1.8.2)\n",
|
||||||
|
"Requirement already satisfied: preshed<3.1.0,>=3.0.2 in ./.env/lib/python3.10/site-packages (from spacy<3.4.0,>=3.3.0.dev0->en-core-web-sm==3.3.0) (3.0.6)\n",
|
||||||
|
"Requirement already satisfied: thinc<8.1.0,>=8.0.14 in ./.env/lib/python3.10/site-packages (from spacy<3.4.0,>=3.3.0.dev0->en-core-web-sm==3.3.0) (8.0.17)\n",
|
||||||
|
"Requirement already satisfied: packaging>=20.0 in ./.env/lib/python3.10/site-packages (from spacy<3.4.0,>=3.3.0.dev0->en-core-web-sm==3.3.0) (21.3)\n",
|
||||||
|
"Requirement already satisfied: pyparsing!=3.0.5,>=2.0.2 in ./.env/lib/python3.10/site-packages (from packaging>=20.0->spacy<3.4.0,>=3.3.0.dev0->en-core-web-sm==3.3.0) (3.0.9)\n",
|
||||||
|
"Requirement already satisfied: smart-open<6.0.0,>=5.0.0 in ./.env/lib/python3.10/site-packages (from pathy>=0.3.5->spacy<3.4.0,>=3.3.0.dev0->en-core-web-sm==3.3.0) (5.2.1)\n",
|
||||||
|
"Requirement already satisfied: typing-extensions>=3.7.4.3 in ./.env/lib/python3.10/site-packages (from pydantic!=1.8,!=1.8.1,<1.9.0,>=1.7.4->spacy<3.4.0,>=3.3.0.dev0->en-core-web-sm==3.3.0) (4.2.0)\n",
|
||||||
|
"Requirement already satisfied: charset-normalizer~=2.0.0 in ./.env/lib/python3.10/site-packages (from requests<3.0.0,>=2.13.0->spacy<3.4.0,>=3.3.0.dev0->en-core-web-sm==3.3.0) (2.0.12)\n",
|
||||||
|
"Requirement already satisfied: certifi>=2017.4.17 in ./.env/lib/python3.10/site-packages (from requests<3.0.0,>=2.13.0->spacy<3.4.0,>=3.3.0.dev0->en-core-web-sm==3.3.0) (2022.6.15)\n",
|
||||||
|
"Requirement already satisfied: idna<4,>=2.5 in ./.env/lib/python3.10/site-packages (from requests<3.0.0,>=2.13.0->spacy<3.4.0,>=3.3.0.dev0->en-core-web-sm==3.3.0) (3.3)\n",
|
||||||
|
"Requirement already satisfied: urllib3<1.27,>=1.21.1 in ./.env/lib/python3.10/site-packages (from requests<3.0.0,>=2.13.0->spacy<3.4.0,>=3.3.0.dev0->en-core-web-sm==3.3.0) (1.26.9)\n",
|
||||||
|
"Requirement already satisfied: click<9.0.0,>=7.1.1 in ./.env/lib/python3.10/site-packages (from typer<0.5.0,>=0.3.0->spacy<3.4.0,>=3.3.0.dev0->en-core-web-sm==3.3.0) (8.1.3)\n",
|
||||||
|
"Requirement already satisfied: MarkupSafe>=2.0 in ./.env/lib/python3.10/site-packages (from jinja2->spacy<3.4.0,>=3.3.0.dev0->en-core-web-sm==3.3.0) (2.1.1)\n",
|
||||||
|
"Installing collected packages: en-core-web-sm\n",
|
||||||
|
"Successfully installed en-core-web-sm-3.3.0\n"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "stderr",
|
||||||
|
"output_type": "stream",
|
||||||
|
"text": [
|
||||||
|
"\u001b[38;5;226m2022-06-25 14:21:57,983 | WARNING | Limiting concurrency to 1 because there are only 1 chunks\u001b[0m\n",
|
||||||
|
"\u001b[38;5;39m2022-06-25 14:21:57,984 | INFO | Starting parallel map (concurrency: 1, chunk size: 1)\u001b[0m\n",
|
||||||
|
"\u001b[38;5;226m2022-06-25 14:21:57,984 | WARNING | Running in series, there is no reason for parallelism\u001b[0m\n",
|
||||||
|
"100%|██████████| 1/1 [03:26<00:00, 206.86s/it]\n"
|
||||||
|
]
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"source": [
|
||||||
|
"from typing import List, Tuple\n",
|
||||||
|
"import json\n",
|
||||||
|
"import gzip\n",
|
||||||
|
"from great_ai import parallel_map, clean, is_english, predict_language\n",
|
||||||
|
"\n",
|
||||||
|
"\n",
|
||||||
|
"def preprocess_chunk(chunk_key: str) -> List[Tuple[str, List[str]]]:\n",
|
||||||
|
" # Extract\n",
|
||||||
|
" response = urllib.request.urlopen(\n",
|
||||||
|
" f\"https://s3-us-west-2.amazonaws.com/ai2-s2-research-public/open-corpus/2022-02-01/{chunk_key}\"\n",
|
||||||
|
" ) # a gzipped JSON Lines file\n",
|
||||||
|
"\n",
|
||||||
|
" decompressed = gzip.decompress(response.read())\n",
|
||||||
|
" decoded = decompressed.decode()\n",
|
||||||
|
" chunk = [json.loads(line) for line in decoded.split(\"\\n\") if line]\n",
|
||||||
|
"\n",
|
||||||
|
" # Transform\n",
|
||||||
|
" return [\n",
|
||||||
|
" (\n",
|
||||||
|
" clean(\n",
|
||||||
|
" f'{c[\"title\"]} {c[\"paperAbstract\"]} {c[\"journalName\"]} {c[\"venue\"]}',\n",
|
||||||
|
" convert_to_ascii=True,\n",
|
||||||
|
" ), # The text is cleaned to remove PDF extraction, web scraping, and other common artifacts\n",
|
||||||
|
" c[\"fieldsOfStudy\"],\n",
|
||||||
|
" ) # Create pairs of `(text, [...domains])`\n",
|
||||||
|
" for c in chunk\n",
|
||||||
|
" if c[\"fieldsOfStudy\"] and is_english(predict_language(c[\"paperAbstract\"]))\n",
|
||||||
|
" ]\n",
|
||||||
|
"\n",
|
||||||
|
"\n",
|
||||||
|
"preprocessed_chunks = parallel_map(preprocess_chunk, chunks)"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": 4,
|
||||||
|
"metadata": {},
|
||||||
|
"outputs": [],
|
||||||
|
"source": [
|
||||||
|
"from itertools import chain\n",
|
||||||
|
"\n",
|
||||||
|
"preprocessed_data = list(chain(*preprocessed_chunks))\n",
|
||||||
|
"X, y = zip(\n",
|
||||||
|
" *preprocessed_data\n",
|
||||||
|
") # X is the input, y is the expected (ground truth) output"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "markdown",
|
||||||
|
"metadata": {},
|
||||||
|
"source": [
|
||||||
|
"### Load\n",
|
||||||
|
"\n",
|
||||||
|
"Upload the dataset (or a part of it) to a central repository using `great_ai.add_ground_truth`. This step automatically tags each datapoint with a split label according to the ratios we set. Additional tags can be also given.\n",
|
||||||
|
"\n",
|
||||||
|
"#### Production-ready backend\n",
|
||||||
|
"\n",
|
||||||
|
"The MongoDB driver is automatically configured if `mongo.ini` exists with the following scheme:\n",
|
||||||
|
"\n",
|
||||||
|
"```ini\n",
|
||||||
|
"mongo_connection_string=mongodb://localhost:27017/\n",
|
||||||
|
"mongo_database=my_great_ai_db\n",
|
||||||
|
"```\n",
|
||||||
|
"> You can install MongoDB from [here](https://www.mongodb.com/docs/manual/installation) or [use it as a service](https://www.mongodb.com/cloud/atlas/register)\n"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": 5,
|
||||||
|
"metadata": {},
|
||||||
|
"outputs": [
|
||||||
|
{
|
||||||
|
"name": "stderr",
|
||||||
|
"output_type": "stream",
|
||||||
|
"text": [
|
||||||
|
"\u001b[38;5;226m2022-06-25 14:25:24,989 | WARNING | Environment variable ENVIRONMENT is not set, defaulting to development mode ‼️\u001b[0m\n",
|
||||||
|
"\u001b[38;5;39m2022-06-25 14:25:24,990 | INFO | Found credentials file (/data/projects/great_ai_example/mongo.ini), initialising MongodbDriver\u001b[0m\n",
|
||||||
|
"\u001b[38;5;39m2022-06-25 14:25:24,991 | INFO | Found credentials file (/data/projects/great_ai_example/mongo.ini), initialising LargeFileMongo\u001b[0m\n",
|
||||||
|
"\u001b[38;5;39m2022-06-25 14:25:24,992 | INFO | Settings: configured ✅\u001b[0m\n",
|
||||||
|
"\u001b[38;5;39m2022-06-25 14:25:24,993 | INFO | 🔩 tracing_database: MongodbDriver\u001b[0m\n",
|
||||||
|
"\u001b[38;5;39m2022-06-25 14:25:24,994 | INFO | 🔩 large_file_implementation: LargeFileMongo\u001b[0m\n",
|
||||||
|
"\u001b[38;5;39m2022-06-25 14:25:24,994 | INFO | 🔩 is_production: False\u001b[0m\n",
|
||||||
|
"\u001b[38;5;39m2022-06-25 14:25:24,995 | INFO | 🔩 should_log_exception_stack: True\u001b[0m\n",
|
||||||
|
"\u001b[38;5;39m2022-06-25 14:25:24,996 | INFO | 🔩 prediction_cache_size: 512\u001b[0m\n",
|
||||||
|
"\u001b[38;5;39m2022-06-25 14:25:24,997 | INFO | 🔩 dashboard_table_size: 50\u001b[0m\n",
|
||||||
|
"\u001b[38;5;226m2022-06-25 14:25:24,998 | WARNING | You still need to check whether you follow all best practices before trusting your deployment.\u001b[0m\n",
|
||||||
|
"\u001b[38;5;226m2022-06-25 14:25:24,998 | WARNING | > Find out more at https://se-ml.github.io/practices/\u001b[0m\n"
|
||||||
|
]
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"source": [
|
||||||
|
"from great_ai import add_ground_truth\n",
|
||||||
|
"\n",
|
||||||
|
"add_ground_truth(X, y, train_split_ratio=0.8, test_split_ratio=0.2)"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "markdown",
|
||||||
|
"metadata": {},
|
||||||
|
"source": [
|
||||||
|
"### Next: [Part 2](train.ipynb)"
|
||||||
|
]
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"metadata": {
|
||||||
|
"kernelspec": {
|
||||||
|
"display_name": "Python 3.10.4 ('.env': venv)",
|
||||||
|
"language": "python",
|
||||||
|
"name": "python3"
|
||||||
|
},
|
||||||
|
"language_info": {
|
||||||
|
"codemirror_mode": {
|
||||||
|
"name": "ipython",
|
||||||
|
"version": 3
|
||||||
|
},
|
||||||
|
"file_extension": ".py",
|
||||||
|
"mimetype": "text/x-python",
|
||||||
|
"name": "python",
|
||||||
|
"nbconvert_exporter": "python",
|
||||||
|
"pygments_lexer": "ipython3",
|
||||||
|
"version": "3.10.4"
|
||||||
|
},
|
||||||
|
"orig_nbformat": 4,
|
||||||
|
"vscode": {
|
||||||
|
"interpreter": {
|
||||||
|
"hash": "c1f394f9662881005685eeb18d8f9f77079b1b8b9a5ece1f825bfa01fcb7f52f"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"nbformat": 4,
|
||||||
|
"nbformat_minor": 2
|
||||||
|
}
|
||||||
1950
great_ai_example-main/src/data/index.html
Normal file
1950
great_ai_example-main/src/data/index.html
Normal file
File diff suppressed because one or more lines are too long
239
great_ai_example-main/src/deploy/deploy.ipynb
Normal file
239
great_ai_example-main/src/deploy/deploy.ipynb
Normal file
File diff suppressed because one or more lines are too long
1857
great_ai_example-main/src/deploy/index.html
Normal file
1857
great_ai_example-main/src/deploy/index.html
Normal file
File diff suppressed because one or more lines are too long
2
great_ai_example-main/src/mongo.ini
Normal file
2
great_ai_example-main/src/mongo.ini
Normal file
|
|
@ -0,0 +1,2 @@
|
||||||
|
mongo_connection_string=mongodb://localhost:27017/ # change this
|
||||||
|
mongo_database=great_ai_example # this will be automatically created
|
||||||
2470
great_ai_example-main/src/train/index.html
Normal file
2470
great_ai_example-main/src/train/index.html
Normal file
File diff suppressed because one or more lines are too long
1811
great_ai_example-main/src/train/train.ipynb
Normal file
1811
great_ai_example-main/src/train/train.ipynb
Normal file
File diff suppressed because it is too large
Load diff
|
|
@ -1,26 +0,0 @@
|
||||||
{
|
|
||||||
"cells": [
|
|
||||||
{
|
|
||||||
"cell_type": "code",
|
|
||||||
"execution_count": null,
|
|
||||||
"metadata": {},
|
|
||||||
"outputs": [],
|
|
||||||
"source": [
|
|
||||||
"from great_ai import GreatAI\n",
|
|
||||||
"\n",
|
|
||||||
"\n",
|
|
||||||
"@GreatAI.create\n",
|
|
||||||
"def hello_world(name: str) -> str:\n",
|
|
||||||
" return f\"Hello {name}!\""
|
|
||||||
]
|
|
||||||
}
|
|
||||||
],
|
|
||||||
"metadata": {
|
|
||||||
"language_info": {
|
|
||||||
"name": "python"
|
|
||||||
},
|
|
||||||
"orig_nbformat": 4
|
|
||||||
},
|
|
||||||
"nbformat": 4,
|
|
||||||
"nbformat_minor": 2
|
|
||||||
}
|
|
||||||
87
hello_world/hello_world.ipynb
Normal file
87
hello_world/hello_world.ipynb
Normal file
|
|
@ -0,0 +1,87 @@
|
||||||
|
{
|
||||||
|
"cells": [
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": 1,
|
||||||
|
"metadata": {},
|
||||||
|
"outputs": [
|
||||||
|
{
|
||||||
|
"name": "stderr",
|
||||||
|
"output_type": "stream",
|
||||||
|
"text": [
|
||||||
|
"\u001b[38;5;226m2022-07-08 10:26:12,806 | WARNING | Environment variable ENVIRONMENT is not set, defaulting to development mode ‼️\u001b[0m\n",
|
||||||
|
"\u001b[38;5;226m2022-07-08 10:26:12,807 | WARNING | Cannot find credentials files, defaulting to using ParallelTinyDbDriver\u001b[0m\n",
|
||||||
|
"\u001b[38;5;226m2022-07-08 10:26:12,807 | WARNING | The selected tracing database (ParallelTinyDbDriver) is not recommended for production\u001b[0m\n",
|
||||||
|
"\u001b[38;5;226m2022-07-08 10:26:12,808 | WARNING | Cannot find credentials files, defaulting to using LargeFileLocal\u001b[0m\n",
|
||||||
|
"\u001b[38;5;39m2022-07-08 10:26:12,808 | INFO | Settings: configured ✅\u001b[0m\n",
|
||||||
|
"\u001b[38;5;39m2022-07-08 10:26:12,809 | INFO | 🔩 tracing_database: ParallelTinyDbDriver\u001b[0m\n",
|
||||||
|
"\u001b[38;5;39m2022-07-08 10:26:12,809 | INFO | 🔩 large_file_implementation: LargeFileLocal\u001b[0m\n",
|
||||||
|
"\u001b[38;5;39m2022-07-08 10:26:12,811 | INFO | 🔩 is_production: False\u001b[0m\n",
|
||||||
|
"\u001b[38;5;39m2022-07-08 10:26:12,811 | INFO | 🔩 should_log_exception_stack: True\u001b[0m\n",
|
||||||
|
"\u001b[38;5;39m2022-07-08 10:26:12,812 | INFO | 🔩 prediction_cache_size: 512\u001b[0m\n",
|
||||||
|
"\u001b[38;5;39m2022-07-08 10:26:12,813 | INFO | 🔩 dashboard_table_size: 20\u001b[0m\n",
|
||||||
|
"\u001b[38;5;226m2022-07-08 10:26:12,813 | WARNING | You still need to check whether you follow all best practices before trusting your deployment.\u001b[0m\n",
|
||||||
|
"\u001b[38;5;226m2022-07-08 10:26:12,814 | WARNING | > Find out more at https://se-ml.github.io/practices/\u001b[0m\n"
|
||||||
|
]
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"source": [
|
||||||
|
"from great_ai import GreatAI\n",
|
||||||
|
"from asyncio import sleep\n",
|
||||||
|
"\n",
|
||||||
|
"\n",
|
||||||
|
"@GreatAI.create\n",
|
||||||
|
"async def hello_world(name: str) -> str:\n",
|
||||||
|
" await sleep(1)\n",
|
||||||
|
" return f\"Hello {name}!\""
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": 2,
|
||||||
|
"metadata": {},
|
||||||
|
"outputs": [
|
||||||
|
{
|
||||||
|
"data": {
|
||||||
|
"text/plain": [
|
||||||
|
"Trace(trace_id='27c5e581-ad6e-4d50-bb77-862dfb7963fb', created='2022-07-08T08:26:12.911975', original_execution_time_ms=1.933, logged_values={'arg:name:value': 'hi', 'arg:name:length': 2}, models=[], exception=None, output='Hello hi!', feedback=None, tags=['hello_world', 'online', 'development'])"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"execution_count": 2,
|
||||||
|
"metadata": {},
|
||||||
|
"output_type": "execute_result"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"source": [
|
||||||
|
"await hello_world('hi')"
|
||||||
|
]
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"metadata": {
|
||||||
|
"kernelspec": {
|
||||||
|
"display_name": "Python 3.10.4 ('.env': venv)",
|
||||||
|
"language": "python",
|
||||||
|
"name": "python3"
|
||||||
|
},
|
||||||
|
"language_info": {
|
||||||
|
"codemirror_mode": {
|
||||||
|
"name": "ipython",
|
||||||
|
"version": 3
|
||||||
|
},
|
||||||
|
"file_extension": ".py",
|
||||||
|
"mimetype": "text/x-python",
|
||||||
|
"name": "python",
|
||||||
|
"nbconvert_exporter": "python",
|
||||||
|
"pygments_lexer": "ipython3",
|
||||||
|
"version": "3.10.4"
|
||||||
|
},
|
||||||
|
"orig_nbformat": 4,
|
||||||
|
"vscode": {
|
||||||
|
"interpreter": {
|
||||||
|
"hash": "02dd6d3afbfa9fbbe1037d64ad9014965528a1ccad21929d6e72f466389a68ad"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"nbformat": 4,
|
||||||
|
"nbformat_minor": 2
|
||||||
|
}
|
||||||
|
|
@ -3,4 +3,6 @@ from great_ai import GreatAI
|
||||||
|
|
||||||
@GreatAI.create
|
@GreatAI.create
|
||||||
def hello_world(name: str) -> str:
|
def hello_world(name: str) -> str:
|
||||||
|
"""Learn more about GreatAI at https://github.com/schmelczer/great-ai"""
|
||||||
|
|
||||||
return f"Hello {name}!"
|
return f"Hello {name}!"
|
||||||
1509
hello_world/index.html
Normal file
1509
hello_world/index.html
Normal file
File diff suppressed because one or more lines are too long
0
how-to-guides/how-to-guides.md
Normal file
0
how-to-guides/how-to-guides.md
Normal file
|
|
@ -9,7 +9,7 @@
|
||||||
|
|
||||||
|
|
||||||
<link rel="icon" href="../assets/images/favicon.png">
|
<link rel="icon" href="../assets/images/favicon.png">
|
||||||
<meta name="generator" content="mkdocs-1.3.0, mkdocs-material-8.3.8">
|
<meta name="generator" content="mkdocs-1.3.0, mkdocs-material-8.3.9">
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
@ -49,13 +49,17 @@
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
<body dir="ltr" data-md-color-scheme="" data-md-color-primary="none" data-md-color-accent="none">
|
|
||||||
|
<body dir="ltr" data-md-color-scheme="default" data-md-color-primary="" data-md-color-accent="">
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
<script>var palette=__md_get("__palette");if(palette&&"object"==typeof palette.color)for(var key of Object.keys(palette.color))document.body.setAttribute("data-md-color-"+key,palette.color[key])</script>
|
||||||
|
|
||||||
<input class="md-toggle" data-md-toggle="drawer" type="checkbox" id="__drawer" autocomplete="off">
|
<input class="md-toggle" data-md-toggle="drawer" type="checkbox" id="__drawer" autocomplete="off">
|
||||||
<input class="md-toggle" data-md-toggle="search" type="checkbox" id="__search" autocomplete="off">
|
<input class="md-toggle" data-md-toggle="search" type="checkbox" id="__search" autocomplete="off">
|
||||||
<label class="md-overlay" for="__drawer"></label>
|
<label class="md-overlay" for="__drawer"></label>
|
||||||
|
|
@ -97,9 +101,76 @@
|
||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
|
<form class="md-header__option" data-md-component="palette">
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
<input class="md-option" data-md-color-media="" data-md-color-scheme="default" data-md-color-primary="" data-md-color-accent="" aria-label="Switch to dark mode" type="radio" name="__palette" id="__palette_1">
|
||||||
|
|
||||||
|
<label class="md-header__button md-icon" title="Switch to dark mode" for="__palette_2" hidden>
|
||||||
|
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M12 8a4 4 0 0 0-4 4 4 4 0 0 0 4 4 4 4 0 0 0 4-4 4 4 0 0 0-4-4m0 10a6 6 0 0 1-6-6 6 6 0 0 1 6-6 6 6 0 0 1 6 6 6 6 0 0 1-6 6m8-9.31V4h-4.69L12 .69 8.69 4H4v4.69L.69 12 4 15.31V20h4.69L12 23.31 15.31 20H20v-4.69L23.31 12 20 8.69Z"/></svg>
|
||||||
|
</label>
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
<input class="md-option" data-md-color-media="" data-md-color-scheme="slate" data-md-color-primary="" data-md-color-accent="" aria-label="Switch to light mode" type="radio" name="__palette" id="__palette_2">
|
||||||
|
|
||||||
|
<label class="md-header__button md-icon" title="Switch to light mode" for="__palette_1" hidden>
|
||||||
|
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M12 18c-.89 0-1.74-.2-2.5-.55C11.56 16.5 13 14.42 13 12c0-2.42-1.44-4.5-3.5-5.45C10.26 6.2 11.11 6 12 6a6 6 0 0 1 6 6 6 6 0 0 1-6 6m8-9.31V4h-4.69L12 .69 8.69 4H4v4.69L.69 12 4 15.31V20h4.69L12 23.31 15.31 20H20v-4.69L23.31 12 20 8.69Z"/></svg>
|
||||||
|
</label>
|
||||||
|
|
||||||
|
|
||||||
|
</form>
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
<label class="md-header__button md-icon" for="__search">
|
||||||
|
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M9.5 3A6.5 6.5 0 0 1 16 9.5c0 1.61-.59 3.09-1.56 4.23l.27.27h.79l5 5-1.5 1.5-5-5v-.79l-.27-.27A6.516 6.516 0 0 1 9.5 16 6.5 6.5 0 0 1 3 9.5 6.5 6.5 0 0 1 9.5 3m0 2C7 5 5 7 5 9.5S7 14 9.5 14 14 12 14 9.5 12 5 9.5 5Z"/></svg>
|
||||||
|
</label>
|
||||||
|
<div class="md-search" data-md-component="search" role="dialog">
|
||||||
|
<label class="md-search__overlay" for="__search"></label>
|
||||||
|
<div class="md-search__inner" role="search">
|
||||||
|
<form class="md-search__form" name="search">
|
||||||
|
<input type="text" class="md-search__input" name="query" aria-label="Search" placeholder="Search" autocapitalize="off" autocorrect="off" autocomplete="off" spellcheck="false" data-md-component="search-query" required>
|
||||||
|
<label class="md-search__icon md-icon" for="__search">
|
||||||
|
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M9.5 3A6.5 6.5 0 0 1 16 9.5c0 1.61-.59 3.09-1.56 4.23l.27.27h.79l5 5-1.5 1.5-5-5v-.79l-.27-.27A6.516 6.516 0 0 1 9.5 16 6.5 6.5 0 0 1 3 9.5 6.5 6.5 0 0 1 9.5 3m0 2C7 5 5 7 5 9.5S7 14 9.5 14 14 12 14 9.5 12 5 9.5 5Z"/></svg>
|
||||||
|
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M20 11v2H8l5.5 5.5-1.42 1.42L4.16 12l7.92-7.92L13.5 5.5 8 11h12Z"/></svg>
|
||||||
|
</label>
|
||||||
|
<nav class="md-search__options" aria-label="Search">
|
||||||
|
|
||||||
|
<button type="reset" class="md-search__icon md-icon" aria-label="Clear" tabindex="-1">
|
||||||
|
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M19 6.41 17.59 5 12 10.59 6.41 5 5 6.41 10.59 12 5 17.59 6.41 19 12 13.41 17.59 19 19 17.59 13.41 12 19 6.41Z"/></svg>
|
||||||
|
</button>
|
||||||
|
</nav>
|
||||||
|
|
||||||
|
</form>
|
||||||
|
<div class="md-search__output">
|
||||||
|
<div class="md-search__scrollwrap" data-md-scrollfix>
|
||||||
|
<div class="md-search-result" data-md-component="search-result">
|
||||||
|
<div class="md-search-result__meta">
|
||||||
|
Initializing search
|
||||||
|
</div>
|
||||||
|
<ol class="md-search-result__list"></ol>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
|
||||||
|
<div class="md-header__source">
|
||||||
|
<a href="https://github.com/schmelczer/great-ai" title="Go to repository" class="md-source" data-md-component="source">
|
||||||
|
<div class="md-source__icon md-icon">
|
||||||
|
|
||||||
|
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 448 512"><!--! Font Awesome Free 6.1.1 by @fontawesome - https://fontawesome.com License - https://fontawesome.com/license/free (Icons: CC BY 4.0, Fonts: SIL OFL 1.1, Code: MIT License) Copyright 2022 Fonticons, Inc.--><path d="M439.55 236.05 244 40.45a28.87 28.87 0 0 0-40.81 0l-40.66 40.63 51.52 51.52c27.06-9.14 52.68 16.77 43.39 43.68l49.66 49.66c34.23-11.8 61.18 31 35.47 56.69-26.49 26.49-70.21-2.87-56-37.34L240.22 199v121.85c25.3 12.54 22.26 41.85 9.08 55a34.34 34.34 0 0 1-48.55 0c-17.57-17.6-11.07-46.91 11.25-56v-123c-20.8-8.51-24.6-30.74-18.64-45L142.57 101 8.45 235.14a28.86 28.86 0 0 0 0 40.81l195.61 195.6a28.86 28.86 0 0 0 40.8 0l194.69-194.69a28.86 28.86 0 0 0 0-40.81z"/></svg>
|
||||||
|
</div>
|
||||||
|
<div class="md-source__repository">
|
||||||
|
GitHub
|
||||||
|
</div>
|
||||||
|
</a>
|
||||||
|
</div>
|
||||||
|
|
||||||
</nav>
|
</nav>
|
||||||
|
|
||||||
</header>
|
</header>
|
||||||
|
|
@ -133,6 +204,18 @@
|
||||||
GreatAI documentation
|
GreatAI documentation
|
||||||
</label>
|
</label>
|
||||||
|
|
||||||
|
<div class="md-nav__source">
|
||||||
|
<a href="https://github.com/schmelczer/great-ai" title="Go to repository" class="md-source" data-md-component="source">
|
||||||
|
<div class="md-source__icon md-icon">
|
||||||
|
|
||||||
|
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 448 512"><!--! Font Awesome Free 6.1.1 by @fontawesome - https://fontawesome.com License - https://fontawesome.com/license/free (Icons: CC BY 4.0, Fonts: SIL OFL 1.1, Code: MIT License) Copyright 2022 Fonticons, Inc.--><path d="M439.55 236.05 244 40.45a28.87 28.87 0 0 0-40.81 0l-40.66 40.63 51.52 51.52c27.06-9.14 52.68 16.77 43.39 43.68l49.66 49.66c34.23-11.8 61.18 31 35.47 56.69-26.49 26.49-70.21-2.87-56-37.34L240.22 199v121.85c25.3 12.54 22.26 41.85 9.08 55a34.34 34.34 0 0 1-48.55 0c-17.57-17.6-11.07-46.91 11.25-56v-123c-20.8-8.51-24.6-30.74-18.64-45L142.57 101 8.45 235.14a28.86 28.86 0 0 0 0 40.81l195.61 195.6a28.86 28.86 0 0 0 40.8 0l194.69-194.69a28.86 28.86 0 0 0 0-40.81z"/></svg>
|
||||||
|
</div>
|
||||||
|
<div class="md-source__repository">
|
||||||
|
GitHub
|
||||||
|
</div>
|
||||||
|
</a>
|
||||||
|
</div>
|
||||||
|
|
||||||
<ul class="md-nav__list" data-md-scrollfix>
|
<ul class="md-nav__list" data-md-scrollfix>
|
||||||
|
|
||||||
|
|
||||||
|
|
@ -218,6 +301,21 @@
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
<li class="md-nav__item">
|
||||||
|
<a href="../simple-mag/train/" class="md-nav__link">
|
||||||
|
Train a field of study (domain) classification of sentences on the MAG dataset
|
||||||
|
</a>
|
||||||
|
</li>
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
</ul>
|
</ul>
|
||||||
</nav>
|
</nav>
|
||||||
</div>
|
</div>
|
||||||
|
|
@ -245,14 +343,33 @@
|
||||||
<div class="md-content" data-md-component="content">
|
<div class="md-content" data-md-component="content">
|
||||||
<article class="md-content__inner md-typeset">
|
<article class="md-content__inner md-typeset">
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
<a href="https://github.com/schmelczer/great-ai/edit/main/docs/how-to-guides.md" title="Edit this page" class="md-content__button md-icon">
|
||||||
|
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M20.71 7.04c.39-.39.39-1.04 0-1.41l-2.34-2.34c-.37-.39-1.02-.39-1.41 0l-1.84 1.83 3.75 3.75M3 17.25V21h3.75L17.81 9.93l-3.75-3.75L3 17.25Z"/></svg>
|
||||||
|
</a>
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
<h1>How-To Guides</h1>
|
<h1>How-To Guides</h1>
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
<hr>
|
||||||
|
<div class="md-source-file">
|
||||||
|
<small>
|
||||||
|
|
||||||
|
Last update:
|
||||||
|
<span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-date">July 8, 2022</span>
|
||||||
|
|
||||||
|
|
||||||
|
</small>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
</article>
|
</article>
|
||||||
|
|
||||||
</div>
|
</div>
|
||||||
|
|
@ -321,7 +438,7 @@
|
||||||
<script id="__config" type="application/json">{"base": "..", "features": [], "search": "../assets/javascripts/workers/search.b97dbffb.min.js", "translations": {"clipboard.copied": "Copied to clipboard", "clipboard.copy": "Copy to clipboard", "search.config.lang": "en", "search.config.pipeline": "trimmer, stopWordFilter", "search.config.separator": "[\\s\\-]+", "search.placeholder": "Search", "search.result.more.one": "1 more on this page", "search.result.more.other": "# more on this page", "search.result.none": "No matching documents", "search.result.one": "1 matching document", "search.result.other": "# matching documents", "search.result.placeholder": "Type to start searching", "search.result.term.missing": "Missing", "select.version.title": "Select version"}}</script>
|
<script id="__config" type="application/json">{"base": "..", "features": [], "search": "../assets/javascripts/workers/search.b97dbffb.min.js", "translations": {"clipboard.copied": "Copied to clipboard", "clipboard.copy": "Copy to clipboard", "search.config.lang": "en", "search.config.pipeline": "trimmer, stopWordFilter", "search.config.separator": "[\\s\\-]+", "search.placeholder": "Search", "search.result.more.one": "1 more on this page", "search.result.more.other": "# more on this page", "search.result.none": "No matching documents", "search.result.one": "1 matching document", "search.result.other": "# matching documents", "search.result.placeholder": "Type to start searching", "search.result.term.missing": "Missing", "select.version.title": "Select version"}}</script>
|
||||||
|
|
||||||
|
|
||||||
<script src="../assets/javascripts/bundle.0238f547.min.js"></script>
|
<script src="../assets/javascripts/bundle.6c7ad80a.min.js"></script>
|
||||||
|
|
||||||
|
|
||||||
</body>
|
</body>
|
||||||
|
|
|
||||||
123
index.html
123
index.html
|
|
@ -9,7 +9,7 @@
|
||||||
|
|
||||||
|
|
||||||
<link rel="icon" href="assets/images/favicon.png">
|
<link rel="icon" href="assets/images/favicon.png">
|
||||||
<meta name="generator" content="mkdocs-1.3.0, mkdocs-material-8.3.8">
|
<meta name="generator" content="mkdocs-1.3.0, mkdocs-material-8.3.9">
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
@ -49,13 +49,17 @@
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
<body dir="ltr" data-md-color-scheme="" data-md-color-primary="none" data-md-color-accent="none">
|
|
||||||
|
<body dir="ltr" data-md-color-scheme="default" data-md-color-primary="" data-md-color-accent="">
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
<script>var palette=__md_get("__palette");if(palette&&"object"==typeof palette.color)for(var key of Object.keys(palette.color))document.body.setAttribute("data-md-color-"+key,palette.color[key])</script>
|
||||||
|
|
||||||
<input class="md-toggle" data-md-toggle="drawer" type="checkbox" id="__drawer" autocomplete="off">
|
<input class="md-toggle" data-md-toggle="drawer" type="checkbox" id="__drawer" autocomplete="off">
|
||||||
<input class="md-toggle" data-md-toggle="search" type="checkbox" id="__search" autocomplete="off">
|
<input class="md-toggle" data-md-toggle="search" type="checkbox" id="__search" autocomplete="off">
|
||||||
<label class="md-overlay" for="__drawer"></label>
|
<label class="md-overlay" for="__drawer"></label>
|
||||||
|
|
@ -102,9 +106,76 @@
|
||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
|
<form class="md-header__option" data-md-component="palette">
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
<input class="md-option" data-md-color-media="" data-md-color-scheme="default" data-md-color-primary="" data-md-color-accent="" aria-label="Switch to dark mode" type="radio" name="__palette" id="__palette_1">
|
||||||
|
|
||||||
|
<label class="md-header__button md-icon" title="Switch to dark mode" for="__palette_2" hidden>
|
||||||
|
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M12 8a4 4 0 0 0-4 4 4 4 0 0 0 4 4 4 4 0 0 0 4-4 4 4 0 0 0-4-4m0 10a6 6 0 0 1-6-6 6 6 0 0 1 6-6 6 6 0 0 1 6 6 6 6 0 0 1-6 6m8-9.31V4h-4.69L12 .69 8.69 4H4v4.69L.69 12 4 15.31V20h4.69L12 23.31 15.31 20H20v-4.69L23.31 12 20 8.69Z"/></svg>
|
||||||
|
</label>
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
<input class="md-option" data-md-color-media="" data-md-color-scheme="slate" data-md-color-primary="" data-md-color-accent="" aria-label="Switch to light mode" type="radio" name="__palette" id="__palette_2">
|
||||||
|
|
||||||
|
<label class="md-header__button md-icon" title="Switch to light mode" for="__palette_1" hidden>
|
||||||
|
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M12 18c-.89 0-1.74-.2-2.5-.55C11.56 16.5 13 14.42 13 12c0-2.42-1.44-4.5-3.5-5.45C10.26 6.2 11.11 6 12 6a6 6 0 0 1 6 6 6 6 0 0 1-6 6m8-9.31V4h-4.69L12 .69 8.69 4H4v4.69L.69 12 4 15.31V20h4.69L12 23.31 15.31 20H20v-4.69L23.31 12 20 8.69Z"/></svg>
|
||||||
|
</label>
|
||||||
|
|
||||||
|
|
||||||
|
</form>
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
<label class="md-header__button md-icon" for="__search">
|
||||||
|
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M9.5 3A6.5 6.5 0 0 1 16 9.5c0 1.61-.59 3.09-1.56 4.23l.27.27h.79l5 5-1.5 1.5-5-5v-.79l-.27-.27A6.516 6.516 0 0 1 9.5 16 6.5 6.5 0 0 1 3 9.5 6.5 6.5 0 0 1 9.5 3m0 2C7 5 5 7 5 9.5S7 14 9.5 14 14 12 14 9.5 12 5 9.5 5Z"/></svg>
|
||||||
|
</label>
|
||||||
|
<div class="md-search" data-md-component="search" role="dialog">
|
||||||
|
<label class="md-search__overlay" for="__search"></label>
|
||||||
|
<div class="md-search__inner" role="search">
|
||||||
|
<form class="md-search__form" name="search">
|
||||||
|
<input type="text" class="md-search__input" name="query" aria-label="Search" placeholder="Search" autocapitalize="off" autocorrect="off" autocomplete="off" spellcheck="false" data-md-component="search-query" required>
|
||||||
|
<label class="md-search__icon md-icon" for="__search">
|
||||||
|
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M9.5 3A6.5 6.5 0 0 1 16 9.5c0 1.61-.59 3.09-1.56 4.23l.27.27h.79l5 5-1.5 1.5-5-5v-.79l-.27-.27A6.516 6.516 0 0 1 9.5 16 6.5 6.5 0 0 1 3 9.5 6.5 6.5 0 0 1 9.5 3m0 2C7 5 5 7 5 9.5S7 14 9.5 14 14 12 14 9.5 12 5 9.5 5Z"/></svg>
|
||||||
|
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M20 11v2H8l5.5 5.5-1.42 1.42L4.16 12l7.92-7.92L13.5 5.5 8 11h12Z"/></svg>
|
||||||
|
</label>
|
||||||
|
<nav class="md-search__options" aria-label="Search">
|
||||||
|
|
||||||
|
<button type="reset" class="md-search__icon md-icon" aria-label="Clear" tabindex="-1">
|
||||||
|
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M19 6.41 17.59 5 12 10.59 6.41 5 5 6.41 10.59 12 5 17.59 6.41 19 12 13.41 17.59 19 19 17.59 13.41 12 19 6.41Z"/></svg>
|
||||||
|
</button>
|
||||||
|
</nav>
|
||||||
|
|
||||||
|
</form>
|
||||||
|
<div class="md-search__output">
|
||||||
|
<div class="md-search__scrollwrap" data-md-scrollfix>
|
||||||
|
<div class="md-search-result" data-md-component="search-result">
|
||||||
|
<div class="md-search-result__meta">
|
||||||
|
Initializing search
|
||||||
|
</div>
|
||||||
|
<ol class="md-search-result__list"></ol>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
|
||||||
|
<div class="md-header__source">
|
||||||
|
<a href="https://github.com/schmelczer/great-ai" title="Go to repository" class="md-source" data-md-component="source">
|
||||||
|
<div class="md-source__icon md-icon">
|
||||||
|
|
||||||
|
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 448 512"><!--! Font Awesome Free 6.1.1 by @fontawesome - https://fontawesome.com License - https://fontawesome.com/license/free (Icons: CC BY 4.0, Fonts: SIL OFL 1.1, Code: MIT License) Copyright 2022 Fonticons, Inc.--><path d="M439.55 236.05 244 40.45a28.87 28.87 0 0 0-40.81 0l-40.66 40.63 51.52 51.52c27.06-9.14 52.68 16.77 43.39 43.68l49.66 49.66c34.23-11.8 61.18 31 35.47 56.69-26.49 26.49-70.21-2.87-56-37.34L240.22 199v121.85c25.3 12.54 22.26 41.85 9.08 55a34.34 34.34 0 0 1-48.55 0c-17.57-17.6-11.07-46.91 11.25-56v-123c-20.8-8.51-24.6-30.74-18.64-45L142.57 101 8.45 235.14a28.86 28.86 0 0 0 0 40.81l195.61 195.6a28.86 28.86 0 0 0 40.8 0l194.69-194.69a28.86 28.86 0 0 0 0-40.81z"/></svg>
|
||||||
|
</div>
|
||||||
|
<div class="md-source__repository">
|
||||||
|
GitHub
|
||||||
|
</div>
|
||||||
|
</a>
|
||||||
|
</div>
|
||||||
|
|
||||||
</nav>
|
</nav>
|
||||||
|
|
||||||
</header>
|
</header>
|
||||||
|
|
@ -138,6 +209,18 @@
|
||||||
GreatAI documentation
|
GreatAI documentation
|
||||||
</label>
|
</label>
|
||||||
|
|
||||||
|
<div class="md-nav__source">
|
||||||
|
<a href="https://github.com/schmelczer/great-ai" title="Go to repository" class="md-source" data-md-component="source">
|
||||||
|
<div class="md-source__icon md-icon">
|
||||||
|
|
||||||
|
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 448 512"><!--! Font Awesome Free 6.1.1 by @fontawesome - https://fontawesome.com License - https://fontawesome.com/license/free (Icons: CC BY 4.0, Fonts: SIL OFL 1.1, Code: MIT License) Copyright 2022 Fonticons, Inc.--><path d="M439.55 236.05 244 40.45a28.87 28.87 0 0 0-40.81 0l-40.66 40.63 51.52 51.52c27.06-9.14 52.68 16.77 43.39 43.68l49.66 49.66c34.23-11.8 61.18 31 35.47 56.69-26.49 26.49-70.21-2.87-56-37.34L240.22 199v121.85c25.3 12.54 22.26 41.85 9.08 55a34.34 34.34 0 0 1-48.55 0c-17.57-17.6-11.07-46.91 11.25-56v-123c-20.8-8.51-24.6-30.74-18.64-45L142.57 101 8.45 235.14a28.86 28.86 0 0 0 0 40.81l195.61 195.6a28.86 28.86 0 0 0 40.8 0l194.69-194.69a28.86 28.86 0 0 0 0-40.81z"/></svg>
|
||||||
|
</div>
|
||||||
|
<div class="md-source__repository">
|
||||||
|
GitHub
|
||||||
|
</div>
|
||||||
|
</a>
|
||||||
|
</div>
|
||||||
|
|
||||||
<ul class="md-nav__list" data-md-scrollfix>
|
<ul class="md-nav__list" data-md-scrollfix>
|
||||||
|
|
||||||
|
|
||||||
|
|
@ -263,6 +346,21 @@
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
<li class="md-nav__item">
|
||||||
|
<a href="simple-mag/train/" class="md-nav__link">
|
||||||
|
Train a field of study (domain) classification of sentences on the MAG dataset
|
||||||
|
</a>
|
||||||
|
</li>
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
</ul>
|
</ul>
|
||||||
</nav>
|
</nav>
|
||||||
</div>
|
</div>
|
||||||
|
|
@ -314,7 +412,14 @@
|
||||||
<div class="md-content" data-md-component="content">
|
<div class="md-content" data-md-component="content">
|
||||||
<article class="md-content__inner md-typeset">
|
<article class="md-content__inner md-typeset">
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
<a href="https://github.com/schmelczer/great-ai/edit/main/docs/index.md" title="Edit this page" class="md-content__button md-icon">
|
||||||
|
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M20.71 7.04c.39-.39.39-1.04 0-1.41l-2.34-2.34c-.37-.39-1.02-.39-1.41 0l-1.84 1.83 3.75 3.75M3 17.25V21h3.75L17.81 9.93l-3.75-3.75L3 17.25Z"/></svg>
|
||||||
|
</a>
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
<h1 id="welcome-to-mkdocs">Welcome to MkDocs</h1>
|
<h1 id="welcome-to-mkdocs">Welcome to MkDocs</h1>
|
||||||
|
|
@ -333,7 +438,19 @@ docs/
|
||||||
... # Other markdown pages, images and other files.
|
... # Other markdown pages, images and other files.
|
||||||
</code></pre>
|
</code></pre>
|
||||||
|
|
||||||
|
<hr>
|
||||||
|
<div class="md-source-file">
|
||||||
|
<small>
|
||||||
|
|
||||||
|
Last update:
|
||||||
|
<span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-date">July 8, 2022</span>
|
||||||
|
|
||||||
|
|
||||||
|
</small>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
</article>
|
</article>
|
||||||
|
|
||||||
</div>
|
</div>
|
||||||
|
|
@ -387,7 +504,7 @@ docs/
|
||||||
<script id="__config" type="application/json">{"base": ".", "features": [], "search": "assets/javascripts/workers/search.b97dbffb.min.js", "translations": {"clipboard.copied": "Copied to clipboard", "clipboard.copy": "Copy to clipboard", "search.config.lang": "en", "search.config.pipeline": "trimmer, stopWordFilter", "search.config.separator": "[\\s\\-]+", "search.placeholder": "Search", "search.result.more.one": "1 more on this page", "search.result.more.other": "# more on this page", "search.result.none": "No matching documents", "search.result.one": "1 matching document", "search.result.other": "# matching documents", "search.result.placeholder": "Type to start searching", "search.result.term.missing": "Missing", "select.version.title": "Select version"}}</script>
|
<script id="__config" type="application/json">{"base": ".", "features": [], "search": "assets/javascripts/workers/search.b97dbffb.min.js", "translations": {"clipboard.copied": "Copied to clipboard", "clipboard.copy": "Copy to clipboard", "search.config.lang": "en", "search.config.pipeline": "trimmer, stopWordFilter", "search.config.separator": "[\\s\\-]+", "search.placeholder": "Search", "search.result.more.one": "1 more on this page", "search.result.more.other": "# more on this page", "search.result.none": "No matching documents", "search.result.one": "1 matching document", "search.result.other": "# matching documents", "search.result.placeholder": "Type to start searching", "search.result.term.missing": "Missing", "select.version.title": "Select version"}}</script>
|
||||||
|
|
||||||
|
|
||||||
<script src="assets/javascripts/bundle.0238f547.min.js"></script>
|
<script src="assets/javascripts/bundle.6c7ad80a.min.js"></script>
|
||||||
|
|
||||||
|
|
||||||
</body>
|
</body>
|
||||||
|
|
|
||||||
17
index.md
Normal file
17
index.md
Normal file
|
|
@ -0,0 +1,17 @@
|
||||||
|
# Welcome to MkDocs
|
||||||
|
|
||||||
|
For full documentation visit [mkdocs.org](https://www.mkdocs.org).
|
||||||
|
|
||||||
|
## Commands
|
||||||
|
|
||||||
|
* `mkdocs new [dir-name]` - Create a new project.
|
||||||
|
* `mkdocs serve` - Start the live-reloading docs server.
|
||||||
|
* `mkdocs build` - Build the documentation site.
|
||||||
|
* `mkdocs -h` - Print help message and exit.
|
||||||
|
|
||||||
|
## Project layout
|
||||||
|
|
||||||
|
mkdocs.yml # The configuration file.
|
||||||
|
docs/
|
||||||
|
index.md # The documentation homepage.
|
||||||
|
... # Other markdown pages, images and other files.
|
||||||
189
map.ipynb
189
map.ipynb
|
|
@ -1,189 +0,0 @@
|
||||||
{
|
|
||||||
"cells": [
|
|
||||||
{
|
|
||||||
"cell_type": "code",
|
|
||||||
"execution_count": 1,
|
|
||||||
"metadata": {},
|
|
||||||
"outputs": [
|
|
||||||
{
|
|
||||||
"ename": "WorkerException",
|
|
||||||
"evalue": "",
|
|
||||||
"output_type": "error",
|
|
||||||
"traceback": [
|
|
||||||
"\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
|
|
||||||
"\u001b[0;31mValueError\u001b[0m Traceback (most recent call last)",
|
|
||||||
"\u001b[0;31mValueError\u001b[0m: hi",
|
|
||||||
"\nThe above exception was the direct cause of the following exception:\n",
|
|
||||||
"\u001b[0;31mWorkerException\u001b[0m Traceback (most recent call last)",
|
|
||||||
"\u001b[1;32m/data/projects/great_ai/docs/map.ipynb Cell 1'\u001b[0m in \u001b[0;36m<cell line: 8>\u001b[0;34m()\u001b[0m\n\u001b[1;32m <a href='vscode-notebook-cell:/data/projects/great_ai/docs/map.ipynb#ch0000000?line=3'>4</a>\u001b[0m \u001b[39mdef\u001b[39;00m \u001b[39moh_no\u001b[39m(_):\n\u001b[1;32m <a href='vscode-notebook-cell:/data/projects/great_ai/docs/map.ipynb#ch0000000?line=4'>5</a>\u001b[0m \u001b[39mraise\u001b[39;00m \u001b[39mValueError\u001b[39;00m(\u001b[39m\"\u001b[39m\u001b[39mhi\u001b[39m\u001b[39m\"\u001b[39m)\n\u001b[0;32m----> <a href='vscode-notebook-cell:/data/projects/great_ai/docs/map.ipynb#ch0000000?line=7'>8</a>\u001b[0m \u001b[39mlist\u001b[39;49m(parallel_map(oh_no, \u001b[39mrange\u001b[39;49m(\u001b[39m10000\u001b[39;49m), concurrency\u001b[39m=\u001b[39;49m\u001b[39m2\u001b[39;49m))\n",
|
|
||||||
"File \u001b[0;32m/data/projects/great_ai/src/great_ai/utilities/parallel_map/parallel_map.py:120\u001b[0m, in \u001b[0;36mparallel_map\u001b[0;34m(function, input_values, chunk_size, concurrency, unordered, ignore_exceptions)\u001b[0m\n\u001b[1;32m 117\u001b[0m p\u001b[39m.\u001b[39mstart()\n\u001b[1;32m 119\u001b[0m \u001b[39mtry\u001b[39;00m:\n\u001b[0;32m--> 120\u001b[0m \u001b[39myield from\u001b[39;00m manage_communication(\n\u001b[1;32m 121\u001b[0m input_values\u001b[39m=\u001b[39minput_values,\n\u001b[1;32m 122\u001b[0m chunk_size\u001b[39m=\u001b[39mconfig\u001b[39m.\u001b[39mchunk_size,\n\u001b[1;32m 123\u001b[0m input_queue\u001b[39m=\u001b[39minput_queue,\n\u001b[1;32m 124\u001b[0m output_queue\u001b[39m=\u001b[39moutput_queue,\n\u001b[1;32m 125\u001b[0m unordered\u001b[39m=\u001b[39munordered,\n\u001b[1;32m 126\u001b[0m ignore_exceptions\u001b[39m=\u001b[39mignore_exceptions,\n\u001b[1;32m 127\u001b[0m )\n\u001b[1;32m 128\u001b[0m should_stop\u001b[39m.\u001b[39mset()\n\u001b[1;32m 129\u001b[0m \u001b[39mexcept\u001b[39;00m WorkerException:\n",
|
|
||||||
"File \u001b[0;32m/data/projects/great_ai/src/great_ai/utilities/parallel_map/manage_communication.py:53\u001b[0m, in \u001b[0;36mmanage_communication\u001b[0;34m(input_values, chunk_size, input_queue, output_queue, unordered, ignore_exceptions)\u001b[0m\n\u001b[1;32m 51\u001b[0m e, tb \u001b[39m=\u001b[39m exception\n\u001b[1;32m 52\u001b[0m \u001b[39mif\u001b[39;00m \u001b[39mnot\u001b[39;00m ignore_exceptions:\n\u001b[0;32m---> 53\u001b[0m \u001b[39mraise\u001b[39;00m WorkerException \u001b[39mfrom\u001b[39;00m \u001b[39me\u001b[39;00m\n\u001b[1;32m 54\u001b[0m \u001b[39melse\u001b[39;00m:\n\u001b[1;32m 55\u001b[0m logger\u001b[39m.\u001b[39merror(\n\u001b[1;32m 56\u001b[0m \u001b[39mf\u001b[39m\u001b[39m\"\u001b[39m\u001b[39mException \u001b[39m\u001b[39m{\u001b[39;00me\u001b[39m}\u001b[39;00m\u001b[39m encountered in worker, traceback:\u001b[39m\u001b[39m\\n\u001b[39;00m\u001b[39m{\u001b[39;00mtb\u001b[39m}\u001b[39;00m\u001b[39m\"\u001b[39m\n\u001b[1;32m 57\u001b[0m )\n",
|
|
||||||
"\u001b[0;31mWorkerException\u001b[0m: "
|
|
||||||
]
|
|
||||||
}
|
|
||||||
],
|
|
||||||
"source": [
|
|
||||||
"from great_ai.utilities import parallel_map\n",
|
|
||||||
"\n",
|
|
||||||
"\n",
|
|
||||||
"def oh_no(_):\n",
|
|
||||||
" raise ValueError(\"hi\")\n",
|
|
||||||
"\n",
|
|
||||||
"\n",
|
|
||||||
"list(parallel_map(oh_no, range(10000), concurrency=2))\n",
|
|
||||||
"\n",
|
|
||||||
"# list(parallel_map(oh_no, range(10), concurrency=1))"
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"cell_type": "code",
|
|
||||||
"execution_count": null,
|
|
||||||
"metadata": {},
|
|
||||||
"outputs": [],
|
|
||||||
"source": [
|
|
||||||
"from great_ai.utilities import parallel_map\n",
|
|
||||||
"\n",
|
|
||||||
"COUNT = int(1e5) + 3\n",
|
|
||||||
"\n",
|
|
||||||
"\n",
|
|
||||||
"inputs = range(COUNT)\n",
|
|
||||||
"expected = [v**2 for v in range(COUNT)]\n",
|
|
||||||
"\n",
|
|
||||||
"assert list(parallel_map(lambda v: v**2, inputs, disable_logging=False)) == expected"
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"cell_type": "code",
|
|
||||||
"execution_count": null,
|
|
||||||
"metadata": {},
|
|
||||||
"outputs": [],
|
|
||||||
"source": [
|
|
||||||
"# from time import sleep\n",
|
|
||||||
"# from great_ai.utilities import chunk, threaded_parallel_map, parallel_map, unchunk\n",
|
|
||||||
"\n",
|
|
||||||
"\n",
|
|
||||||
"# def generate_urls():\n",
|
|
||||||
"# for i in range(20000):\n",
|
|
||||||
"# # print(f\"yield {i}.com\")\n",
|
|
||||||
"# yield f\"{i}.com\"\n",
|
|
||||||
"\n",
|
|
||||||
"\n",
|
|
||||||
"# def scrape(url):\n",
|
|
||||||
"# sleep(2) # scrape\n",
|
|
||||||
"# # print(f\"scraped {url}\")\n",
|
|
||||||
"# return url + \" html\"\n",
|
|
||||||
"\n",
|
|
||||||
"\n",
|
|
||||||
"# def scrape_bulk(urls):\n",
|
|
||||||
"# return list(\n",
|
|
||||||
"# threaded_parallel_map(scrape, urls, concurrency=100, disable_logging=True)\n",
|
|
||||||
"# )\n",
|
|
||||||
"\n",
|
|
||||||
"\n",
|
|
||||||
"# def process(html):\n",
|
|
||||||
"# sleep(1) # process\n",
|
|
||||||
"# # print(f\"processed {html}\")\n",
|
|
||||||
"\n",
|
|
||||||
"\n",
|
|
||||||
"# scraped_pages_stream = parallel_map(\n",
|
|
||||||
"# scrape_bulk,\n",
|
|
||||||
"# chunk(generate_urls(), 100),\n",
|
|
||||||
"# chunk_size=1,\n",
|
|
||||||
"# concurrency=1,\n",
|
|
||||||
"# disable_logging=True,\n",
|
|
||||||
"# unordered=True,\n",
|
|
||||||
"# )\n",
|
|
||||||
"# list(\n",
|
|
||||||
"# threaded_parallel_map(\n",
|
|
||||||
"# process,\n",
|
|
||||||
"# unchunk(scraped_pages_stream),\n",
|
|
||||||
"# chunk_size=1,\n",
|
|
||||||
"# concurrency=50,\n",
|
|
||||||
"# unordered=True,\n",
|
|
||||||
"# )\n",
|
|
||||||
"# )\n",
|
|
||||||
"\n",
|
|
||||||
"# None"
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"cell_type": "code",
|
|
||||||
"execution_count": null,
|
|
||||||
"metadata": {},
|
|
||||||
"outputs": [],
|
|
||||||
"source": [
|
|
||||||
"# def a():\n",
|
|
||||||
"# yield [2]\n",
|
|
||||||
"# yield [6]\n",
|
|
||||||
"# raise Exception()\n",
|
|
||||||
"# yield 3\n",
|
|
||||||
"\n",
|
|
||||||
"\n",
|
|
||||||
"# a = a()"
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"cell_type": "code",
|
|
||||||
"execution_count": null,
|
|
||||||
"metadata": {},
|
|
||||||
"outputs": [],
|
|
||||||
"source": [
|
|
||||||
"list(unchunk(a))"
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"cell_type": "code",
|
|
||||||
"execution_count": null,
|
|
||||||
"metadata": {},
|
|
||||||
"outputs": [],
|
|
||||||
"source": [
|
|
||||||
"# list(map(lambda x: print(x), [1, 2, 4]))"
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"cell_type": "code",
|
|
||||||
"execution_count": null,
|
|
||||||
"metadata": {},
|
|
||||||
"outputs": [],
|
|
||||||
"source": [
|
|
||||||
"# from time import sleep\n",
|
|
||||||
"\n",
|
|
||||||
"\n",
|
|
||||||
"# def my_generator():\n",
|
|
||||||
"# for i in range(10):\n",
|
|
||||||
"# yield i\n",
|
|
||||||
"# sleep(0.2)\n",
|
|
||||||
"\n",
|
|
||||||
"\n",
|
|
||||||
"# list(parallel_map(lambda x: x**3, my_generator(), chunk_length=1))"
|
|
||||||
]
|
|
||||||
}
|
|
||||||
],
|
|
||||||
"metadata": {
|
|
||||||
"kernelspec": {
|
|
||||||
"display_name": "Python 3.10.4 ('.env': venv)",
|
|
||||||
"language": "python",
|
|
||||||
"name": "python3"
|
|
||||||
},
|
|
||||||
"language_info": {
|
|
||||||
"codemirror_mode": {
|
|
||||||
"name": "ipython",
|
|
||||||
"version": 3
|
|
||||||
},
|
|
||||||
"file_extension": ".py",
|
|
||||||
"mimetype": "text/x-python",
|
|
||||||
"name": "python",
|
|
||||||
"nbconvert_exporter": "python",
|
|
||||||
"pygments_lexer": "ipython3",
|
|
||||||
"version": "3.10.4"
|
|
||||||
},
|
|
||||||
"orig_nbformat": 4,
|
|
||||||
"vscode": {
|
|
||||||
"interpreter": {
|
|
||||||
"hash": "d88b0dd276e3f918f7798b7e97af2e3c2f843817b9e5b55a9df0a682ffd80f44"
|
|
||||||
}
|
|
||||||
}
|
|
||||||
},
|
|
||||||
"nbformat": 4,
|
|
||||||
"nbformat_minor": 2
|
|
||||||
}
|
|
||||||
BIN
objects.inv
BIN
objects.inv
Binary file not shown.
|
|
@ -9,7 +9,7 @@
|
||||||
|
|
||||||
|
|
||||||
<link rel="icon" href="../assets/images/favicon.png">
|
<link rel="icon" href="../assets/images/favicon.png">
|
||||||
<meta name="generator" content="mkdocs-1.3.0, mkdocs-material-8.3.8">
|
<meta name="generator" content="mkdocs-1.3.0, mkdocs-material-8.3.9">
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
@ -49,13 +49,17 @@
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
<body dir="ltr" data-md-color-scheme="" data-md-color-primary="none" data-md-color-accent="none">
|
|
||||||
|
<body dir="ltr" data-md-color-scheme="default" data-md-color-primary="" data-md-color-accent="">
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
<script>var palette=__md_get("__palette");if(palette&&"object"==typeof palette.color)for(var key of Object.keys(palette.color))document.body.setAttribute("data-md-color-"+key,palette.color[key])</script>
|
||||||
|
|
||||||
<input class="md-toggle" data-md-toggle="drawer" type="checkbox" id="__drawer" autocomplete="off">
|
<input class="md-toggle" data-md-toggle="drawer" type="checkbox" id="__drawer" autocomplete="off">
|
||||||
<input class="md-toggle" data-md-toggle="search" type="checkbox" id="__search" autocomplete="off">
|
<input class="md-toggle" data-md-toggle="search" type="checkbox" id="__search" autocomplete="off">
|
||||||
<label class="md-overlay" for="__drawer"></label>
|
<label class="md-overlay" for="__drawer"></label>
|
||||||
|
|
@ -102,9 +106,76 @@
|
||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
|
<form class="md-header__option" data-md-component="palette">
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
<input class="md-option" data-md-color-media="" data-md-color-scheme="default" data-md-color-primary="" data-md-color-accent="" aria-label="Switch to dark mode" type="radio" name="__palette" id="__palette_1">
|
||||||
|
|
||||||
|
<label class="md-header__button md-icon" title="Switch to dark mode" for="__palette_2" hidden>
|
||||||
|
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M12 8a4 4 0 0 0-4 4 4 4 0 0 0 4 4 4 4 0 0 0 4-4 4 4 0 0 0-4-4m0 10a6 6 0 0 1-6-6 6 6 0 0 1 6-6 6 6 0 0 1 6 6 6 6 0 0 1-6 6m8-9.31V4h-4.69L12 .69 8.69 4H4v4.69L.69 12 4 15.31V20h4.69L12 23.31 15.31 20H20v-4.69L23.31 12 20 8.69Z"/></svg>
|
||||||
|
</label>
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
<input class="md-option" data-md-color-media="" data-md-color-scheme="slate" data-md-color-primary="" data-md-color-accent="" aria-label="Switch to light mode" type="radio" name="__palette" id="__palette_2">
|
||||||
|
|
||||||
|
<label class="md-header__button md-icon" title="Switch to light mode" for="__palette_1" hidden>
|
||||||
|
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M12 18c-.89 0-1.74-.2-2.5-.55C11.56 16.5 13 14.42 13 12c0-2.42-1.44-4.5-3.5-5.45C10.26 6.2 11.11 6 12 6a6 6 0 0 1 6 6 6 6 0 0 1-6 6m8-9.31V4h-4.69L12 .69 8.69 4H4v4.69L.69 12 4 15.31V20h4.69L12 23.31 15.31 20H20v-4.69L23.31 12 20 8.69Z"/></svg>
|
||||||
|
</label>
|
||||||
|
|
||||||
|
|
||||||
|
</form>
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
<label class="md-header__button md-icon" for="__search">
|
||||||
|
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M9.5 3A6.5 6.5 0 0 1 16 9.5c0 1.61-.59 3.09-1.56 4.23l.27.27h.79l5 5-1.5 1.5-5-5v-.79l-.27-.27A6.516 6.516 0 0 1 9.5 16 6.5 6.5 0 0 1 3 9.5 6.5 6.5 0 0 1 9.5 3m0 2C7 5 5 7 5 9.5S7 14 9.5 14 14 12 14 9.5 12 5 9.5 5Z"/></svg>
|
||||||
|
</label>
|
||||||
|
<div class="md-search" data-md-component="search" role="dialog">
|
||||||
|
<label class="md-search__overlay" for="__search"></label>
|
||||||
|
<div class="md-search__inner" role="search">
|
||||||
|
<form class="md-search__form" name="search">
|
||||||
|
<input type="text" class="md-search__input" name="query" aria-label="Search" placeholder="Search" autocapitalize="off" autocorrect="off" autocomplete="off" spellcheck="false" data-md-component="search-query" required>
|
||||||
|
<label class="md-search__icon md-icon" for="__search">
|
||||||
|
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M9.5 3A6.5 6.5 0 0 1 16 9.5c0 1.61-.59 3.09-1.56 4.23l.27.27h.79l5 5-1.5 1.5-5-5v-.79l-.27-.27A6.516 6.516 0 0 1 9.5 16 6.5 6.5 0 0 1 3 9.5 6.5 6.5 0 0 1 9.5 3m0 2C7 5 5 7 5 9.5S7 14 9.5 14 14 12 14 9.5 12 5 9.5 5Z"/></svg>
|
||||||
|
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M20 11v2H8l5.5 5.5-1.42 1.42L4.16 12l7.92-7.92L13.5 5.5 8 11h12Z"/></svg>
|
||||||
|
</label>
|
||||||
|
<nav class="md-search__options" aria-label="Search">
|
||||||
|
|
||||||
|
<button type="reset" class="md-search__icon md-icon" aria-label="Clear" tabindex="-1">
|
||||||
|
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M19 6.41 17.59 5 12 10.59 6.41 5 5 6.41 10.59 12 5 17.59 6.41 19 12 13.41 17.59 19 19 17.59 13.41 12 19 6.41Z"/></svg>
|
||||||
|
</button>
|
||||||
|
</nav>
|
||||||
|
|
||||||
|
</form>
|
||||||
|
<div class="md-search__output">
|
||||||
|
<div class="md-search__scrollwrap" data-md-scrollfix>
|
||||||
|
<div class="md-search-result" data-md-component="search-result">
|
||||||
|
<div class="md-search-result__meta">
|
||||||
|
Initializing search
|
||||||
|
</div>
|
||||||
|
<ol class="md-search-result__list"></ol>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
|
||||||
|
<div class="md-header__source">
|
||||||
|
<a href="https://github.com/schmelczer/great-ai" title="Go to repository" class="md-source" data-md-component="source">
|
||||||
|
<div class="md-source__icon md-icon">
|
||||||
|
|
||||||
|
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 448 512"><!--! Font Awesome Free 6.1.1 by @fontawesome - https://fontawesome.com License - https://fontawesome.com/license/free (Icons: CC BY 4.0, Fonts: SIL OFL 1.1, Code: MIT License) Copyright 2022 Fonticons, Inc.--><path d="M439.55 236.05 244 40.45a28.87 28.87 0 0 0-40.81 0l-40.66 40.63 51.52 51.52c27.06-9.14 52.68 16.77 43.39 43.68l49.66 49.66c34.23-11.8 61.18 31 35.47 56.69-26.49 26.49-70.21-2.87-56-37.34L240.22 199v121.85c25.3 12.54 22.26 41.85 9.08 55a34.34 34.34 0 0 1-48.55 0c-17.57-17.6-11.07-46.91 11.25-56v-123c-20.8-8.51-24.6-30.74-18.64-45L142.57 101 8.45 235.14a28.86 28.86 0 0 0 0 40.81l195.61 195.6a28.86 28.86 0 0 0 40.8 0l194.69-194.69a28.86 28.86 0 0 0 0-40.81z"/></svg>
|
||||||
|
</div>
|
||||||
|
<div class="md-source__repository">
|
||||||
|
GitHub
|
||||||
|
</div>
|
||||||
|
</a>
|
||||||
|
</div>
|
||||||
|
|
||||||
</nav>
|
</nav>
|
||||||
|
|
||||||
</header>
|
</header>
|
||||||
|
|
@ -138,6 +209,18 @@
|
||||||
GreatAI documentation
|
GreatAI documentation
|
||||||
</label>
|
</label>
|
||||||
|
|
||||||
|
<div class="md-nav__source">
|
||||||
|
<a href="https://github.com/schmelczer/great-ai" title="Go to repository" class="md-source" data-md-component="source">
|
||||||
|
<div class="md-source__icon md-icon">
|
||||||
|
|
||||||
|
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 448 512"><!--! Font Awesome Free 6.1.1 by @fontawesome - https://fontawesome.com License - https://fontawesome.com/license/free (Icons: CC BY 4.0, Fonts: SIL OFL 1.1, Code: MIT License) Copyright 2022 Fonticons, Inc.--><path d="M439.55 236.05 244 40.45a28.87 28.87 0 0 0-40.81 0l-40.66 40.63 51.52 51.52c27.06-9.14 52.68 16.77 43.39 43.68l49.66 49.66c34.23-11.8 61.18 31 35.47 56.69-26.49 26.49-70.21-2.87-56-37.34L240.22 199v121.85c25.3 12.54 22.26 41.85 9.08 55a34.34 34.34 0 0 1-48.55 0c-17.57-17.6-11.07-46.91 11.25-56v-123c-20.8-8.51-24.6-30.74-18.64-45L142.57 101 8.45 235.14a28.86 28.86 0 0 0 0 40.81l195.61 195.6a28.86 28.86 0 0 0 40.8 0l194.69-194.69a28.86 28.86 0 0 0 0-40.81z"/></svg>
|
||||||
|
</div>
|
||||||
|
<div class="md-source__repository">
|
||||||
|
GitHub
|
||||||
|
</div>
|
||||||
|
</a>
|
||||||
|
</div>
|
||||||
|
|
||||||
<ul class="md-nav__list" data-md-scrollfix>
|
<ul class="md-nav__list" data-md-scrollfix>
|
||||||
|
|
||||||
|
|
||||||
|
|
@ -215,6 +298,21 @@
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
<li class="md-nav__item">
|
||||||
|
<a href="../simple-mag/train/" class="md-nav__link">
|
||||||
|
Train a field of study (domain) classification of sentences on the MAG dataset
|
||||||
|
</a>
|
||||||
|
</li>
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
</ul>
|
</ul>
|
||||||
</nav>
|
</nav>
|
||||||
</div>
|
</div>
|
||||||
|
|
@ -347,7 +445,14 @@
|
||||||
<div class="md-content" data-md-component="content">
|
<div class="md-content" data-md-component="content">
|
||||||
<article class="md-content__inner md-typeset">
|
<article class="md-content__inner md-typeset">
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
<a href="https://github.com/schmelczer/great-ai/edit/main/docs/open_s3.md" title="Edit this page" class="md-content__button md-icon">
|
||||||
|
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M20.71 7.04c.39-.39.39-1.04 0-1.41l-2.34-2.34c-.37-.39-1.02-.39-1.41 0l-1.84 1.83 3.75 3.75M3 17.25V21h3.75L17.81 9.93l-3.75-3.75L3 17.25Z"/></svg>
|
||||||
|
</a>
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
<h1 id="opens3"><a href="https://pypi.org/project/open-large/">open(S3)</a></h1>
|
<h1 id="opens3"><a href="https://pypi.org/project/open-large/">open(S3)</a></h1>
|
||||||
|
|
@ -358,7 +463,7 @@
|
||||||
<pre><code>pip install open-large
|
<pre><code>pip install open-large
|
||||||
</code></pre>
|
</code></pre>
|
||||||
<h3 id="simple-example">Simple example</h3>
|
<h3 id="simple-example">Simple example</h3>
|
||||||
<pre><code class="language-python">from large_file import LargeFileS3
|
<pre><code class="language-python">from great_ai.large_file import LargeFileS3
|
||||||
|
|
||||||
LargeFileS3.configure_credentials({
|
LargeFileS3.configure_credentials({
|
||||||
"aws_region_name": "your_region_like_eu-west-2",
|
"aws_region_name": "your_region_like_eu-west-2",
|
||||||
|
|
@ -434,7 +539,19 @@ endpoint_url = this is optional, for backblaze, use this: https://s3.us-west-002
|
||||||
<pre><code class="language-sh">python3 -m large_file --backend s3 --secrets ~/.aws/credentials --delete my_first_file.json
|
<pre><code class="language-sh">python3 -m large_file --backend s3 --secrets ~/.aws/credentials --delete my_first_file.json
|
||||||
</code></pre>
|
</code></pre>
|
||||||
|
|
||||||
|
<hr>
|
||||||
|
<div class="md-source-file">
|
||||||
|
<small>
|
||||||
|
|
||||||
|
Last update:
|
||||||
|
<span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-date">July 8, 2022</span>
|
||||||
|
|
||||||
|
|
||||||
|
</small>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
</article>
|
</article>
|
||||||
|
|
||||||
</div>
|
</div>
|
||||||
|
|
@ -467,7 +584,7 @@ endpoint_url = this is optional, for backblaze, use this: https://s3.us-west-002
|
||||||
<script id="__config" type="application/json">{"base": "..", "features": [], "search": "../assets/javascripts/workers/search.b97dbffb.min.js", "translations": {"clipboard.copied": "Copied to clipboard", "clipboard.copy": "Copy to clipboard", "search.config.lang": "en", "search.config.pipeline": "trimmer, stopWordFilter", "search.config.separator": "[\\s\\-]+", "search.placeholder": "Search", "search.result.more.one": "1 more on this page", "search.result.more.other": "# more on this page", "search.result.none": "No matching documents", "search.result.one": "1 matching document", "search.result.other": "# matching documents", "search.result.placeholder": "Type to start searching", "search.result.term.missing": "Missing", "select.version.title": "Select version"}}</script>
|
<script id="__config" type="application/json">{"base": "..", "features": [], "search": "../assets/javascripts/workers/search.b97dbffb.min.js", "translations": {"clipboard.copied": "Copied to clipboard", "clipboard.copy": "Copy to clipboard", "search.config.lang": "en", "search.config.pipeline": "trimmer, stopWordFilter", "search.config.separator": "[\\s\\-]+", "search.placeholder": "Search", "search.result.more.one": "1 more on this page", "search.result.more.other": "# more on this page", "search.result.none": "No matching documents", "search.result.one": "1 matching document", "search.result.other": "# matching documents", "search.result.placeholder": "Type to start searching", "search.result.term.missing": "Missing", "select.version.title": "Select version"}}</script>
|
||||||
|
|
||||||
|
|
||||||
<script src="../assets/javascripts/bundle.0238f547.min.js"></script>
|
<script src="../assets/javascripts/bundle.6c7ad80a.min.js"></script>
|
||||||
|
|
||||||
|
|
||||||
</body>
|
</body>
|
||||||
|
|
|
||||||
118
open_s3/open_s3.md
Normal file
118
open_s3/open_s3.md
Normal file
|
|
@ -0,0 +1,118 @@
|
||||||
|
# [open(S3)](https://pypi.org/project/open-large/)
|
||||||
|
|
||||||
|
Storing, versioning, and downloading files from S3 made as easy as using `open()` in Python. Caching included.
|
||||||
|
|
||||||
|
## Motivation
|
||||||
|
|
||||||
|
Oftentimes, especially when working with data-heavy applications, large files can proliferate in a repository. Version controlling them is an obvious next step, however, GitHub's git LFS implementation [doesn't support the deletion](https://docs.github.com/en/repositories/working-with-files/managing-large-files/removing-files-from-git-large-file-storage#git-lfs-objects-in-your-repository) of large files, making it easy for them to eat-up the LFS quota and explode the size of your repos.
|
||||||
|
|
||||||
|
## Solution
|
||||||
|
|
||||||
|
```
|
||||||
|
pip install open-large
|
||||||
|
```
|
||||||
|
|
||||||
|
### Simple example
|
||||||
|
|
||||||
|
```python
|
||||||
|
from great_ai.large_file import LargeFileS3
|
||||||
|
|
||||||
|
LargeFileS3.configure_credentials({
|
||||||
|
"aws_region_name": "your_region_like_eu-west-2",
|
||||||
|
"aws_access_key_id": "YOUR_ACCESS_KEY_ID",
|
||||||
|
"aws_secret_access_key": "YOUR_VERY_SECRET_ACCESS_KEY",
|
||||||
|
"large_files_bucket_name": "create_a_bucket_and_put_its_name_here",
|
||||||
|
})
|
||||||
|
|
||||||
|
# Creates a new version and deletes the older version leaving the 3 most recently used intact
|
||||||
|
with LargeFileS3("test.txt", "w", keep_last_n=3) as f:
|
||||||
|
for i in range(100000):
|
||||||
|
f.write('test\n')
|
||||||
|
|
||||||
|
# By default the latest version is returned
|
||||||
|
# but an optional `version` keyword argument can be provided as well
|
||||||
|
with LargeFileS3("test.txt", "r") as f:
|
||||||
|
print(f.readlines()[0])
|
||||||
|
```
|
||||||
|
|
||||||
|
> Automatically creates a file, writes to it, uploads it to S3, and then queries the most recent version of it.
|
||||||
|
> In this case, the latest version is already in the local cache, no download is required.
|
||||||
|
|
||||||
|
### More details
|
||||||
|
|
||||||
|
`LargeFile` behaves like an opened file (in the background it is a temp file after all). Binary reading and writing is supported along with the [different keywords](https://docs.python.org/3/library/functions.html#open) `open()` accepts.
|
||||||
|
|
||||||
|
The local cache can be configured with these properties:
|
||||||
|
|
||||||
|
```python
|
||||||
|
LargeFile.cache_path = Path('.cache')
|
||||||
|
LargeFile.max_cache_size = "30 GB"
|
||||||
|
```
|
||||||
|
|
||||||
|
#### I only need a path
|
||||||
|
|
||||||
|
In case you only need a path to the "remote" file, this pattern can be applied:
|
||||||
|
|
||||||
|
```python
|
||||||
|
path_to_model = LargeFile("folder-of-my-bert-model", version=31).get()
|
||||||
|
```
|
||||||
|
|
||||||
|
> This will first download the file/folder into your local cache folder. Then, it returns a `Path` object to the local version. Which can be turned into a string with `str(path_to_model)`.
|
||||||
|
|
||||||
|
The same approach works for uploads:
|
||||||
|
|
||||||
|
```python
|
||||||
|
LargeFile("folder-of-my-bert-model").push('path_to_local/folder_or_file')
|
||||||
|
```
|
||||||
|
|
||||||
|
> This way, both regular files and folders can be handled. The uploaded file is called **folder-of-my-bert-model**, the local name is ignored.
|
||||||
|
|
||||||
|
Lastly, all version of the remote object can be deleted by calling `LargeFile("my-file").delete()`. It will still reside in your local cache afterwards, its deletion will happen next time your local cache has to be pruned.
|
||||||
|
|
||||||
|
### Command-line example
|
||||||
|
|
||||||
|
The package can be used as a module from the command-line to give you more flexibility.
|
||||||
|
|
||||||
|
#### Setup
|
||||||
|
|
||||||
|
Create an .ini file (or use _~/.aws/credentials_). It may look like this:
|
||||||
|
|
||||||
|
```ini
|
||||||
|
aws_region_name = your_region_like_eu-west-2
|
||||||
|
aws_access_key_id = YOUR_ACCESS_KEY_ID
|
||||||
|
aws_secret_access_key = YOUR_VERY_SECRET_ACCESS_KEY
|
||||||
|
large_files_bucket_name = my_large_files
|
||||||
|
endpoint_url = this is optional, for backblaze, use this: https://s3.us-west-002.backblazeb2.com
|
||||||
|
```
|
||||||
|
|
||||||
|
> Just like in [example secrets](example_secrets.ini).
|
||||||
|
|
||||||
|
#### Print the expected options
|
||||||
|
|
||||||
|
```sh
|
||||||
|
python3 -m large_file --help
|
||||||
|
```
|
||||||
|
|
||||||
|
#### Upload some files
|
||||||
|
|
||||||
|
```sh
|
||||||
|
python3 -m large_file --backend s3 --secrets secrets.ini --push my_first_file.json folder/my_second_file my_folder
|
||||||
|
```
|
||||||
|
|
||||||
|
> Only the filename is used as the S3 name, the rest of the path is ignored.
|
||||||
|
|
||||||
|
#### Download some files to the local cache
|
||||||
|
|
||||||
|
This can be useful when building a Docker image for example. This way, the files can already reside inside the container and need not be downloaded later.
|
||||||
|
|
||||||
|
```sh
|
||||||
|
python3 -m large_file --backend s3 -secrets ~/.aws/credentials --cache my_first_file.json:3 my_second_file my_folder:0
|
||||||
|
```
|
||||||
|
|
||||||
|
> Versions may be specified by using `:`-s.
|
||||||
|
|
||||||
|
#### Delete remote files
|
||||||
|
|
||||||
|
```sh
|
||||||
|
python3 -m large_file --backend s3 --secrets ~/.aws/credentials --delete my_first_file.json
|
||||||
|
```
|
||||||
11
overrides/main.html
Normal file
11
overrides/main.html
Normal file
|
|
@ -0,0 +1,11 @@
|
||||||
|
{% extends "base.html" %}
|
||||||
|
|
||||||
|
{% block content %}
|
||||||
|
{% if page.nb_url %}
|
||||||
|
<a href="{{ page.nb_url }}" title="Download Notebook" class="md-content__button md-icon">
|
||||||
|
{% include ".icons/material/download.svg" %}
|
||||||
|
</a>
|
||||||
|
{% endif %}
|
||||||
|
|
||||||
|
{{ super() }}
|
||||||
|
{% endblock content %}
|
||||||
|
|
@ -9,7 +9,7 @@
|
||||||
|
|
||||||
|
|
||||||
<link rel="icon" href="../assets/images/favicon.png">
|
<link rel="icon" href="../assets/images/favicon.png">
|
||||||
<meta name="generator" content="mkdocs-1.3.0, mkdocs-material-8.3.8">
|
<meta name="generator" content="mkdocs-1.3.0, mkdocs-material-8.3.9">
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
@ -49,20 +49,24 @@
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
<body dir="ltr" data-md-color-scheme="" data-md-color-primary="none" data-md-color-accent="none">
|
|
||||||
|
<body dir="ltr" data-md-color-scheme="default" data-md-color-primary="" data-md-color-accent="">
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
<script>var palette=__md_get("__palette");if(palette&&"object"==typeof palette.color)for(var key of Object.keys(palette.color))document.body.setAttribute("data-md-color-"+key,palette.color[key])</script>
|
||||||
|
|
||||||
<input class="md-toggle" data-md-toggle="drawer" type="checkbox" id="__drawer" autocomplete="off">
|
<input class="md-toggle" data-md-toggle="drawer" type="checkbox" id="__drawer" autocomplete="off">
|
||||||
<input class="md-toggle" data-md-toggle="search" type="checkbox" id="__search" autocomplete="off">
|
<input class="md-toggle" data-md-toggle="search" type="checkbox" id="__search" autocomplete="off">
|
||||||
<label class="md-overlay" for="__drawer"></label>
|
<label class="md-overlay" for="__drawer"></label>
|
||||||
<div data-md-component="skip">
|
<div data-md-component="skip">
|
||||||
|
|
||||||
|
|
||||||
<a href="#utilities" class="md-skip">
|
<a href="#great_ai.utilities.unique" class="md-skip">
|
||||||
Skip to content
|
Skip to content
|
||||||
</a>
|
</a>
|
||||||
|
|
||||||
|
|
@ -102,9 +106,76 @@
|
||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
|
<form class="md-header__option" data-md-component="palette">
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
<input class="md-option" data-md-color-media="" data-md-color-scheme="default" data-md-color-primary="" data-md-color-accent="" aria-label="Switch to dark mode" type="radio" name="__palette" id="__palette_1">
|
||||||
|
|
||||||
|
<label class="md-header__button md-icon" title="Switch to dark mode" for="__palette_2" hidden>
|
||||||
|
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M12 8a4 4 0 0 0-4 4 4 4 0 0 0 4 4 4 4 0 0 0 4-4 4 4 0 0 0-4-4m0 10a6 6 0 0 1-6-6 6 6 0 0 1 6-6 6 6 0 0 1 6 6 6 6 0 0 1-6 6m8-9.31V4h-4.69L12 .69 8.69 4H4v4.69L.69 12 4 15.31V20h4.69L12 23.31 15.31 20H20v-4.69L23.31 12 20 8.69Z"/></svg>
|
||||||
|
</label>
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
<input class="md-option" data-md-color-media="" data-md-color-scheme="slate" data-md-color-primary="" data-md-color-accent="" aria-label="Switch to light mode" type="radio" name="__palette" id="__palette_2">
|
||||||
|
|
||||||
|
<label class="md-header__button md-icon" title="Switch to light mode" for="__palette_1" hidden>
|
||||||
|
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M12 18c-.89 0-1.74-.2-2.5-.55C11.56 16.5 13 14.42 13 12c0-2.42-1.44-4.5-3.5-5.45C10.26 6.2 11.11 6 12 6a6 6 0 0 1 6 6 6 6 0 0 1-6 6m8-9.31V4h-4.69L12 .69 8.69 4H4v4.69L.69 12 4 15.31V20h4.69L12 23.31 15.31 20H20v-4.69L23.31 12 20 8.69Z"/></svg>
|
||||||
|
</label>
|
||||||
|
|
||||||
|
|
||||||
|
</form>
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
<label class="md-header__button md-icon" for="__search">
|
||||||
|
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M9.5 3A6.5 6.5 0 0 1 16 9.5c0 1.61-.59 3.09-1.56 4.23l.27.27h.79l5 5-1.5 1.5-5-5v-.79l-.27-.27A6.516 6.516 0 0 1 9.5 16 6.5 6.5 0 0 1 3 9.5 6.5 6.5 0 0 1 9.5 3m0 2C7 5 5 7 5 9.5S7 14 9.5 14 14 12 14 9.5 12 5 9.5 5Z"/></svg>
|
||||||
|
</label>
|
||||||
|
<div class="md-search" data-md-component="search" role="dialog">
|
||||||
|
<label class="md-search__overlay" for="__search"></label>
|
||||||
|
<div class="md-search__inner" role="search">
|
||||||
|
<form class="md-search__form" name="search">
|
||||||
|
<input type="text" class="md-search__input" name="query" aria-label="Search" placeholder="Search" autocapitalize="off" autocorrect="off" autocomplete="off" spellcheck="false" data-md-component="search-query" required>
|
||||||
|
<label class="md-search__icon md-icon" for="__search">
|
||||||
|
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M9.5 3A6.5 6.5 0 0 1 16 9.5c0 1.61-.59 3.09-1.56 4.23l.27.27h.79l5 5-1.5 1.5-5-5v-.79l-.27-.27A6.516 6.516 0 0 1 9.5 16 6.5 6.5 0 0 1 3 9.5 6.5 6.5 0 0 1 9.5 3m0 2C7 5 5 7 5 9.5S7 14 9.5 14 14 12 14 9.5 12 5 9.5 5Z"/></svg>
|
||||||
|
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M20 11v2H8l5.5 5.5-1.42 1.42L4.16 12l7.92-7.92L13.5 5.5 8 11h12Z"/></svg>
|
||||||
|
</label>
|
||||||
|
<nav class="md-search__options" aria-label="Search">
|
||||||
|
|
||||||
|
<button type="reset" class="md-search__icon md-icon" aria-label="Clear" tabindex="-1">
|
||||||
|
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M19 6.41 17.59 5 12 10.59 6.41 5 5 6.41 10.59 12 5 17.59 6.41 19 12 13.41 17.59 19 19 17.59 13.41 12 19 6.41Z"/></svg>
|
||||||
|
</button>
|
||||||
|
</nav>
|
||||||
|
|
||||||
|
</form>
|
||||||
|
<div class="md-search__output">
|
||||||
|
<div class="md-search__scrollwrap" data-md-scrollfix>
|
||||||
|
<div class="md-search-result" data-md-component="search-result">
|
||||||
|
<div class="md-search-result__meta">
|
||||||
|
Initializing search
|
||||||
|
</div>
|
||||||
|
<ol class="md-search-result__list"></ol>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
|
||||||
|
<div class="md-header__source">
|
||||||
|
<a href="https://github.com/schmelczer/great-ai" title="Go to repository" class="md-source" data-md-component="source">
|
||||||
|
<div class="md-source__icon md-icon">
|
||||||
|
|
||||||
|
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 448 512"><!--! Font Awesome Free 6.1.1 by @fontawesome - https://fontawesome.com License - https://fontawesome.com/license/free (Icons: CC BY 4.0, Fonts: SIL OFL 1.1, Code: MIT License) Copyright 2022 Fonticons, Inc.--><path d="M439.55 236.05 244 40.45a28.87 28.87 0 0 0-40.81 0l-40.66 40.63 51.52 51.52c27.06-9.14 52.68 16.77 43.39 43.68l49.66 49.66c34.23-11.8 61.18 31 35.47 56.69-26.49 26.49-70.21-2.87-56-37.34L240.22 199v121.85c25.3 12.54 22.26 41.85 9.08 55a34.34 34.34 0 0 1-48.55 0c-17.57-17.6-11.07-46.91 11.25-56v-123c-20.8-8.51-24.6-30.74-18.64-45L142.57 101 8.45 235.14a28.86 28.86 0 0 0 0 40.81l195.61 195.6a28.86 28.86 0 0 0 40.8 0l194.69-194.69a28.86 28.86 0 0 0 0-40.81z"/></svg>
|
||||||
|
</div>
|
||||||
|
<div class="md-source__repository">
|
||||||
|
GitHub
|
||||||
|
</div>
|
||||||
|
</a>
|
||||||
|
</div>
|
||||||
|
|
||||||
</nav>
|
</nav>
|
||||||
|
|
||||||
</header>
|
</header>
|
||||||
|
|
@ -138,6 +209,18 @@
|
||||||
GreatAI documentation
|
GreatAI documentation
|
||||||
</label>
|
</label>
|
||||||
|
|
||||||
|
<div class="md-nav__source">
|
||||||
|
<a href="https://github.com/schmelczer/great-ai" title="Go to repository" class="md-source" data-md-component="source">
|
||||||
|
<div class="md-source__icon md-icon">
|
||||||
|
|
||||||
|
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 448 512"><!--! Font Awesome Free 6.1.1 by @fontawesome - https://fontawesome.com License - https://fontawesome.com/license/free (Icons: CC BY 4.0, Fonts: SIL OFL 1.1, Code: MIT License) Copyright 2022 Fonticons, Inc.--><path d="M439.55 236.05 244 40.45a28.87 28.87 0 0 0-40.81 0l-40.66 40.63 51.52 51.52c27.06-9.14 52.68 16.77 43.39 43.68l49.66 49.66c34.23-11.8 61.18 31 35.47 56.69-26.49 26.49-70.21-2.87-56-37.34L240.22 199v121.85c25.3 12.54 22.26 41.85 9.08 55a34.34 34.34 0 0 1-48.55 0c-17.57-17.6-11.07-46.91 11.25-56v-123c-20.8-8.51-24.6-30.74-18.64-45L142.57 101 8.45 235.14a28.86 28.86 0 0 0 0 40.81l195.61 195.6a28.86 28.86 0 0 0 40.8 0l194.69-194.69a28.86 28.86 0 0 0 0-40.81z"/></svg>
|
||||||
|
</div>
|
||||||
|
<div class="md-source__repository">
|
||||||
|
GitHub
|
||||||
|
</div>
|
||||||
|
</a>
|
||||||
|
</div>
|
||||||
|
|
||||||
<ul class="md-nav__list" data-md-scrollfix>
|
<ul class="md-nav__list" data-md-scrollfix>
|
||||||
|
|
||||||
|
|
||||||
|
|
@ -223,8 +306,15 @@
|
||||||
<ul class="md-nav__list" data-md-component="toc" data-md-scrollfix>
|
<ul class="md-nav__list" data-md-component="toc" data-md-scrollfix>
|
||||||
|
|
||||||
<li class="md-nav__item">
|
<li class="md-nav__item">
|
||||||
<a href="#utilities" class="md-nav__link">
|
<a href="#great_ai.utilities.unique" class="md-nav__link">
|
||||||
utilities
|
great_ai.utilities.unique
|
||||||
|
</a>
|
||||||
|
|
||||||
|
</li>
|
||||||
|
|
||||||
|
<li class="md-nav__item">
|
||||||
|
<a href="#great_ai.utilities.unique.unique" class="md-nav__link">
|
||||||
|
unique()
|
||||||
</a>
|
</a>
|
||||||
|
|
||||||
</li>
|
</li>
|
||||||
|
|
@ -252,6 +342,21 @@
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
<li class="md-nav__item">
|
||||||
|
<a href="../simple-mag/train/" class="md-nav__link">
|
||||||
|
Train a field of study (domain) classification of sentences on the MAG dataset
|
||||||
|
</a>
|
||||||
|
</li>
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
</ul>
|
</ul>
|
||||||
</nav>
|
</nav>
|
||||||
</div>
|
</div>
|
||||||
|
|
@ -277,8 +382,15 @@
|
||||||
<ul class="md-nav__list" data-md-component="toc" data-md-scrollfix>
|
<ul class="md-nav__list" data-md-component="toc" data-md-scrollfix>
|
||||||
|
|
||||||
<li class="md-nav__item">
|
<li class="md-nav__item">
|
||||||
<a href="#utilities" class="md-nav__link">
|
<a href="#great_ai.utilities.unique" class="md-nav__link">
|
||||||
utilities
|
great_ai.utilities.unique
|
||||||
|
</a>
|
||||||
|
|
||||||
|
</li>
|
||||||
|
|
||||||
|
<li class="md-nav__item">
|
||||||
|
<a href="#great_ai.utilities.unique.unique" class="md-nav__link">
|
||||||
|
unique()
|
||||||
</a>
|
</a>
|
||||||
|
|
||||||
</li>
|
</li>
|
||||||
|
|
@ -294,7 +406,14 @@
|
||||||
<div class="md-content" data-md-component="content">
|
<div class="md-content" data-md-component="content">
|
||||||
<article class="md-content__inner md-typeset">
|
<article class="md-content__inner md-typeset">
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
<a href="https://github.com/schmelczer/great-ai/edit/main/docs/reference.md" title="Edit this page" class="md-content__button md-icon">
|
||||||
|
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M20.71 7.04c.39-.39.39-1.04 0-1.41l-2.34-2.34c-.37-.39-1.02-.39-1.41 0l-1.84 1.83 3.75 3.75M3 17.25V21h3.75L17.81 9.93l-3.75-3.75L3 17.25Z"/></svg>
|
||||||
|
</a>
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
<h1>Reference</h1>
|
<h1>Reference</h1>
|
||||||
|
|
@ -302,7 +421,7 @@
|
||||||
<div class="doc doc-object doc-module">
|
<div class="doc doc-object doc-module">
|
||||||
|
|
||||||
|
|
||||||
<a id="utilities"></a>
|
<a id="great_ai.utilities.unique"></a>
|
||||||
<div class="doc doc-contents first">
|
<div class="doc doc-contents first">
|
||||||
|
|
||||||
|
|
||||||
|
|
@ -317,6 +436,147 @@
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
<div class="doc doc-object doc-function">
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
<h2 id="great_ai.utilities.unique.unique" class="doc doc-heading">
|
||||||
|
<code class="highlight language-python"><span class="n">unique</span><span class="p">(</span><span class="n">values</span><span class="p">,</span> <span class="o">*</span><span class="p">,</span> <span class="n">key</span><span class="o">=</span><span class="k">lambda</span> <span class="n">v</span><span class="p">:</span> <span class="n">v</span><span class="p">)</span></code>
|
||||||
|
|
||||||
|
</h2>
|
||||||
|
|
||||||
|
|
||||||
|
<div class="doc doc-contents ">
|
||||||
|
|
||||||
|
<p>Keep only the first occurrences while maintaining original order.</p>
|
||||||
|
<p>The equality check used for deduplication can be overridden using the <code>key</code> argument.</p>
|
||||||
|
|
||||||
|
<p><strong>Examples:</strong></p>
|
||||||
|
<div class="highlight"><pre><span></span><code><span class="gp">>>> </span><span class="n">unique</span><span class="p">([</span><span class="mi">1</span><span class="p">,</span> <span class="mi">1</span><span class="p">,</span> <span class="mi">5</span><span class="p">,</span> <span class="mi">3</span><span class="p">,</span> <span class="mi">3</span><span class="p">])</span>
|
||||||
|
<span class="go">[1, 5, 3]</span>
|
||||||
|
<span class="gp">>>> </span><span class="n">unique</span><span class="p">([{</span><span class="s1">'a'</span><span class="p">:</span> <span class="mi">1</span><span class="p">,</span> <span class="s1">'b'</span><span class="p">:</span> <span class="mi">2</span><span class="p">},</span> <span class="p">{</span><span class="s1">'a'</span><span class="p">:</span> <span class="mi">1</span><span class="p">,</span> <span class="s1">'b'</span><span class="p">:</span> <span class="mi">3</span><span class="p">}],</span> <span class="n">key</span><span class="o">=</span><span class="k">lambda</span> <span class="n">v</span><span class="p">:</span> <span class="n">v</span><span class="p">[</span><span class="s1">'a'</span><span class="p">])</span>
|
||||||
|
<span class="go">[{'a': 1, 'b': 2}]</span>
|
||||||
|
<span class="gp">>>> </span><span class="n">unique</span><span class="p">([{</span><span class="s1">'a'</span><span class="p">:</span> <span class="mi">1</span><span class="p">,</span> <span class="s1">'b'</span><span class="p">:</span> <span class="mi">2</span><span class="p">},</span> <span class="p">{</span><span class="s1">'a'</span><span class="p">:</span> <span class="mi">1</span><span class="p">,</span> <span class="s1">'b'</span><span class="p">:</span> <span class="mi">3</span><span class="p">}],</span> <span class="n">key</span><span class="o">=</span><span class="k">lambda</span> <span class="n">v</span><span class="p">:</span> <span class="n">v</span><span class="p">[</span><span class="s1">'b'</span><span class="p">])</span>
|
||||||
|
<span class="go">[{'a': 1, 'b': 2}, {'a': 1, 'b': 3}]</span>
|
||||||
|
</code></pre></div>
|
||||||
|
|
||||||
|
<p><strong>Parameters:</strong></p>
|
||||||
|
<table>
|
||||||
|
<thead>
|
||||||
|
<tr>
|
||||||
|
<th>Name</th>
|
||||||
|
<th>Type</th>
|
||||||
|
<th>Description</th>
|
||||||
|
<th>Default</th>
|
||||||
|
</tr>
|
||||||
|
</thead>
|
||||||
|
<tbody>
|
||||||
|
<tr>
|
||||||
|
<td><code>values</code></td>
|
||||||
|
<td>
|
||||||
|
<code><span title="typing.Iterable">Iterable</span>[<span title="great_ai.utilities.unique.T">T</span>]</code>
|
||||||
|
</td>
|
||||||
|
<td><p>An iterable containing your values</p></td>
|
||||||
|
<td>
|
||||||
|
<em>required</em>
|
||||||
|
</td>
|
||||||
|
</tr>
|
||||||
|
<tr>
|
||||||
|
<td><code>key</code></td>
|
||||||
|
<td>
|
||||||
|
<code><span title="typing.Callable">Callable</span>[[<span title="great_ai.utilities.unique.T">T</span>], <span title="typing.Any">Any</span>]</code>
|
||||||
|
</td>
|
||||||
|
<td><p>Override the identity function of the equality check.</p></td>
|
||||||
|
<td>
|
||||||
|
<code>lambda v: v</code>
|
||||||
|
</td>
|
||||||
|
</tr>
|
||||||
|
</tbody>
|
||||||
|
</table>
|
||||||
|
|
||||||
|
<p><strong>Returns:</strong></p>
|
||||||
|
<table>
|
||||||
|
<thead>
|
||||||
|
<tr>
|
||||||
|
<th>Type</th>
|
||||||
|
<th>Description</th>
|
||||||
|
</tr>
|
||||||
|
</thead>
|
||||||
|
<tbody>
|
||||||
|
<tr>
|
||||||
|
<td>
|
||||||
|
<code><span title="typing.List">List</span>[<span title="great_ai.utilities.unique.T">T</span>]</code>
|
||||||
|
</td>
|
||||||
|
<td><p>A deduplicated list.</p></td>
|
||||||
|
</tr>
|
||||||
|
</tbody>
|
||||||
|
</table>
|
||||||
|
|
||||||
|
<details class="quote">
|
||||||
|
<summary>Source code in <code>great_ai/utilities/unique.py</code></summary>
|
||||||
|
<div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"> 6</span>
|
||||||
|
<span class="normal"> 7</span>
|
||||||
|
<span class="normal"> 8</span>
|
||||||
|
<span class="normal"> 9</span>
|
||||||
|
<span class="normal">10</span>
|
||||||
|
<span class="normal">11</span>
|
||||||
|
<span class="normal">12</span>
|
||||||
|
<span class="normal">13</span>
|
||||||
|
<span class="normal">14</span>
|
||||||
|
<span class="normal">15</span>
|
||||||
|
<span class="normal">16</span>
|
||||||
|
<span class="normal">17</span>
|
||||||
|
<span class="normal">18</span>
|
||||||
|
<span class="normal">19</span>
|
||||||
|
<span class="normal">20</span>
|
||||||
|
<span class="normal">21</span>
|
||||||
|
<span class="normal">22</span>
|
||||||
|
<span class="normal">23</span>
|
||||||
|
<span class="normal">24</span>
|
||||||
|
<span class="normal">25</span>
|
||||||
|
<span class="normal">26</span>
|
||||||
|
<span class="normal">27</span>
|
||||||
|
<span class="normal">28</span>
|
||||||
|
<span class="normal">29</span>
|
||||||
|
<span class="normal">30</span>
|
||||||
|
<span class="normal">31</span>
|
||||||
|
<span class="normal">32</span>
|
||||||
|
<span class="normal">33</span>
|
||||||
|
<span class="normal">34</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">def</span> <span class="nf">unique</span><span class="p">(</span><span class="n">values</span><span class="p">:</span> <span class="n">Iterable</span><span class="p">[</span><span class="n">T</span><span class="p">],</span> <span class="o">*</span><span class="p">,</span> <span class="n">key</span><span class="p">:</span> <span class="n">Callable</span><span class="p">[[</span><span class="n">T</span><span class="p">],</span> <span class="n">Any</span><span class="p">]</span> <span class="o">=</span> <span class="k">lambda</span> <span class="n">v</span><span class="p">:</span> <span class="n">v</span><span class="p">)</span> <span class="o">-></span> <span class="n">List</span><span class="p">[</span><span class="n">T</span><span class="p">]:</span>
|
||||||
|
<span class="sd">"""Keep only the first occurrences while maintaining original order.</span>
|
||||||
|
|
||||||
|
<span class="sd"> The equality check used for deduplication can be overridden using the `key` argument.</span>
|
||||||
|
|
||||||
|
<span class="sd"> Examples:</span>
|
||||||
|
<span class="sd"> >>> unique([1, 1, 5, 3, 3])</span>
|
||||||
|
<span class="sd"> [1, 5, 3]</span>
|
||||||
|
<span class="sd"> >>> unique([{'a': 1, 'b': 2}, {'a': 1, 'b': 3}], key=lambda v: v['a'])</span>
|
||||||
|
<span class="sd"> [{'a': 1, 'b': 2}]</span>
|
||||||
|
<span class="sd"> >>> unique([{'a': 1, 'b': 2}, {'a': 1, 'b': 3}], key=lambda v: v['b'])</span>
|
||||||
|
<span class="sd"> [{'a': 1, 'b': 2}, {'a': 1, 'b': 3}]</span>
|
||||||
|
|
||||||
|
<span class="sd"> Args:</span>
|
||||||
|
<span class="sd"> values: An iterable containing your values</span>
|
||||||
|
<span class="sd"> key: Override the identity function of the equality check.</span>
|
||||||
|
|
||||||
|
<span class="sd"> Returns:</span>
|
||||||
|
<span class="sd"> A deduplicated list.</span>
|
||||||
|
<span class="sd"> """</span>
|
||||||
|
|
||||||
|
<span class="n">key_values</span> <span class="o">=</span> <span class="p">{}</span>
|
||||||
|
<span class="k">for</span> <span class="n">v</span> <span class="ow">in</span> <span class="n">values</span><span class="p">:</span>
|
||||||
|
<span class="n">k</span> <span class="o">=</span> <span class="n">key</span><span class="p">(</span><span class="n">v</span><span class="p">)</span>
|
||||||
|
<span class="k">if</span> <span class="n">k</span> <span class="ow">not</span> <span class="ow">in</span> <span class="n">key_values</span><span class="p">:</span>
|
||||||
|
<span class="c1"># dicts maintain insertion order: https://mail.python.org/pipermail/python-dev/2017-December/151283.html</span>
|
||||||
|
<span class="n">key_values</span><span class="p">[</span><span class="n">k</span><span class="p">]</span> <span class="o">=</span> <span class="n">v</span>
|
||||||
|
|
||||||
|
<span class="k">return</span> <span class="nb">list</span><span class="p">(</span><span class="n">key_values</span><span class="o">.</span><span class="n">values</span><span class="p">())</span>
|
||||||
|
</code></pre></div></td></tr></table></div>
|
||||||
|
</details>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
</div>
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
</div>
|
</div>
|
||||||
|
|
@ -325,7 +585,19 @@
|
||||||
|
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
|
<hr>
|
||||||
|
<div class="md-source-file">
|
||||||
|
<small>
|
||||||
|
|
||||||
|
Last update:
|
||||||
|
<span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-date">July 8, 2022</span>
|
||||||
|
|
||||||
|
|
||||||
|
</small>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
</article>
|
</article>
|
||||||
|
|
||||||
</div>
|
</div>
|
||||||
|
|
@ -394,7 +666,7 @@
|
||||||
<script id="__config" type="application/json">{"base": "..", "features": [], "search": "../assets/javascripts/workers/search.b97dbffb.min.js", "translations": {"clipboard.copied": "Copied to clipboard", "clipboard.copy": "Copy to clipboard", "search.config.lang": "en", "search.config.pipeline": "trimmer, stopWordFilter", "search.config.separator": "[\\s\\-]+", "search.placeholder": "Search", "search.result.more.one": "1 more on this page", "search.result.more.other": "# more on this page", "search.result.none": "No matching documents", "search.result.one": "1 matching document", "search.result.other": "# matching documents", "search.result.placeholder": "Type to start searching", "search.result.term.missing": "Missing", "select.version.title": "Select version"}}</script>
|
<script id="__config" type="application/json">{"base": "..", "features": [], "search": "../assets/javascripts/workers/search.b97dbffb.min.js", "translations": {"clipboard.copied": "Copied to clipboard", "clipboard.copy": "Copy to clipboard", "search.config.lang": "en", "search.config.pipeline": "trimmer, stopWordFilter", "search.config.separator": "[\\s\\-]+", "search.placeholder": "Search", "search.result.more.one": "1 more on this page", "search.result.more.other": "# more on this page", "search.result.none": "No matching documents", "search.result.one": "1 matching document", "search.result.other": "# matching documents", "search.result.placeholder": "Type to start searching", "search.result.term.missing": "Missing", "select.version.title": "Select version"}}</script>
|
||||||
|
|
||||||
|
|
||||||
<script src="../assets/javascripts/bundle.0238f547.min.js"></script>
|
<script src="../assets/javascripts/bundle.6c7ad80a.min.js"></script>
|
||||||
|
|
||||||
|
|
||||||
</body>
|
</body>
|
||||||
|
|
|
||||||
1
reference/reference.md
Normal file
1
reference/reference.md
Normal file
|
|
@ -0,0 +1 @@
|
||||||
|
::: great_ai.utilities.unique
|
||||||
|
|
@ -1,90 +0,0 @@
|
||||||
{
|
|
||||||
"cells": [
|
|
||||||
{
|
|
||||||
"cell_type": "code",
|
|
||||||
"execution_count": 1,
|
|
||||||
"metadata": {},
|
|
||||||
"outputs": [
|
|
||||||
{
|
|
||||||
"data": {
|
|
||||||
"text/plain": [
|
|
||||||
"Trace(trace_id='07719c28-e248-43f8-a652-608c49ad5a17', created='2022-06-26T07:55:30.587753', original_execution_time_ms=178.268, logged_values={'arg:text:value': 'I love chemical compounds and nuclear fission.', 'arg:text:length': 46, 'arg:target_confidence:value': 50}, models=[Model(key='small-domain-prediction', version=0)], exception=None, output={'labels': [{'label': 'Physics', 'confidence': 28.0, 'explanation': ['nuclear', 'chemical', 'fission', 'compounds', 'love']}, {'label': 'Chemistry', 'confidence': 22.0, 'explanation': ['chemical', 'compounds', 'nuclear', 'fission', 'love']}]}, feedback=None, tags=['predict_domain', 'online', 'development'])"
|
|
||||||
]
|
|
||||||
},
|
|
||||||
"execution_count": 1,
|
|
||||||
"metadata": {},
|
|
||||||
"output_type": "execute_result"
|
|
||||||
}
|
|
||||||
],
|
|
||||||
"source": [
|
|
||||||
"from great_ai import call_remote_great_ai_async\n",
|
|
||||||
"\n",
|
|
||||||
"\n",
|
|
||||||
"await call_remote_great_ai_async(\n",
|
|
||||||
" \"http://localhost:6060\", {\"text\": \"I love chemical compounds and nuclear fission.\"}\n",
|
|
||||||
")"
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"cell_type": "code",
|
|
||||||
"execution_count": 1,
|
|
||||||
"metadata": {},
|
|
||||||
"outputs": [
|
|
||||||
{
|
|
||||||
"name": "stdout",
|
|
||||||
"output_type": "stream",
|
|
||||||
"text": [
|
|
||||||
"<_UnixSelectorEventLoop running=True closed=False debug=False>\n"
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"ename": "Exception",
|
|
||||||
"evalue": "Already running in an event loop, you have to call call_remote_great_ai_async.",
|
|
||||||
"output_type": "error",
|
|
||||||
"traceback": [
|
|
||||||
"\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
|
|
||||||
"\u001b[0;31mException\u001b[0m Traceback (most recent call last)",
|
|
||||||
"\u001b[1;32m/data/projects/great_ai/docs/remote-test/remote.ipynb Cell 1'\u001b[0m in \u001b[0;36m<cell line: 4>\u001b[0;34m()\u001b[0m\n\u001b[1;32m <a href='vscode-notebook-cell:/data/projects/great_ai/docs/remote-test/remote.ipynb#ch0000000?line=0'>1</a>\u001b[0m \u001b[39mfrom\u001b[39;00m \u001b[39mgreat_ai\u001b[39;00m \u001b[39mimport\u001b[39;00m call_remote_great_ai\n\u001b[0;32m----> <a href='vscode-notebook-cell:/data/projects/great_ai/docs/remote-test/remote.ipynb#ch0000000?line=3'>4</a>\u001b[0m call_remote_great_ai(\u001b[39m'\u001b[39;49m\u001b[39mhttp://localhost:6060\u001b[39;49m\u001b[39m'\u001b[39;49m, {\n\u001b[1;32m <a href='vscode-notebook-cell:/data/projects/great_ai/docs/remote-test/remote.ipynb#ch0000000?line=4'>5</a>\u001b[0m \u001b[39m'\u001b[39;49m\u001b[39mtext\u001b[39;49m\u001b[39m'\u001b[39;49m: \u001b[39m'\u001b[39;49m\u001b[39mI love chemical compounds and nuclear fission.\u001b[39;49m\u001b[39m'\u001b[39;49m\n\u001b[1;32m <a href='vscode-notebook-cell:/data/projects/great_ai/docs/remote-test/remote.ipynb#ch0000000?line=5'>6</a>\u001b[0m })\n",
|
|
||||||
"File \u001b[0;32m/data/projects/great_ai/src/great_ai/great_ai/remote/call_remote_great_ai.py:17\u001b[0m, in \u001b[0;36mcall_remote_great_ai\u001b[0;34m(base_uri, data, retry_count)\u001b[0m\n\u001b[1;32m 15\u001b[0m \u001b[39mtry\u001b[39;00m:\n\u001b[1;32m 16\u001b[0m \u001b[39mprint\u001b[39m(asyncio\u001b[39m.\u001b[39mget_running_loop())\n\u001b[0;32m---> 17\u001b[0m \u001b[39mraise\u001b[39;00m \u001b[39mException\u001b[39;00m(\u001b[39mf\u001b[39m\u001b[39m'\u001b[39m\u001b[39mAlready running in an event loop, you have to call \u001b[39m\u001b[39m{\u001b[39;00mcall_remote_great_ai_async\u001b[39m.\u001b[39m\u001b[39m__name__\u001b[39m\u001b[39m}\u001b[39;00m\u001b[39m.\u001b[39m\u001b[39m'\u001b[39m)\n\u001b[1;32m 18\u001b[0m \u001b[39mexcept\u001b[39;00m \u001b[39mRuntimeError\u001b[39;00m:\n\u001b[1;32m 19\u001b[0m \u001b[39mpass\u001b[39;00m\n",
|
|
||||||
"\u001b[0;31mException\u001b[0m: Already running in an event loop, you have to call call_remote_great_ai_async."
|
|
||||||
]
|
|
||||||
}
|
|
||||||
],
|
|
||||||
"source": [
|
|
||||||
"from great_ai import call_remote_great_ai\n",
|
|
||||||
"\n",
|
|
||||||
"\n",
|
|
||||||
"call_remote_great_ai(\n",
|
|
||||||
" \"http://localhost:6060\", {\"text\": \"I love chemical compounds and nuclear fission.\"}\n",
|
|
||||||
")"
|
|
||||||
]
|
|
||||||
}
|
|
||||||
],
|
|
||||||
"metadata": {
|
|
||||||
"kernelspec": {
|
|
||||||
"display_name": "Python 3.10.4 ('.env': venv)",
|
|
||||||
"language": "python",
|
|
||||||
"name": "python3"
|
|
||||||
},
|
|
||||||
"language_info": {
|
|
||||||
"codemirror_mode": {
|
|
||||||
"name": "ipython",
|
|
||||||
"version": 3
|
|
||||||
},
|
|
||||||
"file_extension": ".py",
|
|
||||||
"mimetype": "text/x-python",
|
|
||||||
"name": "python",
|
|
||||||
"nbconvert_exporter": "python",
|
|
||||||
"pygments_lexer": "ipython3",
|
|
||||||
"version": "3.10.4"
|
|
||||||
},
|
|
||||||
"orig_nbformat": 4,
|
|
||||||
"vscode": {
|
|
||||||
"interpreter": {
|
|
||||||
"hash": "d88b0dd276e3f918f7798b7e97af2e3c2f843817b9e5b55a9df0a682ffd80f44"
|
|
||||||
}
|
|
||||||
}
|
|
||||||
},
|
|
||||||
"nbformat": 4,
|
|
||||||
"nbformat_minor": 2
|
|
||||||
}
|
|
||||||
1
search/search_index.json
Normal file
1
search/search_index.json
Normal file
File diff suppressed because one or more lines are too long
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
2477
simple-mag/train/index.html
Normal file
2477
simple-mag/train/index.html
Normal file
File diff suppressed because one or more lines are too long
|
|
@ -19,49 +19,16 @@
|
||||||
"name": "stderr",
|
"name": "stderr",
|
||||||
"output_type": "stream",
|
"output_type": "stream",
|
||||||
"text": [
|
"text": [
|
||||||
"\u001b[38;5;39m2022-06-20 14:33:17,570 | INFO | Starting parallel map (concurrency: 12, chunk size: 700)\u001b[0m\n"
|
"84000it [00:09, 8647.60it/s] \n",
|
||||||
|
"22399it [00:03, 6368.63it/s]\n"
|
||||||
]
|
]
|
||||||
},
|
|
||||||
{
|
|
||||||
"data": {
|
|
||||||
"application/vnd.jupyter.widget-view+json": {
|
|
||||||
"model_id": "e6f76bf83615422cb4352dcb3af48a26",
|
|
||||||
"version_major": 2,
|
|
||||||
"version_minor": 0
|
|
||||||
},
|
|
||||||
"text/plain": [
|
|
||||||
" 0%| | 0/84000 [00:00<?, ?it/s]"
|
|
||||||
]
|
|
||||||
},
|
|
||||||
"metadata": {},
|
|
||||||
"output_type": "display_data"
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"name": "stderr",
|
|
||||||
"output_type": "stream",
|
|
||||||
"text": [
|
|
||||||
"\u001b[38;5;39m2022-06-20 14:33:24,846 | INFO | Starting parallel map (concurrency: 12, chunk size: 187)\u001b[0m\n"
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"data": {
|
|
||||||
"application/vnd.jupyter.widget-view+json": {
|
|
||||||
"model_id": "ab012f9276d74bdda15b00cfec6c8f56",
|
|
||||||
"version_major": 2,
|
|
||||||
"version_minor": 0
|
|
||||||
},
|
|
||||||
"text/plain": [
|
|
||||||
" 0%| | 0/22399 [00:00<?, ?it/s]"
|
|
||||||
]
|
|
||||||
},
|
|
||||||
"metadata": {},
|
|
||||||
"output_type": "display_data"
|
|
||||||
}
|
}
|
||||||
],
|
],
|
||||||
"source": [
|
"source": [
|
||||||
"import json\n",
|
"import json\n",
|
||||||
"from typing import Tuple\n",
|
"from typing import Tuple\n",
|
||||||
"from great_ai.utilities import clean, parallel_map\n",
|
"from great_ai.utilities import clean, parallel_map\n",
|
||||||
|
"from tqdm.cli import tqdm\n",
|
||||||
"\n",
|
"\n",
|
||||||
"\n",
|
"\n",
|
||||||
"def preprocess(line: str) -> Tuple[str, str]:\n",
|
"def preprocess(line: str) -> Tuple[str, str]:\n",
|
||||||
|
|
@ -71,14 +38,14 @@
|
||||||
"\n",
|
"\n",
|
||||||
"\n",
|
"\n",
|
||||||
"with open(\"mag/train.txt\", encoding=\"utf-8\") as f:\n",
|
"with open(\"mag/train.txt\", encoding=\"utf-8\") as f:\n",
|
||||||
" training_data = parallel_map(preprocess, f.readlines())\n",
|
" training_data = list(tqdm(parallel_map(preprocess, f.readlines())))\n",
|
||||||
"\n",
|
"\n",
|
||||||
"X_train = [d[0] for d in training_data]\n",
|
"X_train = [d[0] for d in training_data]\n",
|
||||||
"y_train = [d[1] for d in training_data]\n",
|
"y_train = [d[1] for d in training_data]\n",
|
||||||
"\n",
|
"\n",
|
||||||
"\n",
|
"\n",
|
||||||
"with open(\"mag/test.txt\", encoding=\"utf-8\") as f:\n",
|
"with open(\"mag/test.txt\", encoding=\"utf-8\") as f:\n",
|
||||||
" test_data = parallel_map(preprocess, f.readlines())\n",
|
" test_data = list(tqdm(parallel_map(preprocess, f.readlines())))\n",
|
||||||
"\n",
|
"\n",
|
||||||
"X_test = [d[0] for d in test_data]\n",
|
"X_test = [d[0] for d in test_data]\n",
|
||||||
"y_test = [d[1] for d in test_data]"
|
"y_test = [d[1] for d in test_data]"
|
||||||
|
|
@ -1070,10 +1037,10 @@
|
||||||
" <tbody>\n",
|
" <tbody>\n",
|
||||||
" <tr>\n",
|
" <tr>\n",
|
||||||
" <th>12</th>\n",
|
" <th>12</th>\n",
|
||||||
" <td>1.172014</td>\n",
|
" <td>1.227354</td>\n",
|
||||||
" <td>0.033318</td>\n",
|
" <td>0.011710</td>\n",
|
||||||
" <td>0.625773</td>\n",
|
" <td>0.652844</td>\n",
|
||||||
" <td>0.013720</td>\n",
|
" <td>0.040511</td>\n",
|
||||||
" <td>1</td>\n",
|
" <td>1</td>\n",
|
||||||
" <td>True</td>\n",
|
" <td>True</td>\n",
|
||||||
" <td>0.05</td>\n",
|
" <td>0.05</td>\n",
|
||||||
|
|
@ -1088,10 +1055,10 @@
|
||||||
" </tr>\n",
|
" </tr>\n",
|
||||||
" <tr>\n",
|
" <tr>\n",
|
||||||
" <th>15</th>\n",
|
" <th>15</th>\n",
|
||||||
" <td>1.201231</td>\n",
|
" <td>1.279783</td>\n",
|
||||||
" <td>0.016728</td>\n",
|
" <td>0.037969</td>\n",
|
||||||
" <td>0.613568</td>\n",
|
" <td>0.634761</td>\n",
|
||||||
" <td>0.013720</td>\n",
|
" <td>0.068606</td>\n",
|
||||||
" <td>1</td>\n",
|
" <td>1</td>\n",
|
||||||
" <td>True</td>\n",
|
" <td>True</td>\n",
|
||||||
" <td>0.1</td>\n",
|
" <td>0.1</td>\n",
|
||||||
|
|
@ -1106,10 +1073,10 @@
|
||||||
" </tr>\n",
|
" </tr>\n",
|
||||||
" <tr>\n",
|
" <tr>\n",
|
||||||
" <th>18</th>\n",
|
" <th>18</th>\n",
|
||||||
" <td>1.211270</td>\n",
|
" <td>1.332854</td>\n",
|
||||||
" <td>0.036636</td>\n",
|
" <td>0.171405</td>\n",
|
||||||
" <td>0.670754</td>\n",
|
" <td>0.743611</td>\n",
|
||||||
" <td>0.076857</td>\n",
|
" <td>0.109792</td>\n",
|
||||||
" <td>1</td>\n",
|
" <td>1</td>\n",
|
||||||
" <td>False</td>\n",
|
" <td>False</td>\n",
|
||||||
" <td>0.05</td>\n",
|
" <td>0.05</td>\n",
|
||||||
|
|
@ -1124,10 +1091,10 @@
|
||||||
" </tr>\n",
|
" </tr>\n",
|
||||||
" <tr>\n",
|
" <tr>\n",
|
||||||
" <th>21</th>\n",
|
" <th>21</th>\n",
|
||||||
" <td>1.165763</td>\n",
|
" <td>1.253222</td>\n",
|
||||||
" <td>0.061206</td>\n",
|
" <td>0.072134</td>\n",
|
||||||
" <td>0.549646</td>\n",
|
" <td>0.612344</td>\n",
|
||||||
" <td>0.046776</td>\n",
|
" <td>0.016771</td>\n",
|
||||||
" <td>1</td>\n",
|
" <td>1</td>\n",
|
||||||
" <td>False</td>\n",
|
" <td>False</td>\n",
|
||||||
" <td>0.1</td>\n",
|
" <td>0.1</td>\n",
|
||||||
|
|
@ -1142,10 +1109,10 @@
|
||||||
" </tr>\n",
|
" </tr>\n",
|
||||||
" <tr>\n",
|
" <tr>\n",
|
||||||
" <th>3</th>\n",
|
" <th>3</th>\n",
|
||||||
" <td>1.277603</td>\n",
|
" <td>1.472035</td>\n",
|
||||||
" <td>0.127785</td>\n",
|
" <td>0.080849</td>\n",
|
||||||
" <td>0.650135</td>\n",
|
" <td>0.654935</td>\n",
|
||||||
" <td>0.023038</td>\n",
|
" <td>0.044302</td>\n",
|
||||||
" <td>0.5</td>\n",
|
" <td>0.5</td>\n",
|
||||||
" <td>True</td>\n",
|
" <td>True</td>\n",
|
||||||
" <td>0.1</td>\n",
|
" <td>0.1</td>\n",
|
||||||
|
|
@ -1160,10 +1127,10 @@
|
||||||
" </tr>\n",
|
" </tr>\n",
|
||||||
" <tr>\n",
|
" <tr>\n",
|
||||||
" <th>0</th>\n",
|
" <th>0</th>\n",
|
||||||
" <td>1.118055</td>\n",
|
" <td>1.380641</td>\n",
|
||||||
" <td>0.058978</td>\n",
|
" <td>0.054306</td>\n",
|
||||||
" <td>0.622284</td>\n",
|
" <td>0.739966</td>\n",
|
||||||
" <td>0.014230</td>\n",
|
" <td>0.053385</td>\n",
|
||||||
" <td>0.5</td>\n",
|
" <td>0.5</td>\n",
|
||||||
" <td>True</td>\n",
|
" <td>True</td>\n",
|
||||||
" <td>0.05</td>\n",
|
" <td>0.05</td>\n",
|
||||||
|
|
@ -1178,10 +1145,10 @@
|
||||||
" </tr>\n",
|
" </tr>\n",
|
||||||
" <tr>\n",
|
" <tr>\n",
|
||||||
" <th>9</th>\n",
|
" <th>9</th>\n",
|
||||||
" <td>1.166192</td>\n",
|
" <td>1.284987</td>\n",
|
||||||
" <td>0.044232</td>\n",
|
" <td>0.113903</td>\n",
|
||||||
" <td>0.619351</td>\n",
|
" <td>0.696876</td>\n",
|
||||||
" <td>0.024824</td>\n",
|
" <td>0.003757</td>\n",
|
||||||
" <td>0.5</td>\n",
|
" <td>0.5</td>\n",
|
||||||
" <td>False</td>\n",
|
" <td>False</td>\n",
|
||||||
" <td>0.1</td>\n",
|
" <td>0.1</td>\n",
|
||||||
|
|
@ -1196,10 +1163,10 @@
|
||||||
" </tr>\n",
|
" </tr>\n",
|
||||||
" <tr>\n",
|
" <tr>\n",
|
||||||
" <th>6</th>\n",
|
" <th>6</th>\n",
|
||||||
" <td>1.233377</td>\n",
|
" <td>1.291148</td>\n",
|
||||||
" <td>0.063715</td>\n",
|
" <td>0.101837</td>\n",
|
||||||
" <td>0.666418</td>\n",
|
" <td>0.686561</td>\n",
|
||||||
" <td>0.061664</td>\n",
|
" <td>0.083989</td>\n",
|
||||||
" <td>0.5</td>\n",
|
" <td>0.5</td>\n",
|
||||||
" <td>False</td>\n",
|
" <td>False</td>\n",
|
||||||
" <td>0.05</td>\n",
|
" <td>0.05</td>\n",
|
||||||
|
|
@ -1214,10 +1181,10 @@
|
||||||
" </tr>\n",
|
" </tr>\n",
|
||||||
" <tr>\n",
|
" <tr>\n",
|
||||||
" <th>13</th>\n",
|
" <th>13</th>\n",
|
||||||
" <td>1.245147</td>\n",
|
" <td>1.268873</td>\n",
|
||||||
" <td>0.089528</td>\n",
|
" <td>0.042412</td>\n",
|
||||||
" <td>0.645836</td>\n",
|
" <td>0.645649</td>\n",
|
||||||
" <td>0.006279</td>\n",
|
" <td>0.022738</td>\n",
|
||||||
" <td>1</td>\n",
|
" <td>1</td>\n",
|
||||||
" <td>True</td>\n",
|
" <td>True</td>\n",
|
||||||
" <td>0.05</td>\n",
|
" <td>0.05</td>\n",
|
||||||
|
|
@ -1232,10 +1199,10 @@
|
||||||
" </tr>\n",
|
" </tr>\n",
|
||||||
" <tr>\n",
|
" <tr>\n",
|
||||||
" <th>16</th>\n",
|
" <th>16</th>\n",
|
||||||
" <td>1.145362</td>\n",
|
" <td>1.147120</td>\n",
|
||||||
" <td>0.071149</td>\n",
|
" <td>0.009335</td>\n",
|
||||||
" <td>0.637540</td>\n",
|
" <td>0.599006</td>\n",
|
||||||
" <td>0.028854</td>\n",
|
" <td>0.045637</td>\n",
|
||||||
" <td>1</td>\n",
|
" <td>1</td>\n",
|
||||||
" <td>True</td>\n",
|
" <td>True</td>\n",
|
||||||
" <td>0.1</td>\n",
|
" <td>0.1</td>\n",
|
||||||
|
|
@ -1250,10 +1217,10 @@
|
||||||
" </tr>\n",
|
" </tr>\n",
|
||||||
" <tr>\n",
|
" <tr>\n",
|
||||||
" <th>4</th>\n",
|
" <th>4</th>\n",
|
||||||
" <td>1.179663</td>\n",
|
" <td>1.186988</td>\n",
|
||||||
" <td>0.046808</td>\n",
|
" <td>0.057330</td>\n",
|
||||||
" <td>0.598699</td>\n",
|
" <td>0.642233</td>\n",
|
||||||
" <td>0.035973</td>\n",
|
" <td>0.078248</td>\n",
|
||||||
" <td>0.5</td>\n",
|
" <td>0.5</td>\n",
|
||||||
" <td>True</td>\n",
|
" <td>True</td>\n",
|
||||||
" <td>0.1</td>\n",
|
" <td>0.1</td>\n",
|
||||||
|
|
@ -1268,10 +1235,10 @@
|
||||||
" </tr>\n",
|
" </tr>\n",
|
||||||
" <tr>\n",
|
" <tr>\n",
|
||||||
" <th>1</th>\n",
|
" <th>1</th>\n",
|
||||||
" <td>1.097773</td>\n",
|
" <td>1.413231</td>\n",
|
||||||
" <td>0.005871</td>\n",
|
" <td>0.153288</td>\n",
|
||||||
" <td>0.596514</td>\n",
|
" <td>0.765960</td>\n",
|
||||||
" <td>0.025899</td>\n",
|
" <td>0.099535</td>\n",
|
||||||
" <td>0.5</td>\n",
|
" <td>0.5</td>\n",
|
||||||
" <td>True</td>\n",
|
" <td>True</td>\n",
|
||||||
" <td>0.05</td>\n",
|
" <td>0.05</td>\n",
|
||||||
|
|
@ -1286,10 +1253,10 @@
|
||||||
" </tr>\n",
|
" </tr>\n",
|
||||||
" <tr>\n",
|
" <tr>\n",
|
||||||
" <th>19</th>\n",
|
" <th>19</th>\n",
|
||||||
" <td>1.161582</td>\n",
|
" <td>1.193013</td>\n",
|
||||||
" <td>0.040011</td>\n",
|
" <td>0.079000</td>\n",
|
||||||
" <td>0.586831</td>\n",
|
" <td>0.691768</td>\n",
|
||||||
" <td>0.006144</td>\n",
|
" <td>0.027448</td>\n",
|
||||||
" <td>1</td>\n",
|
" <td>1</td>\n",
|
||||||
" <td>False</td>\n",
|
" <td>False</td>\n",
|
||||||
" <td>0.05</td>\n",
|
" <td>0.05</td>\n",
|
||||||
|
|
@ -1304,10 +1271,10 @@
|
||||||
" </tr>\n",
|
" </tr>\n",
|
||||||
" <tr>\n",
|
" <tr>\n",
|
||||||
" <th>22</th>\n",
|
" <th>22</th>\n",
|
||||||
" <td>1.106277</td>\n",
|
" <td>1.043618</td>\n",
|
||||||
" <td>0.060809</td>\n",
|
" <td>0.098733</td>\n",
|
||||||
" <td>0.371986</td>\n",
|
" <td>0.450375</td>\n",
|
||||||
" <td>0.014114</td>\n",
|
" <td>0.061660</td>\n",
|
||||||
" <td>1</td>\n",
|
" <td>1</td>\n",
|
||||||
" <td>False</td>\n",
|
" <td>False</td>\n",
|
||||||
" <td>0.1</td>\n",
|
" <td>0.1</td>\n",
|
||||||
|
|
@ -1322,10 +1289,10 @@
|
||||||
" </tr>\n",
|
" </tr>\n",
|
||||||
" <tr>\n",
|
" <tr>\n",
|
||||||
" <th>7</th>\n",
|
" <th>7</th>\n",
|
||||||
" <td>1.217713</td>\n",
|
" <td>1.301459</td>\n",
|
||||||
" <td>0.044787</td>\n",
|
" <td>0.062143</td>\n",
|
||||||
" <td>0.587896</td>\n",
|
" <td>0.660748</td>\n",
|
||||||
" <td>0.024613</td>\n",
|
" <td>0.056054</td>\n",
|
||||||
" <td>0.5</td>\n",
|
" <td>0.5</td>\n",
|
||||||
" <td>False</td>\n",
|
" <td>False</td>\n",
|
||||||
" <td>0.05</td>\n",
|
" <td>0.05</td>\n",
|
||||||
|
|
@ -1340,10 +1307,10 @@
|
||||||
" </tr>\n",
|
" </tr>\n",
|
||||||
" <tr>\n",
|
" <tr>\n",
|
||||||
" <th>10</th>\n",
|
" <th>10</th>\n",
|
||||||
" <td>1.176654</td>\n",
|
" <td>1.433934</td>\n",
|
||||||
" <td>0.045971</td>\n",
|
" <td>0.155240</td>\n",
|
||||||
" <td>0.628005</td>\n",
|
" <td>0.636608</td>\n",
|
||||||
" <td>0.082375</td>\n",
|
" <td>0.024064</td>\n",
|
||||||
" <td>0.5</td>\n",
|
" <td>0.5</td>\n",
|
||||||
" <td>False</td>\n",
|
" <td>False</td>\n",
|
||||||
" <td>0.1</td>\n",
|
" <td>0.1</td>\n",
|
||||||
|
|
@ -1358,10 +1325,10 @@
|
||||||
" </tr>\n",
|
" </tr>\n",
|
||||||
" <tr>\n",
|
" <tr>\n",
|
||||||
" <th>14</th>\n",
|
" <th>14</th>\n",
|
||||||
" <td>1.207308</td>\n",
|
" <td>1.325535</td>\n",
|
||||||
" <td>0.080923</td>\n",
|
" <td>0.073539</td>\n",
|
||||||
" <td>0.610284</td>\n",
|
" <td>0.672542</td>\n",
|
||||||
" <td>0.039157</td>\n",
|
" <td>0.085835</td>\n",
|
||||||
" <td>1</td>\n",
|
" <td>1</td>\n",
|
||||||
" <td>True</td>\n",
|
" <td>True</td>\n",
|
||||||
" <td>0.05</td>\n",
|
" <td>0.05</td>\n",
|
||||||
|
|
@ -1376,10 +1343,10 @@
|
||||||
" </tr>\n",
|
" </tr>\n",
|
||||||
" <tr>\n",
|
" <tr>\n",
|
||||||
" <th>17</th>\n",
|
" <th>17</th>\n",
|
||||||
" <td>1.201498</td>\n",
|
" <td>1.237677</td>\n",
|
||||||
" <td>0.035161</td>\n",
|
" <td>0.070063</td>\n",
|
||||||
" <td>0.574234</td>\n",
|
" <td>0.651091</td>\n",
|
||||||
" <td>0.019247</td>\n",
|
" <td>0.102538</td>\n",
|
||||||
" <td>1</td>\n",
|
" <td>1</td>\n",
|
||||||
" <td>True</td>\n",
|
" <td>True</td>\n",
|
||||||
" <td>0.1</td>\n",
|
" <td>0.1</td>\n",
|
||||||
|
|
@ -1394,10 +1361,10 @@
|
||||||
" </tr>\n",
|
" </tr>\n",
|
||||||
" <tr>\n",
|
" <tr>\n",
|
||||||
" <th>2</th>\n",
|
" <th>2</th>\n",
|
||||||
" <td>1.157947</td>\n",
|
" <td>1.394873</td>\n",
|
||||||
" <td>0.096841</td>\n",
|
" <td>0.105286</td>\n",
|
||||||
" <td>0.586462</td>\n",
|
" <td>0.637073</td>\n",
|
||||||
" <td>0.010952</td>\n",
|
" <td>0.041708</td>\n",
|
||||||
" <td>0.5</td>\n",
|
" <td>0.5</td>\n",
|
||||||
" <td>True</td>\n",
|
" <td>True</td>\n",
|
||||||
" <td>0.05</td>\n",
|
" <td>0.05</td>\n",
|
||||||
|
|
@ -1412,10 +1379,10 @@
|
||||||
" </tr>\n",
|
" </tr>\n",
|
||||||
" <tr>\n",
|
" <tr>\n",
|
||||||
" <th>5</th>\n",
|
" <th>5</th>\n",
|
||||||
" <td>1.150511</td>\n",
|
" <td>1.270732</td>\n",
|
||||||
" <td>0.027807</td>\n",
|
" <td>0.013414</td>\n",
|
||||||
" <td>0.562870</td>\n",
|
" <td>0.620984</td>\n",
|
||||||
" <td>0.013724</td>\n",
|
" <td>0.025393</td>\n",
|
||||||
" <td>0.5</td>\n",
|
" <td>0.5</td>\n",
|
||||||
" <td>True</td>\n",
|
" <td>True</td>\n",
|
||||||
" <td>0.1</td>\n",
|
" <td>0.1</td>\n",
|
||||||
|
|
@ -1430,10 +1397,10 @@
|
||||||
" </tr>\n",
|
" </tr>\n",
|
||||||
" <tr>\n",
|
" <tr>\n",
|
||||||
" <th>20</th>\n",
|
" <th>20</th>\n",
|
||||||
" <td>1.237030</td>\n",
|
" <td>1.177219</td>\n",
|
||||||
" <td>0.006805</td>\n",
|
" <td>0.039633</td>\n",
|
||||||
" <td>0.529591</td>\n",
|
" <td>0.586308</td>\n",
|
||||||
" <td>0.064449</td>\n",
|
" <td>0.044247</td>\n",
|
||||||
" <td>1</td>\n",
|
" <td>1</td>\n",
|
||||||
" <td>False</td>\n",
|
" <td>False</td>\n",
|
||||||
" <td>0.05</td>\n",
|
" <td>0.05</td>\n",
|
||||||
|
|
@ -1448,10 +1415,10 @@
|
||||||
" </tr>\n",
|
" </tr>\n",
|
||||||
" <tr>\n",
|
" <tr>\n",
|
||||||
" <th>8</th>\n",
|
" <th>8</th>\n",
|
||||||
" <td>1.103937</td>\n",
|
" <td>1.165583</td>\n",
|
||||||
" <td>0.046513</td>\n",
|
" <td>0.046999</td>\n",
|
||||||
" <td>0.569367</td>\n",
|
" <td>0.603675</td>\n",
|
||||||
" <td>0.011001</td>\n",
|
" <td>0.031774</td>\n",
|
||||||
" <td>0.5</td>\n",
|
" <td>0.5</td>\n",
|
||||||
" <td>False</td>\n",
|
" <td>False</td>\n",
|
||||||
" <td>0.05</td>\n",
|
" <td>0.05</td>\n",
|
||||||
|
|
@ -1466,10 +1433,10 @@
|
||||||
" </tr>\n",
|
" </tr>\n",
|
||||||
" <tr>\n",
|
" <tr>\n",
|
||||||
" <th>23</th>\n",
|
" <th>23</th>\n",
|
||||||
" <td>1.080252</td>\n",
|
" <td>0.907388</td>\n",
|
||||||
" <td>0.040597</td>\n",
|
" <td>0.121543</td>\n",
|
||||||
" <td>0.326627</td>\n",
|
" <td>0.354655</td>\n",
|
||||||
" <td>0.002905</td>\n",
|
" <td>0.023775</td>\n",
|
||||||
" <td>1</td>\n",
|
" <td>1</td>\n",
|
||||||
" <td>False</td>\n",
|
" <td>False</td>\n",
|
||||||
" <td>0.1</td>\n",
|
" <td>0.1</td>\n",
|
||||||
|
|
@ -1484,10 +1451,10 @@
|
||||||
" </tr>\n",
|
" </tr>\n",
|
||||||
" <tr>\n",
|
" <tr>\n",
|
||||||
" <th>11</th>\n",
|
" <th>11</th>\n",
|
||||||
" <td>1.275601</td>\n",
|
" <td>1.210845</td>\n",
|
||||||
" <td>0.087971</td>\n",
|
" <td>0.041339</td>\n",
|
||||||
" <td>0.580025</td>\n",
|
" <td>0.653606</td>\n",
|
||||||
" <td>0.034914</td>\n",
|
" <td>0.028117</td>\n",
|
||||||
" <td>0.5</td>\n",
|
" <td>0.5</td>\n",
|
||||||
" <td>False</td>\n",
|
" <td>False</td>\n",
|
||||||
" <td>0.1</td>\n",
|
" <td>0.1</td>\n",
|
||||||
|
|
@ -1506,30 +1473,30 @@
|
||||||
],
|
],
|
||||||
"text/plain": [
|
"text/plain": [
|
||||||
" mean_fit_time std_fit_time mean_score_time std_score_time \\\n",
|
" mean_fit_time std_fit_time mean_score_time std_score_time \\\n",
|
||||||
"12 1.172014 0.033318 0.625773 0.013720 \n",
|
"12 1.227354 0.011710 0.652844 0.040511 \n",
|
||||||
"15 1.201231 0.016728 0.613568 0.013720 \n",
|
"15 1.279783 0.037969 0.634761 0.068606 \n",
|
||||||
"18 1.211270 0.036636 0.670754 0.076857 \n",
|
"18 1.332854 0.171405 0.743611 0.109792 \n",
|
||||||
"21 1.165763 0.061206 0.549646 0.046776 \n",
|
"21 1.253222 0.072134 0.612344 0.016771 \n",
|
||||||
"3 1.277603 0.127785 0.650135 0.023038 \n",
|
"3 1.472035 0.080849 0.654935 0.044302 \n",
|
||||||
"0 1.118055 0.058978 0.622284 0.014230 \n",
|
"0 1.380641 0.054306 0.739966 0.053385 \n",
|
||||||
"9 1.166192 0.044232 0.619351 0.024824 \n",
|
"9 1.284987 0.113903 0.696876 0.003757 \n",
|
||||||
"6 1.233377 0.063715 0.666418 0.061664 \n",
|
"6 1.291148 0.101837 0.686561 0.083989 \n",
|
||||||
"13 1.245147 0.089528 0.645836 0.006279 \n",
|
"13 1.268873 0.042412 0.645649 0.022738 \n",
|
||||||
"16 1.145362 0.071149 0.637540 0.028854 \n",
|
"16 1.147120 0.009335 0.599006 0.045637 \n",
|
||||||
"4 1.179663 0.046808 0.598699 0.035973 \n",
|
"4 1.186988 0.057330 0.642233 0.078248 \n",
|
||||||
"1 1.097773 0.005871 0.596514 0.025899 \n",
|
"1 1.413231 0.153288 0.765960 0.099535 \n",
|
||||||
"19 1.161582 0.040011 0.586831 0.006144 \n",
|
"19 1.193013 0.079000 0.691768 0.027448 \n",
|
||||||
"22 1.106277 0.060809 0.371986 0.014114 \n",
|
"22 1.043618 0.098733 0.450375 0.061660 \n",
|
||||||
"7 1.217713 0.044787 0.587896 0.024613 \n",
|
"7 1.301459 0.062143 0.660748 0.056054 \n",
|
||||||
"10 1.176654 0.045971 0.628005 0.082375 \n",
|
"10 1.433934 0.155240 0.636608 0.024064 \n",
|
||||||
"14 1.207308 0.080923 0.610284 0.039157 \n",
|
"14 1.325535 0.073539 0.672542 0.085835 \n",
|
||||||
"17 1.201498 0.035161 0.574234 0.019247 \n",
|
"17 1.237677 0.070063 0.651091 0.102538 \n",
|
||||||
"2 1.157947 0.096841 0.586462 0.010952 \n",
|
"2 1.394873 0.105286 0.637073 0.041708 \n",
|
||||||
"5 1.150511 0.027807 0.562870 0.013724 \n",
|
"5 1.270732 0.013414 0.620984 0.025393 \n",
|
||||||
"20 1.237030 0.006805 0.529591 0.064449 \n",
|
"20 1.177219 0.039633 0.586308 0.044247 \n",
|
||||||
"8 1.103937 0.046513 0.569367 0.011001 \n",
|
"8 1.165583 0.046999 0.603675 0.031774 \n",
|
||||||
"23 1.080252 0.040597 0.326627 0.002905 \n",
|
"23 0.907388 0.121543 0.354655 0.023775 \n",
|
||||||
"11 1.275601 0.087971 0.580025 0.034914 \n",
|
"11 1.210845 0.041339 0.653606 0.028117 \n",
|
||||||
"\n",
|
"\n",
|
||||||
" param_classifier__alpha param_classifier__fit_prior \\\n",
|
" param_classifier__alpha param_classifier__fit_prior \\\n",
|
||||||
"12 1 True \n",
|
"12 1 True \n",
|
||||||
|
|
@ -1726,7 +1693,7 @@
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
"cell_type": "code",
|
"cell_type": "code",
|
||||||
"execution_count": 7,
|
"execution_count": 6,
|
||||||
"metadata": {},
|
"metadata": {},
|
||||||
"outputs": [
|
"outputs": [
|
||||||
{
|
{
|
||||||
47
sitemap.xml
47
sitemap.xml
|
|
@ -2,32 +2,67 @@
|
||||||
<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
|
<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
|
||||||
<url>
|
<url>
|
||||||
<loc>None</loc>
|
<loc>None</loc>
|
||||||
<lastmod>2022-07-04</lastmod>
|
<lastmod>2022-07-08</lastmod>
|
||||||
<changefreq>daily</changefreq>
|
<changefreq>daily</changefreq>
|
||||||
</url>
|
</url>
|
||||||
<url>
|
<url>
|
||||||
<loc>None</loc>
|
<loc>None</loc>
|
||||||
<lastmod>2022-07-04</lastmod>
|
<lastmod>2022-07-08</lastmod>
|
||||||
<changefreq>daily</changefreq>
|
<changefreq>daily</changefreq>
|
||||||
</url>
|
</url>
|
||||||
<url>
|
<url>
|
||||||
<loc>None</loc>
|
<loc>None</loc>
|
||||||
<lastmod>2022-07-04</lastmod>
|
<lastmod>2022-07-08</lastmod>
|
||||||
<changefreq>daily</changefreq>
|
<changefreq>daily</changefreq>
|
||||||
</url>
|
</url>
|
||||||
<url>
|
<url>
|
||||||
<loc>None</loc>
|
<loc>None</loc>
|
||||||
<lastmod>2022-07-04</lastmod>
|
<lastmod>2022-07-08</lastmod>
|
||||||
<changefreq>daily</changefreq>
|
<changefreq>daily</changefreq>
|
||||||
</url>
|
</url>
|
||||||
<url>
|
<url>
|
||||||
<loc>None</loc>
|
<loc>None</loc>
|
||||||
<lastmod>2022-07-04</lastmod>
|
<lastmod>2022-07-08</lastmod>
|
||||||
<changefreq>daily</changefreq>
|
<changefreq>daily</changefreq>
|
||||||
</url>
|
</url>
|
||||||
<url>
|
<url>
|
||||||
<loc>None</loc>
|
<loc>None</loc>
|
||||||
<lastmod>2022-07-04</lastmod>
|
<lastmod>2022-07-08</lastmod>
|
||||||
|
<changefreq>daily</changefreq>
|
||||||
|
</url>
|
||||||
|
<url>
|
||||||
|
<loc>None</loc>
|
||||||
|
<lastmod>2022-07-08</lastmod>
|
||||||
|
<changefreq>daily</changefreq>
|
||||||
|
</url>
|
||||||
|
<url>
|
||||||
|
<loc>None</loc>
|
||||||
|
<lastmod>2022-07-08</lastmod>
|
||||||
|
<changefreq>daily</changefreq>
|
||||||
|
</url>
|
||||||
|
<url>
|
||||||
|
<loc>None</loc>
|
||||||
|
<lastmod>2022-07-08</lastmod>
|
||||||
|
<changefreq>daily</changefreq>
|
||||||
|
</url>
|
||||||
|
<url>
|
||||||
|
<loc>None</loc>
|
||||||
|
<lastmod>2022-07-08</lastmod>
|
||||||
|
<changefreq>daily</changefreq>
|
||||||
|
</url>
|
||||||
|
<url>
|
||||||
|
<loc>None</loc>
|
||||||
|
<lastmod>2022-07-08</lastmod>
|
||||||
|
<changefreq>daily</changefreq>
|
||||||
|
</url>
|
||||||
|
<url>
|
||||||
|
<loc>None</loc>
|
||||||
|
<lastmod>2022-07-08</lastmod>
|
||||||
|
<changefreq>daily</changefreq>
|
||||||
|
</url>
|
||||||
|
<url>
|
||||||
|
<loc>None</loc>
|
||||||
|
<lastmod>2022-07-08</lastmod>
|
||||||
<changefreq>daily</changefreq>
|
<changefreq>daily</changefreq>
|
||||||
</url>
|
</url>
|
||||||
</urlset>
|
</urlset>
|
||||||
BIN
sitemap.xml.gz
BIN
sitemap.xml.gz
Binary file not shown.
|
|
@ -9,7 +9,7 @@
|
||||||
|
|
||||||
|
|
||||||
<link rel="icon" href="../assets/images/favicon.png">
|
<link rel="icon" href="../assets/images/favicon.png">
|
||||||
<meta name="generator" content="mkdocs-1.3.0, mkdocs-material-8.3.8">
|
<meta name="generator" content="mkdocs-1.3.0, mkdocs-material-8.3.9">
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
@ -49,13 +49,17 @@
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
<body dir="ltr" data-md-color-scheme="" data-md-color-primary="none" data-md-color-accent="none">
|
|
||||||
|
<body dir="ltr" data-md-color-scheme="default" data-md-color-primary="" data-md-color-accent="">
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
<script>var palette=__md_get("__palette");if(palette&&"object"==typeof palette.color)for(var key of Object.keys(palette.color))document.body.setAttribute("data-md-color-"+key,palette.color[key])</script>
|
||||||
|
|
||||||
<input class="md-toggle" data-md-toggle="drawer" type="checkbox" id="__drawer" autocomplete="off">
|
<input class="md-toggle" data-md-toggle="drawer" type="checkbox" id="__drawer" autocomplete="off">
|
||||||
<input class="md-toggle" data-md-toggle="search" type="checkbox" id="__search" autocomplete="off">
|
<input class="md-toggle" data-md-toggle="search" type="checkbox" id="__search" autocomplete="off">
|
||||||
<label class="md-overlay" for="__drawer"></label>
|
<label class="md-overlay" for="__drawer"></label>
|
||||||
|
|
@ -97,9 +101,76 @@
|
||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
|
<form class="md-header__option" data-md-component="palette">
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
<input class="md-option" data-md-color-media="" data-md-color-scheme="default" data-md-color-primary="" data-md-color-accent="" aria-label="Switch to dark mode" type="radio" name="__palette" id="__palette_1">
|
||||||
|
|
||||||
|
<label class="md-header__button md-icon" title="Switch to dark mode" for="__palette_2" hidden>
|
||||||
|
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M12 8a4 4 0 0 0-4 4 4 4 0 0 0 4 4 4 4 0 0 0 4-4 4 4 0 0 0-4-4m0 10a6 6 0 0 1-6-6 6 6 0 0 1 6-6 6 6 0 0 1 6 6 6 6 0 0 1-6 6m8-9.31V4h-4.69L12 .69 8.69 4H4v4.69L.69 12 4 15.31V20h4.69L12 23.31 15.31 20H20v-4.69L23.31 12 20 8.69Z"/></svg>
|
||||||
|
</label>
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
<input class="md-option" data-md-color-media="" data-md-color-scheme="slate" data-md-color-primary="" data-md-color-accent="" aria-label="Switch to light mode" type="radio" name="__palette" id="__palette_2">
|
||||||
|
|
||||||
|
<label class="md-header__button md-icon" title="Switch to light mode" for="__palette_1" hidden>
|
||||||
|
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M12 18c-.89 0-1.74-.2-2.5-.55C11.56 16.5 13 14.42 13 12c0-2.42-1.44-4.5-3.5-5.45C10.26 6.2 11.11 6 12 6a6 6 0 0 1 6 6 6 6 0 0 1-6 6m8-9.31V4h-4.69L12 .69 8.69 4H4v4.69L.69 12 4 15.31V20h4.69L12 23.31 15.31 20H20v-4.69L23.31 12 20 8.69Z"/></svg>
|
||||||
|
</label>
|
||||||
|
|
||||||
|
|
||||||
|
</form>
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
<label class="md-header__button md-icon" for="__search">
|
||||||
|
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M9.5 3A6.5 6.5 0 0 1 16 9.5c0 1.61-.59 3.09-1.56 4.23l.27.27h.79l5 5-1.5 1.5-5-5v-.79l-.27-.27A6.516 6.516 0 0 1 9.5 16 6.5 6.5 0 0 1 3 9.5 6.5 6.5 0 0 1 9.5 3m0 2C7 5 5 7 5 9.5S7 14 9.5 14 14 12 14 9.5 12 5 9.5 5Z"/></svg>
|
||||||
|
</label>
|
||||||
|
<div class="md-search" data-md-component="search" role="dialog">
|
||||||
|
<label class="md-search__overlay" for="__search"></label>
|
||||||
|
<div class="md-search__inner" role="search">
|
||||||
|
<form class="md-search__form" name="search">
|
||||||
|
<input type="text" class="md-search__input" name="query" aria-label="Search" placeholder="Search" autocapitalize="off" autocorrect="off" autocomplete="off" spellcheck="false" data-md-component="search-query" required>
|
||||||
|
<label class="md-search__icon md-icon" for="__search">
|
||||||
|
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M9.5 3A6.5 6.5 0 0 1 16 9.5c0 1.61-.59 3.09-1.56 4.23l.27.27h.79l5 5-1.5 1.5-5-5v-.79l-.27-.27A6.516 6.516 0 0 1 9.5 16 6.5 6.5 0 0 1 3 9.5 6.5 6.5 0 0 1 9.5 3m0 2C7 5 5 7 5 9.5S7 14 9.5 14 14 12 14 9.5 12 5 9.5 5Z"/></svg>
|
||||||
|
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M20 11v2H8l5.5 5.5-1.42 1.42L4.16 12l7.92-7.92L13.5 5.5 8 11h12Z"/></svg>
|
||||||
|
</label>
|
||||||
|
<nav class="md-search__options" aria-label="Search">
|
||||||
|
|
||||||
|
<button type="reset" class="md-search__icon md-icon" aria-label="Clear" tabindex="-1">
|
||||||
|
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M19 6.41 17.59 5 12 10.59 6.41 5 5 6.41 10.59 12 5 17.59 6.41 19 12 13.41 17.59 19 19 17.59 13.41 12 19 6.41Z"/></svg>
|
||||||
|
</button>
|
||||||
|
</nav>
|
||||||
|
|
||||||
|
</form>
|
||||||
|
<div class="md-search__output">
|
||||||
|
<div class="md-search__scrollwrap" data-md-scrollfix>
|
||||||
|
<div class="md-search-result" data-md-component="search-result">
|
||||||
|
<div class="md-search-result__meta">
|
||||||
|
Initializing search
|
||||||
|
</div>
|
||||||
|
<ol class="md-search-result__list"></ol>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
|
||||||
|
<div class="md-header__source">
|
||||||
|
<a href="https://github.com/schmelczer/great-ai" title="Go to repository" class="md-source" data-md-component="source">
|
||||||
|
<div class="md-source__icon md-icon">
|
||||||
|
|
||||||
|
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 448 512"><!--! Font Awesome Free 6.1.1 by @fontawesome - https://fontawesome.com License - https://fontawesome.com/license/free (Icons: CC BY 4.0, Fonts: SIL OFL 1.1, Code: MIT License) Copyright 2022 Fonticons, Inc.--><path d="M439.55 236.05 244 40.45a28.87 28.87 0 0 0-40.81 0l-40.66 40.63 51.52 51.52c27.06-9.14 52.68 16.77 43.39 43.68l49.66 49.66c34.23-11.8 61.18 31 35.47 56.69-26.49 26.49-70.21-2.87-56-37.34L240.22 199v121.85c25.3 12.54 22.26 41.85 9.08 55a34.34 34.34 0 0 1-48.55 0c-17.57-17.6-11.07-46.91 11.25-56v-123c-20.8-8.51-24.6-30.74-18.64-45L142.57 101 8.45 235.14a28.86 28.86 0 0 0 0 40.81l195.61 195.6a28.86 28.86 0 0 0 40.8 0l194.69-194.69a28.86 28.86 0 0 0 0-40.81z"/></svg>
|
||||||
|
</div>
|
||||||
|
<div class="md-source__repository">
|
||||||
|
GitHub
|
||||||
|
</div>
|
||||||
|
</a>
|
||||||
|
</div>
|
||||||
|
|
||||||
</nav>
|
</nav>
|
||||||
|
|
||||||
</header>
|
</header>
|
||||||
|
|
@ -133,6 +204,18 @@
|
||||||
GreatAI documentation
|
GreatAI documentation
|
||||||
</label>
|
</label>
|
||||||
|
|
||||||
|
<div class="md-nav__source">
|
||||||
|
<a href="https://github.com/schmelczer/great-ai" title="Go to repository" class="md-source" data-md-component="source">
|
||||||
|
<div class="md-source__icon md-icon">
|
||||||
|
|
||||||
|
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 448 512"><!--! Font Awesome Free 6.1.1 by @fontawesome - https://fontawesome.com License - https://fontawesome.com/license/free (Icons: CC BY 4.0, Fonts: SIL OFL 1.1, Code: MIT License) Copyright 2022 Fonticons, Inc.--><path d="M439.55 236.05 244 40.45a28.87 28.87 0 0 0-40.81 0l-40.66 40.63 51.52 51.52c27.06-9.14 52.68 16.77 43.39 43.68l49.66 49.66c34.23-11.8 61.18 31 35.47 56.69-26.49 26.49-70.21-2.87-56-37.34L240.22 199v121.85c25.3 12.54 22.26 41.85 9.08 55a34.34 34.34 0 0 1-48.55 0c-17.57-17.6-11.07-46.91 11.25-56v-123c-20.8-8.51-24.6-30.74-18.64-45L142.57 101 8.45 235.14a28.86 28.86 0 0 0 0 40.81l195.61 195.6a28.86 28.86 0 0 0 40.8 0l194.69-194.69a28.86 28.86 0 0 0 0-40.81z"/></svg>
|
||||||
|
</div>
|
||||||
|
<div class="md-source__repository">
|
||||||
|
GitHub
|
||||||
|
</div>
|
||||||
|
</a>
|
||||||
|
</div>
|
||||||
|
|
||||||
<ul class="md-nav__list" data-md-scrollfix>
|
<ul class="md-nav__list" data-md-scrollfix>
|
||||||
|
|
||||||
|
|
||||||
|
|
@ -218,6 +301,21 @@
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
<li class="md-nav__item">
|
||||||
|
<a href="../simple-mag/train/" class="md-nav__link">
|
||||||
|
Train a field of study (domain) classification of sentences on the MAG dataset
|
||||||
|
</a>
|
||||||
|
</li>
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
</ul>
|
</ul>
|
||||||
</nav>
|
</nav>
|
||||||
</div>
|
</div>
|
||||||
|
|
@ -245,14 +343,33 @@
|
||||||
<div class="md-content" data-md-component="content">
|
<div class="md-content" data-md-component="content">
|
||||||
<article class="md-content__inner md-typeset">
|
<article class="md-content__inner md-typeset">
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
<a href="https://github.com/schmelczer/great-ai/edit/main/docs/tutorials.md" title="Edit this page" class="md-content__button md-icon">
|
||||||
|
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M20.71 7.04c.39-.39.39-1.04 0-1.41l-2.34-2.34c-.37-.39-1.02-.39-1.41 0l-1.84 1.83 3.75 3.75M3 17.25V21h3.75L17.81 9.93l-3.75-3.75L3 17.25Z"/></svg>
|
||||||
|
</a>
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
<h1>Tutorials</h1>
|
<h1>Tutorials</h1>
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
<hr>
|
||||||
|
<div class="md-source-file">
|
||||||
|
<small>
|
||||||
|
|
||||||
|
Last update:
|
||||||
|
<span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-date">July 8, 2022</span>
|
||||||
|
|
||||||
|
|
||||||
|
</small>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
</article>
|
</article>
|
||||||
|
|
||||||
</div>
|
</div>
|
||||||
|
|
@ -321,7 +438,7 @@
|
||||||
<script id="__config" type="application/json">{"base": "..", "features": [], "search": "../assets/javascripts/workers/search.b97dbffb.min.js", "translations": {"clipboard.copied": "Copied to clipboard", "clipboard.copy": "Copy to clipboard", "search.config.lang": "en", "search.config.pipeline": "trimmer, stopWordFilter", "search.config.separator": "[\\s\\-]+", "search.placeholder": "Search", "search.result.more.one": "1 more on this page", "search.result.more.other": "# more on this page", "search.result.none": "No matching documents", "search.result.one": "1 matching document", "search.result.other": "# matching documents", "search.result.placeholder": "Type to start searching", "search.result.term.missing": "Missing", "select.version.title": "Select version"}}</script>
|
<script id="__config" type="application/json">{"base": "..", "features": [], "search": "../assets/javascripts/workers/search.b97dbffb.min.js", "translations": {"clipboard.copied": "Copied to clipboard", "clipboard.copy": "Copy to clipboard", "search.config.lang": "en", "search.config.pipeline": "trimmer, stopWordFilter", "search.config.separator": "[\\s\\-]+", "search.placeholder": "Search", "search.result.more.one": "1 more on this page", "search.result.more.other": "# more on this page", "search.result.none": "No matching documents", "search.result.one": "1 matching document", "search.result.other": "# matching documents", "search.result.placeholder": "Type to start searching", "search.result.term.missing": "Missing", "select.version.title": "Select version"}}</script>
|
||||||
|
|
||||||
|
|
||||||
<script src="../assets/javascripts/bundle.0238f547.min.js"></script>
|
<script src="../assets/javascripts/bundle.6c7ad80a.min.js"></script>
|
||||||
|
|
||||||
|
|
||||||
</body>
|
</body>
|
||||||
|
|
|
||||||
0
tutorials/tutorials.md
Normal file
0
tutorials/tutorials.md
Normal file
Loading…
Add table
Add a link
Reference in a new issue