4049 lines
No EOL
258 KiB
HTML
4049 lines
No EOL
258 KiB
HTML
|
|
<!doctype html>
|
|
<html lang="en" class="no-js">
|
|
<head>
|
|
|
|
<meta charset="utf-8">
|
|
<meta name="viewport" content="width=device-width,initial-scale=1">
|
|
|
|
<meta name="description" content="GreatAI helps you easily transform your prototype AI code into production-ready software.">
|
|
|
|
|
|
<meta name="author" content="András Schmelczer">
|
|
|
|
|
|
<link rel="icon" href="../../media/favicon.ico">
|
|
<meta name="generator" content="mkdocs-1.3.0, mkdocs-material-8.3.9">
|
|
|
|
|
|
|
|
<title>Utilities - GreatAI documentation</title>
|
|
|
|
|
|
|
|
<link rel="stylesheet" href="../../assets/stylesheets/main.1d29e8d0.min.css">
|
|
|
|
|
|
<link rel="stylesheet" href="../../assets/stylesheets/palette.cbb835fc.min.css">
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
<link rel="preconnect" href="https://fonts.gstatic.com" crossorigin>
|
|
<link rel="stylesheet" href="https://fonts.googleapis.com/css?family=Roboto:300,300i,400,400i,700,700i%7CRoboto+Mono:400,400i,700,700i&display=fallback">
|
|
<style>:root{--md-text-font:"Roboto";--md-code-font:"Roboto Mono"}</style>
|
|
|
|
|
|
|
|
<link rel="stylesheet" href="../../assets/_mkdocstrings.css">
|
|
|
|
<script>__md_scope=new URL("../..",location),__md_get=(e,_=localStorage,t=__md_scope)=>JSON.parse(_.getItem(t.pathname+"."+e)),__md_set=(e,_,t=localStorage,a=__md_scope)=>{try{t.setItem(a.pathname+"."+e,JSON.stringify(_))}catch(e){}}</script>
|
|
|
|
|
|
|
|
|
|
|
|
<meta property="og:title" content="">
|
|
<meta property="og:site_name" content="">
|
|
<meta property="og:url" content="">
|
|
<meta property="og:description" content="Transform your prototype AI code into production-ready software.">
|
|
<meta property="og:type" content="">
|
|
<meta property="og:image" content=https://great-ai.scoutinscience.com/media/og-image.png>
|
|
|
|
<style>
|
|
.jupyter-wrapper a {
|
|
color: var(--md-typeset-a-color) !important;
|
|
}
|
|
</style>
|
|
|
|
</head>
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
<body dir="ltr" data-md-color-scheme="default" data-md-color-primary="light-blue" data-md-color-accent="">
|
|
|
|
|
|
|
|
<script>var palette=__md_get("__palette");if(palette&&"object"==typeof palette.color)for(var key of Object.keys(palette.color))document.body.setAttribute("data-md-color-"+key,palette.color[key])</script>
|
|
|
|
<input class="md-toggle" data-md-toggle="drawer" type="checkbox" id="__drawer" autocomplete="off">
|
|
<input class="md-toggle" data-md-toggle="search" type="checkbox" id="__search" autocomplete="off">
|
|
<label class="md-overlay" for="__drawer"></label>
|
|
<div data-md-component="skip">
|
|
|
|
|
|
<a href="#utilities" class="md-skip">
|
|
Skip to content
|
|
</a>
|
|
|
|
</div>
|
|
<div data-md-component="announce">
|
|
|
|
</div>
|
|
|
|
|
|
|
|
|
|
<header class="md-header" data-md-component="header">
|
|
<nav class="md-header__inner md-grid" aria-label="Header">
|
|
<a href="../.." title="GreatAI documentation" class="md-header__button md-logo" aria-label="GreatAI documentation" data-md-component="logo">
|
|
|
|
|
|
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M12 8a3 3 0 0 0 3-3 3 3 0 0 0-3-3 3 3 0 0 0-3 3 3 3 0 0 0 3 3m0 3.54C9.64 9.35 6.5 8 3 8v11c3.5 0 6.64 1.35 9 3.54 2.36-2.19 5.5-3.54 9-3.54V8c-3.5 0-6.64 1.35-9 3.54Z"/></svg>
|
|
|
|
</a>
|
|
<label class="md-header__button md-icon" for="__drawer">
|
|
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M3 6h18v2H3V6m0 5h18v2H3v-2m0 5h18v2H3v-2Z"/></svg>
|
|
</label>
|
|
<div class="md-header__title" data-md-component="header-title">
|
|
<div class="md-header__ellipsis">
|
|
<div class="md-header__topic">
|
|
<span class="md-ellipsis">
|
|
GreatAI documentation
|
|
</span>
|
|
</div>
|
|
<div class="md-header__topic" data-md-component="header-topic">
|
|
<span class="md-ellipsis">
|
|
|
|
Utilities
|
|
|
|
</span>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
|
|
<form class="md-header__option" data-md-component="palette">
|
|
|
|
|
|
|
|
<input class="md-option" data-md-color-media="(prefers-color-scheme: light)" data-md-color-scheme="default" data-md-color-primary="light-blue" data-md-color-accent="" aria-label="Switch to dark mode" type="radio" name="__palette" id="__palette_1">
|
|
|
|
<label class="md-header__button md-icon" title="Switch to dark mode" for="__palette_2" hidden>
|
|
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M12 2a7 7 0 0 0-7 7c0 2.38 1.19 4.47 3 5.74V17a1 1 0 0 0 1 1h6a1 1 0 0 0 1-1v-2.26c1.81-1.27 3-3.36 3-5.74a7 7 0 0 0-7-7M9 21a1 1 0 0 0 1 1h4a1 1 0 0 0 1-1v-1H9v1Z"/></svg>
|
|
</label>
|
|
|
|
|
|
|
|
|
|
<input class="md-option" data-md-color-media="(prefers-color-scheme: dark)" data-md-color-scheme="slate" data-md-color-primary="light-blue" data-md-color-accent="" aria-label="Switch to light mode" type="radio" name="__palette" id="__palette_2">
|
|
|
|
<label class="md-header__button md-icon" title="Switch to light mode" for="__palette_1" hidden>
|
|
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M12 2a7 7 0 0 1 7 7c0 2.38-1.19 4.47-3 5.74V17a1 1 0 0 1-1 1H9a1 1 0 0 1-1-1v-2.26C6.19 13.47 5 11.38 5 9a7 7 0 0 1 7-7M9 21v-1h6v1a1 1 0 0 1-1 1h-4a1 1 0 0 1-1-1m3-17a5 5 0 0 0-5 5c0 2.05 1.23 3.81 3 4.58V16h4v-2.42c1.77-.77 3-2.53 3-4.58a5 5 0 0 0-5-5Z"/></svg>
|
|
</label>
|
|
|
|
|
|
</form>
|
|
|
|
|
|
|
|
<label class="md-header__button md-icon" for="__search">
|
|
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M9.5 3A6.5 6.5 0 0 1 16 9.5c0 1.61-.59 3.09-1.56 4.23l.27.27h.79l5 5-1.5 1.5-5-5v-.79l-.27-.27A6.516 6.516 0 0 1 9.5 16 6.5 6.5 0 0 1 3 9.5 6.5 6.5 0 0 1 9.5 3m0 2C7 5 5 7 5 9.5S7 14 9.5 14 14 12 14 9.5 12 5 9.5 5Z"/></svg>
|
|
</label>
|
|
<div class="md-search" data-md-component="search" role="dialog">
|
|
<label class="md-search__overlay" for="__search"></label>
|
|
<div class="md-search__inner" role="search">
|
|
<form class="md-search__form" name="search">
|
|
<input type="text" class="md-search__input" name="query" aria-label="Search" placeholder="Search" autocapitalize="off" autocorrect="off" autocomplete="off" spellcheck="false" data-md-component="search-query" required>
|
|
<label class="md-search__icon md-icon" for="__search">
|
|
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M9.5 3A6.5 6.5 0 0 1 16 9.5c0 1.61-.59 3.09-1.56 4.23l.27.27h.79l5 5-1.5 1.5-5-5v-.79l-.27-.27A6.516 6.516 0 0 1 9.5 16 6.5 6.5 0 0 1 3 9.5 6.5 6.5 0 0 1 9.5 3m0 2C7 5 5 7 5 9.5S7 14 9.5 14 14 12 14 9.5 12 5 9.5 5Z"/></svg>
|
|
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M20 11v2H8l5.5 5.5-1.42 1.42L4.16 12l7.92-7.92L13.5 5.5 8 11h12Z"/></svg>
|
|
</label>
|
|
<nav class="md-search__options" aria-label="Search">
|
|
|
|
<button type="reset" class="md-search__icon md-icon" aria-label="Clear" tabindex="-1">
|
|
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M19 6.41 17.59 5 12 10.59 6.41 5 5 6.41 10.59 12 5 17.59 6.41 19 12 13.41 17.59 19 19 17.59 13.41 12 19 6.41Z"/></svg>
|
|
</button>
|
|
</nav>
|
|
|
|
</form>
|
|
<div class="md-search__output">
|
|
<div class="md-search__scrollwrap" data-md-scrollfix>
|
|
<div class="md-search-result" data-md-component="search-result">
|
|
<div class="md-search-result__meta">
|
|
Initializing search
|
|
</div>
|
|
<ol class="md-search-result__list"></ol>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
|
|
|
|
<div class="md-header__source">
|
|
<a href="https://github.com/schmelczer/great-ai" title="Go to repository" class="md-source" data-md-component="source">
|
|
<div class="md-source__icon md-icon">
|
|
|
|
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 448 512"><!--! Font Awesome Free 6.1.1 by @fontawesome - https://fontawesome.com License - https://fontawesome.com/license/free (Icons: CC BY 4.0, Fonts: SIL OFL 1.1, Code: MIT License) Copyright 2022 Fonticons, Inc.--><path d="M439.55 236.05 244 40.45a28.87 28.87 0 0 0-40.81 0l-40.66 40.63 51.52 51.52c27.06-9.14 52.68 16.77 43.39 43.68l49.66 49.66c34.23-11.8 61.18 31 35.47 56.69-26.49 26.49-70.21-2.87-56-37.34L240.22 199v121.85c25.3 12.54 22.26 41.85 9.08 55a34.34 34.34 0 0 1-48.55 0c-17.57-17.6-11.07-46.91 11.25-56v-123c-20.8-8.51-24.6-30.74-18.64-45L142.57 101 8.45 235.14a28.86 28.86 0 0 0 0 40.81l195.61 195.6a28.86 28.86 0 0 0 40.8 0l194.69-194.69a28.86 28.86 0 0 0 0-40.81z"/></svg>
|
|
</div>
|
|
<div class="md-source__repository">
|
|
schmelczer/great-ai
|
|
</div>
|
|
</a>
|
|
</div>
|
|
|
|
</nav>
|
|
|
|
</header>
|
|
|
|
<div class="md-container" data-md-component="container">
|
|
|
|
|
|
|
|
|
|
|
|
|
|
<main class="md-main" data-md-component="main">
|
|
<div class="md-main__inner md-grid">
|
|
|
|
|
|
|
|
<div class="md-sidebar md-sidebar--primary" data-md-component="sidebar" data-md-type="navigation" >
|
|
<div class="md-sidebar__scrollwrap">
|
|
<div class="md-sidebar__inner">
|
|
|
|
|
|
|
|
<nav class="md-nav md-nav--primary" aria-label="Navigation" data-md-level="0">
|
|
<label class="md-nav__title" for="__drawer">
|
|
<a href="../.." title="GreatAI documentation" class="md-nav__button md-logo" aria-label="GreatAI documentation" data-md-component="logo">
|
|
|
|
|
|
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M12 8a3 3 0 0 0 3-3 3 3 0 0 0-3-3 3 3 0 0 0-3 3 3 3 0 0 0 3 3m0 3.54C9.64 9.35 6.5 8 3 8v11c3.5 0 6.64 1.35 9 3.54 2.36-2.19 5.5-3.54 9-3.54V8c-3.5 0-6.64 1.35-9 3.54Z"/></svg>
|
|
|
|
</a>
|
|
GreatAI documentation
|
|
</label>
|
|
|
|
<div class="md-nav__source">
|
|
<a href="https://github.com/schmelczer/great-ai" title="Go to repository" class="md-source" data-md-component="source">
|
|
<div class="md-source__icon md-icon">
|
|
|
|
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 448 512"><!--! Font Awesome Free 6.1.1 by @fontawesome - https://fontawesome.com License - https://fontawesome.com/license/free (Icons: CC BY 4.0, Fonts: SIL OFL 1.1, Code: MIT License) Copyright 2022 Fonticons, Inc.--><path d="M439.55 236.05 244 40.45a28.87 28.87 0 0 0-40.81 0l-40.66 40.63 51.52 51.52c27.06-9.14 52.68 16.77 43.39 43.68l49.66 49.66c34.23-11.8 61.18 31 35.47 56.69-26.49 26.49-70.21-2.87-56-37.34L240.22 199v121.85c25.3 12.54 22.26 41.85 9.08 55a34.34 34.34 0 0 1-48.55 0c-17.57-17.6-11.07-46.91 11.25-56v-123c-20.8-8.51-24.6-30.74-18.64-45L142.57 101 8.45 235.14a28.86 28.86 0 0 0 0 40.81l195.61 195.6a28.86 28.86 0 0 0 40.8 0l194.69-194.69a28.86 28.86 0 0 0 0-40.81z"/></svg>
|
|
</div>
|
|
<div class="md-source__repository">
|
|
schmelczer/great-ai
|
|
</div>
|
|
</a>
|
|
</div>
|
|
|
|
<ul class="md-nav__list" data-md-scrollfix>
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
<li class="md-nav__item">
|
|
<a href="../.." class="md-nav__link">
|
|
Overview
|
|
</a>
|
|
</li>
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
<li class="md-nav__item md-nav__item--nested">
|
|
|
|
|
|
<input class="md-nav__toggle md-toggle" data-md-toggle="__nav_2" type="checkbox" id="__nav_2" >
|
|
|
|
|
|
|
|
|
|
<label class="md-nav__link" for="__nav_2">
|
|
Tutorial
|
|
<span class="md-nav__icon md-icon"></span>
|
|
</label>
|
|
|
|
<nav class="md-nav" aria-label="Tutorial" data-md-level="1">
|
|
<label class="md-nav__title" for="__nav_2">
|
|
<span class="md-nav__icon md-icon"></span>
|
|
Tutorial
|
|
</label>
|
|
<ul class="md-nav__list" data-md-scrollfix>
|
|
|
|
|
|
|
|
|
|
|
|
|
|
<li class="md-nav__item">
|
|
<a href="../../tutorial/" class="md-nav__link">
|
|
Tutorial overview
|
|
</a>
|
|
</li>
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
<li class="md-nav__item">
|
|
<a href="../../tutorial/train/" class="md-nav__link">
|
|
Train your model
|
|
</a>
|
|
</li>
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
<li class="md-nav__item">
|
|
<a href="../../tutorial/deploy/" class="md-nav__link">
|
|
Harden and deploy your app
|
|
</a>
|
|
</li>
|
|
|
|
|
|
|
|
|
|
</ul>
|
|
</nav>
|
|
</li>
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
<li class="md-nav__item md-nav__item--nested">
|
|
|
|
|
|
<input class="md-nav__toggle md-toggle" data-md-toggle="__nav_3" type="checkbox" id="__nav_3" >
|
|
|
|
|
|
|
|
|
|
<label class="md-nav__link" for="__nav_3">
|
|
User Guides
|
|
<span class="md-nav__icon md-icon"></span>
|
|
</label>
|
|
|
|
<nav class="md-nav" aria-label="User Guides" data-md-level="1">
|
|
<label class="md-nav__title" for="__nav_3">
|
|
<span class="md-nav__icon md-icon"></span>
|
|
User Guides
|
|
</label>
|
|
<ul class="md-nav__list" data-md-scrollfix>
|
|
|
|
|
|
|
|
|
|
|
|
|
|
<li class="md-nav__item">
|
|
<a href="../../how-to-guides/install/" class="md-nav__link">
|
|
Installation guide
|
|
</a>
|
|
</li>
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
<li class="md-nav__item">
|
|
<a href="../../how-to-guides/create-service/" class="md-nav__link">
|
|
How to create a GreatAI service
|
|
</a>
|
|
</li>
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
<li class="md-nav__item">
|
|
<a href="../../how-to-guides/configure-service/" class="md-nav__link">
|
|
How to configure GreatAI
|
|
</a>
|
|
</li>
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
<li class="md-nav__item">
|
|
<a href="../../how-to-guides/use-service/" class="md-nav__link">
|
|
How to deploy a GreatAI service
|
|
</a>
|
|
</li>
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
<li class="md-nav__item">
|
|
<a href="../../how-to-guides/handle-training-data/" class="md-nav__link">
|
|
How to manage training data
|
|
</a>
|
|
</li>
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
<li class="md-nav__item">
|
|
<a href="../../how-to-guides/large_file/" class="md-nav__link">
|
|
How to use LargeFile-s
|
|
</a>
|
|
</li>
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
<li class="md-nav__item">
|
|
<a href="../../how-to-guides/call-remote/" class="md-nav__link">
|
|
How to call remote GreatAI instances
|
|
</a>
|
|
</li>
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
<li class="md-nav__item">
|
|
<a href="../../how-to-guides/scraping/" class="md-nav__link">
|
|
Scraping
|
|
</a>
|
|
</li>
|
|
|
|
|
|
|
|
|
|
</ul>
|
|
</nav>
|
|
</li>
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
<li class="md-nav__item md-nav__item--active md-nav__item--nested">
|
|
|
|
|
|
<input class="md-nav__toggle md-toggle" data-md-toggle="__nav_4" type="checkbox" id="__nav_4" checked>
|
|
|
|
|
|
|
|
|
|
<label class="md-nav__link" for="__nav_4">
|
|
Reference
|
|
<span class="md-nav__icon md-icon"></span>
|
|
</label>
|
|
|
|
<nav class="md-nav" aria-label="Reference" data-md-level="1">
|
|
<label class="md-nav__title" for="__nav_4">
|
|
<span class="md-nav__icon md-icon"></span>
|
|
Reference
|
|
</label>
|
|
<ul class="md-nav__list" data-md-scrollfix>
|
|
|
|
|
|
|
|
|
|
|
|
|
|
<li class="md-nav__item">
|
|
<a href="../" class="md-nav__link">
|
|
GreatAI reference
|
|
</a>
|
|
</li>
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
<li class="md-nav__item md-nav__item--active">
|
|
|
|
<input class="md-nav__toggle md-toggle" data-md-toggle="toc" type="checkbox" id="__toc">
|
|
|
|
|
|
|
|
|
|
|
|
<label class="md-nav__link md-nav__link--active" for="__toc">
|
|
Utilities
|
|
<span class="md-nav__icon md-icon"></span>
|
|
</label>
|
|
|
|
<a href="./" class="md-nav__link md-nav__link--active">
|
|
Utilities
|
|
</a>
|
|
|
|
|
|
|
|
<nav class="md-nav md-nav--secondary" aria-label="Table of contents">
|
|
|
|
|
|
|
|
|
|
|
|
|
|
<label class="md-nav__title" for="__toc">
|
|
<span class="md-nav__icon md-icon"></span>
|
|
Table of contents
|
|
</label>
|
|
<ul class="md-nav__list" data-md-component="toc" data-md-scrollfix>
|
|
|
|
<li class="md-nav__item">
|
|
<a href="#nlp-tools" class="md-nav__link">
|
|
NLP tools
|
|
</a>
|
|
|
|
<nav class="md-nav" aria-label="NLP tools">
|
|
<ul class="md-nav__list">
|
|
|
|
<li class="md-nav__item">
|
|
<a href="#great_ai.utilities.clean.clean" class="md-nav__link">
|
|
clean()
|
|
</a>
|
|
|
|
</li>
|
|
|
|
<li class="md-nav__item">
|
|
<a href="#great_ai.utilities.get_sentences.get_sentences" class="md-nav__link">
|
|
get_sentences()
|
|
</a>
|
|
|
|
</li>
|
|
|
|
</ul>
|
|
</nav>
|
|
|
|
</li>
|
|
|
|
<li class="md-nav__item">
|
|
<a href="#parallel-processing" class="md-nav__link">
|
|
Parallel processing
|
|
</a>
|
|
|
|
<nav class="md-nav" aria-label="Parallel processing">
|
|
<ul class="md-nav__list">
|
|
|
|
<li class="md-nav__item">
|
|
<a href="#great_ai.utilities.simple_parallel_map" class="md-nav__link">
|
|
simple_parallel_map()
|
|
</a>
|
|
|
|
</li>
|
|
|
|
<li class="md-nav__item">
|
|
<a href="#great_ai.utilities.parallel_map.parallel_map.parallel_map" class="md-nav__link">
|
|
parallel_map()
|
|
</a>
|
|
|
|
</li>
|
|
|
|
<li class="md-nav__item">
|
|
<a href="#great_ai.utilities.threaded_parallel_map" class="md-nav__link">
|
|
threaded_parallel_map()
|
|
</a>
|
|
|
|
</li>
|
|
|
|
</ul>
|
|
</nav>
|
|
|
|
</li>
|
|
|
|
<li class="md-nav__item">
|
|
<a href="#composable-parallel-processing" class="md-nav__link">
|
|
Composable parallel processing
|
|
</a>
|
|
|
|
<nav class="md-nav" aria-label="Composable parallel processing">
|
|
<ul class="md-nav__list">
|
|
|
|
<li class="md-nav__item">
|
|
<a href="#great_ai.utilities.chunk.chunk" class="md-nav__link">
|
|
chunk()
|
|
</a>
|
|
|
|
</li>
|
|
|
|
<li class="md-nav__item">
|
|
<a href="#great_ai.utilities.unchunk.unchunk" class="md-nav__link">
|
|
unchunk()
|
|
</a>
|
|
|
|
</li>
|
|
|
|
</ul>
|
|
</nav>
|
|
|
|
</li>
|
|
|
|
<li class="md-nav__item">
|
|
<a href="#operations" class="md-nav__link">
|
|
Operations
|
|
</a>
|
|
|
|
<nav class="md-nav" aria-label="Operations">
|
|
<ul class="md-nav__list">
|
|
|
|
<li class="md-nav__item">
|
|
<a href="#great_ai.utilities.ConfigFile" class="md-nav__link">
|
|
ConfigFile
|
|
</a>
|
|
|
|
<nav class="md-nav" aria-label="ConfigFile">
|
|
<ul class="md-nav__list">
|
|
|
|
<li class="md-nav__item">
|
|
<a href="#great_ai.utilities.config_file.config_file.ConfigFile.__init__" class="md-nav__link">
|
|
__init__()
|
|
</a>
|
|
|
|
</li>
|
|
|
|
<li class="md-nav__item">
|
|
<a href="#great_ai.utilities.config_file.config_file.ConfigFile.path" class="md-nav__link">
|
|
path()
|
|
</a>
|
|
|
|
</li>
|
|
|
|
</ul>
|
|
</nav>
|
|
|
|
</li>
|
|
|
|
<li class="md-nav__item">
|
|
<a href="#great_ai.utilities.get_logger" class="md-nav__link">
|
|
get_logger()
|
|
</a>
|
|
|
|
</li>
|
|
|
|
</ul>
|
|
</nav>
|
|
|
|
</li>
|
|
|
|
</ul>
|
|
|
|
</nav>
|
|
|
|
</li>
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
<li class="md-nav__item">
|
|
<a href="../large-file/" class="md-nav__link">
|
|
LargeFile
|
|
</a>
|
|
</li>
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
<li class="md-nav__item">
|
|
<a href="../views/" class="md-nav__link">
|
|
View models
|
|
</a>
|
|
</li>
|
|
|
|
|
|
|
|
|
|
</ul>
|
|
</nav>
|
|
</li>
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
<li class="md-nav__item md-nav__item--nested">
|
|
|
|
|
|
<input class="md-nav__toggle md-toggle" data-md-toggle="__nav_5" type="checkbox" id="__nav_5" >
|
|
|
|
|
|
|
|
|
|
<label class="md-nav__link" for="__nav_5">
|
|
Examples
|
|
<span class="md-nav__icon md-icon"></span>
|
|
</label>
|
|
|
|
<nav class="md-nav" aria-label="Examples" data-md-level="1">
|
|
<label class="md-nav__title" for="__nav_5">
|
|
<span class="md-nav__icon md-icon"></span>
|
|
Examples
|
|
</label>
|
|
<ul class="md-nav__list" data-md-scrollfix>
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
<li class="md-nav__item md-nav__item--nested">
|
|
|
|
|
|
<input class="md-nav__toggle md-toggle" data-md-toggle="__nav_5_1" type="checkbox" id="__nav_5_1" >
|
|
|
|
|
|
|
|
|
|
<label class="md-nav__link" for="__nav_5_1">
|
|
Explainable Naive Bayes
|
|
<span class="md-nav__icon md-icon"></span>
|
|
</label>
|
|
|
|
<nav class="md-nav" aria-label="Explainable Naive Bayes" data-md-level="2">
|
|
<label class="md-nav__title" for="__nav_5_1">
|
|
<span class="md-nav__icon md-icon"></span>
|
|
Explainable Naive Bayes
|
|
</label>
|
|
<ul class="md-nav__list" data-md-scrollfix>
|
|
|
|
|
|
|
|
|
|
|
|
|
|
<li class="md-nav__item">
|
|
<a href="../../examples/simple/data/" class="md-nav__link">
|
|
Simple example: data engineering
|
|
</a>
|
|
</li>
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
<li class="md-nav__item">
|
|
<a href="../../examples/simple/train/" class="md-nav__link">
|
|
Optimise and train a model
|
|
</a>
|
|
</li>
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
<li class="md-nav__item">
|
|
<a href="../../examples/simple/deploy/" class="md-nav__link">
|
|
Hardening and deployment
|
|
</a>
|
|
</li>
|
|
|
|
|
|
|
|
|
|
</ul>
|
|
</nav>
|
|
</li>
|
|
|
|
|
|
|
|
|
|
</ul>
|
|
</nav>
|
|
</li>
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
<li class="md-nav__item">
|
|
<a href="../../explanation/" class="md-nav__link">
|
|
Explanation
|
|
</a>
|
|
</li>
|
|
|
|
|
|
|
|
</ul>
|
|
</nav>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
|
|
|
|
|
|
<div class="md-sidebar md-sidebar--secondary" data-md-component="sidebar" data-md-type="toc" >
|
|
<div class="md-sidebar__scrollwrap">
|
|
<div class="md-sidebar__inner">
|
|
|
|
|
|
<nav class="md-nav md-nav--secondary" aria-label="Table of contents">
|
|
|
|
|
|
|
|
|
|
|
|
|
|
<label class="md-nav__title" for="__toc">
|
|
<span class="md-nav__icon md-icon"></span>
|
|
Table of contents
|
|
</label>
|
|
<ul class="md-nav__list" data-md-component="toc" data-md-scrollfix>
|
|
|
|
<li class="md-nav__item">
|
|
<a href="#nlp-tools" class="md-nav__link">
|
|
NLP tools
|
|
</a>
|
|
|
|
<nav class="md-nav" aria-label="NLP tools">
|
|
<ul class="md-nav__list">
|
|
|
|
<li class="md-nav__item">
|
|
<a href="#great_ai.utilities.clean.clean" class="md-nav__link">
|
|
clean()
|
|
</a>
|
|
|
|
</li>
|
|
|
|
<li class="md-nav__item">
|
|
<a href="#great_ai.utilities.get_sentences.get_sentences" class="md-nav__link">
|
|
get_sentences()
|
|
</a>
|
|
|
|
</li>
|
|
|
|
</ul>
|
|
</nav>
|
|
|
|
</li>
|
|
|
|
<li class="md-nav__item">
|
|
<a href="#parallel-processing" class="md-nav__link">
|
|
Parallel processing
|
|
</a>
|
|
|
|
<nav class="md-nav" aria-label="Parallel processing">
|
|
<ul class="md-nav__list">
|
|
|
|
<li class="md-nav__item">
|
|
<a href="#great_ai.utilities.simple_parallel_map" class="md-nav__link">
|
|
simple_parallel_map()
|
|
</a>
|
|
|
|
</li>
|
|
|
|
<li class="md-nav__item">
|
|
<a href="#great_ai.utilities.parallel_map.parallel_map.parallel_map" class="md-nav__link">
|
|
parallel_map()
|
|
</a>
|
|
|
|
</li>
|
|
|
|
<li class="md-nav__item">
|
|
<a href="#great_ai.utilities.threaded_parallel_map" class="md-nav__link">
|
|
threaded_parallel_map()
|
|
</a>
|
|
|
|
</li>
|
|
|
|
</ul>
|
|
</nav>
|
|
|
|
</li>
|
|
|
|
<li class="md-nav__item">
|
|
<a href="#composable-parallel-processing" class="md-nav__link">
|
|
Composable parallel processing
|
|
</a>
|
|
|
|
<nav class="md-nav" aria-label="Composable parallel processing">
|
|
<ul class="md-nav__list">
|
|
|
|
<li class="md-nav__item">
|
|
<a href="#great_ai.utilities.chunk.chunk" class="md-nav__link">
|
|
chunk()
|
|
</a>
|
|
|
|
</li>
|
|
|
|
<li class="md-nav__item">
|
|
<a href="#great_ai.utilities.unchunk.unchunk" class="md-nav__link">
|
|
unchunk()
|
|
</a>
|
|
|
|
</li>
|
|
|
|
</ul>
|
|
</nav>
|
|
|
|
</li>
|
|
|
|
<li class="md-nav__item">
|
|
<a href="#operations" class="md-nav__link">
|
|
Operations
|
|
</a>
|
|
|
|
<nav class="md-nav" aria-label="Operations">
|
|
<ul class="md-nav__list">
|
|
|
|
<li class="md-nav__item">
|
|
<a href="#great_ai.utilities.ConfigFile" class="md-nav__link">
|
|
ConfigFile
|
|
</a>
|
|
|
|
<nav class="md-nav" aria-label="ConfigFile">
|
|
<ul class="md-nav__list">
|
|
|
|
<li class="md-nav__item">
|
|
<a href="#great_ai.utilities.config_file.config_file.ConfigFile.__init__" class="md-nav__link">
|
|
__init__()
|
|
</a>
|
|
|
|
</li>
|
|
|
|
<li class="md-nav__item">
|
|
<a href="#great_ai.utilities.config_file.config_file.ConfigFile.path" class="md-nav__link">
|
|
path()
|
|
</a>
|
|
|
|
</li>
|
|
|
|
</ul>
|
|
</nav>
|
|
|
|
</li>
|
|
|
|
<li class="md-nav__item">
|
|
<a href="#great_ai.utilities.get_logger" class="md-nav__link">
|
|
get_logger()
|
|
</a>
|
|
|
|
</li>
|
|
|
|
</ul>
|
|
</nav>
|
|
|
|
</li>
|
|
|
|
</ul>
|
|
|
|
</nav>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
|
|
|
|
<div class="md-content" data-md-component="content">
|
|
<article class="md-content__inner md-typeset">
|
|
|
|
|
|
|
|
|
|
|
|
|
|
<a href="https://github.com/schmelczer/great-ai/edit/main/docs/reference/utilities.md" title="Edit this page" class="md-content__button md-icon">
|
|
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M20.71 7.04c.39-.39.39-1.04 0-1.41l-2.34-2.34c-.37-.39-1.02-.39-1.41 0l-1.84 1.83 3.75 3.75M3 17.25V21h3.75L17.81 9.93l-3.75-3.75L3 17.25Z"/></svg>
|
|
</a>
|
|
|
|
|
|
|
|
<h1 id="utilities">Utilities<a class="headerlink" href="#utilities" title="Permanent link">#</a></h1>
|
|
<div class="highlight"><pre><span></span><code><a id="__codelineno-0-1" name="__codelineno-0-1" href="#__codelineno-0-1"></a><span class="kn">from</span> <span class="nn">great_ai.utilities</span> <span class="kn">import</span> <span class="o">*</span>
|
|
</code></pre></div>
|
|
<h2 id="nlp-tools">NLP tools<a class="headerlink" href="#nlp-tools" title="Permanent link">#</a></h2>
|
|
<p>Well-tested tools that can be used in production with confidence. The toolbox of feature-extraction functions is expected to grow to cover other domains as well.</p>
|
|
|
|
|
|
<div class="doc doc-object doc-module">
|
|
|
|
|
|
|
|
<div class="doc doc-contents first">
|
|
|
|
|
|
|
|
<div class="doc doc-children">
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
<div class="doc doc-object doc-function">
|
|
|
|
|
|
|
|
<h3 id="great_ai.utilities.clean.clean" class="doc doc-heading">
|
|
<code class="highlight language-python"><span class="n">clean</span><span class="p">(</span><span class="n">text</span><span class="p">,</span> <span class="n">ignore_xml</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span> <span class="n">ignore_latex</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span> <span class="n">remove_brackets</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span> <span class="n">convert_to_ascii</span><span class="o">=</span><span class="kc">False</span><span class="p">)</span></code>
|
|
|
|
<a href="#great_ai.utilities.clean.clean" class="headerlink" title="Permanent link">#</a></h3>
|
|
|
|
|
|
<div class="doc doc-contents ">
|
|
|
|
<p>Clean all XML, LaTeX, PDF-extraction, and Unicode artifacts from the text.</p>
|
|
<p>The cleaning is quite heavy-weight and can be destructive. However, when working
|
|
with text, this is usually required to achieve sufficient cleanliness before further
|
|
processing.</p>
|
|
<p>Optionally, the text can be turned into ASCII. Carefully consider whether this is
|
|
absolutely needed for your use-case.</p>
|
|
|
|
<p><strong>Examples:</strong></p>
|
|
<div class="highlight"><pre><span></span><code><a id="__codelineno-0-1" name="__codelineno-0-1" href="#__codelineno-0-1"></a><span class="gp">>>> </span><span class="n">clean</span><span class="p">(</span><span class="s1">'<h2 color="red">Bj</span><span class="se">\\</span><span class="s1">"</span><span class="si">{o}</span><span class="s1">rn is </span><span class="se">\t</span><span class="s1"> </span><span class="se">\\</span><span class="s1">textit</span><span class="si">{happy}</span><span class="s1"> 🙂 &lt;3</h2>'</span><span class="p">)</span>
|
|
<a id="__codelineno-0-2" name="__codelineno-0-2" href="#__codelineno-0-2"></a><span class="go">'Björn is happy 🙂 <3'</span>
|
|
</code></pre></div>
|
|
<div class="highlight"><pre><span></span><code><a id="__codelineno-0-1" name="__codelineno-0-1" href="#__codelineno-0-1"></a><span class="gp">>>> </span><span class="n">clean</span><span class="p">(</span>
|
|
<a id="__codelineno-0-2" name="__codelineno-0-2" href="#__codelineno-0-2"></a><span class="gp">... </span> <span class="s1">'<h2 color="red">Bj</span><span class="se">\\</span><span class="s1">"</span><span class="si">{o}</span><span class="s1">rn is </span><span class="se">\t</span><span class="s1"> </span><span class="se">\\</span><span class="s1">textit</span><span class="si">{happy}</span><span class="s1"> 🙂 &lt;3</h2>'</span><span class="p">,</span>
|
|
<a id="__codelineno-0-3" name="__codelineno-0-3" href="#__codelineno-0-3"></a><span class="gp">... </span> <span class="n">convert_to_ascii</span><span class="o">=</span><span class="kc">True</span>
|
|
<a id="__codelineno-0-4" name="__codelineno-0-4" href="#__codelineno-0-4"></a><span class="gp">... </span><span class="p">)</span>
|
|
<a id="__codelineno-0-5" name="__codelineno-0-5" href="#__codelineno-0-5"></a><span class="go">'Bjorn is happy <3'</span>
|
|
</code></pre></div>
|
|
<div class="highlight"><pre><span></span><code><a id="__codelineno-0-1" name="__codelineno-0-1" href="#__codelineno-0-1"></a><span class="gp">>>> </span><span class="n">clean</span><span class="p">(</span>
|
|
<a id="__codelineno-0-2" name="__codelineno-0-2" href="#__codelineno-0-2"></a><span class="gp">... </span><span class="s1">'<h2 color="red">Bj</span><span class="se">\\</span><span class="s1">"</span><span class="si">{o}</span><span class="s1">rn is </span><span class="se">\t</span><span class="s1"> </span><span class="se">\\</span><span class="s1">textit</span><span class="si">{happy}</span><span class="s1"> 🙂 &lt;3</h2>'</span><span class="p">,</span>
|
|
<a id="__codelineno-0-3" name="__codelineno-0-3" href="#__codelineno-0-3"></a><span class="gp">... </span><span class="n">ignore_xml</span><span class="o">=</span><span class="kc">True</span>
|
|
<a id="__codelineno-0-4" name="__codelineno-0-4" href="#__codelineno-0-4"></a><span class="gp">... </span><span class="p">)</span>
|
|
<a id="__codelineno-0-5" name="__codelineno-0-5" href="#__codelineno-0-5"></a><span class="go">'<h2 color="red">Björn is happy 🙂 lt;3</h2>'</span>
|
|
</code></pre></div>
|
|
|
|
<p><strong>Parameters:</strong></p>
|
|
<table>
|
|
<thead>
|
|
<tr>
|
|
<th>Name</th>
|
|
<th>Type</th>
|
|
<th>Description</th>
|
|
<th>Default</th>
|
|
</tr>
|
|
</thead>
|
|
<tbody>
|
|
<tr>
|
|
<td><code>text</code></td>
|
|
<td>
|
|
<code>str</code>
|
|
</td>
|
|
<td><p>Text to be cleaned.</p></td>
|
|
<td>
|
|
<em>required</em>
|
|
</td>
|
|
</tr>
|
|
<tr>
|
|
<td><code>ignore_xml</code></td>
|
|
<td>
|
|
<code>bool</code>
|
|
</td>
|
|
<td><p>Do not process/remove XML-tags.</p></td>
|
|
<td>
|
|
<code>False</code>
|
|
</td>
|
|
</tr>
|
|
<tr>
|
|
<td><code>ignore_latex</code></td>
|
|
<td>
|
|
<code>bool</code>
|
|
</td>
|
|
<td><p>Do not process/remove LaTeX-tags.</p></td>
|
|
<td>
|
|
<code>False</code>
|
|
</td>
|
|
</tr>
|
|
<tr>
|
|
<td><code>remove_brackets</code></td>
|
|
<td>
|
|
<code>bool</code>
|
|
</td>
|
|
<td><p>Do not remove brackets ([])</p></td>
|
|
<td>
|
|
<code>False</code>
|
|
</td>
|
|
</tr>
|
|
<tr>
|
|
<td><code>convert_to_ascii</code></td>
|
|
<td>
|
|
<code>bool</code>
|
|
</td>
|
|
<td><p>Strip (or convert) non-ascii characters.</p></td>
|
|
<td>
|
|
<code>False</code>
|
|
</td>
|
|
</tr>
|
|
</tbody>
|
|
</table>
|
|
|
|
<p><strong>Returns:</strong></p>
|
|
<table>
|
|
<thead>
|
|
<tr>
|
|
<th>Type</th>
|
|
<th>Description</th>
|
|
</tr>
|
|
</thead>
|
|
<tbody>
|
|
<tr>
|
|
<td>
|
|
<code>str</code>
|
|
</td>
|
|
<td><p>The cleaned input text with sensibly collapsed whitespace and optionally no
|
|
markup.</p></td>
|
|
</tr>
|
|
</tbody>
|
|
</table>
|
|
|
|
<details class="quote">
|
|
<summary>Source code in <code>great_ai/utilities/clean.py</code></summary>
|
|
<div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"><a href="#__codelineno-0-19"> 19</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-20"> 20</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-21"> 21</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-22"> 22</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-23"> 23</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-24"> 24</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-25"> 25</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-26"> 26</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-27"> 27</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-28"> 28</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-29"> 29</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-30"> 30</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-31"> 31</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-32"> 32</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-33"> 33</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-34"> 34</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-35"> 35</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-36"> 36</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-37"> 37</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-38"> 38</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-39"> 39</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-40"> 40</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-41"> 41</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-42"> 42</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-43"> 43</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-44"> 44</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-45"> 45</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-46"> 46</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-47"> 47</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-48"> 48</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-49"> 49</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-50"> 50</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-51"> 51</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-52"> 52</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-53"> 53</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-54"> 54</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-55"> 55</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-56"> 56</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-57"> 57</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-58"> 58</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-59"> 59</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-60"> 60</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-61"> 61</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-62"> 62</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-63"> 63</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-64"> 64</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-65"> 65</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-66"> 66</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-67"> 67</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-68"> 68</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-69"> 69</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-70"> 70</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-71"> 71</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-72"> 72</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-73"> 73</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-74"> 74</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-75"> 75</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-76"> 76</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-77"> 77</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-78"> 78</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-79"> 79</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-80"> 80</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-81"> 81</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-82"> 82</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-83"> 83</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-84"> 84</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-85"> 85</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-86"> 86</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-87"> 87</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-88"> 88</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-89"> 89</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-90"> 90</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-91"> 91</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-92"> 92</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-93"> 93</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-94"> 94</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-95"> 95</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-96"> 96</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-97"> 97</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-98"> 98</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-99"> 99</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-100">100</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-101">101</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-102">102</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-103">103</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-104">104</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-105">105</a></span></pre></div></td><td class="code"><div><pre><span></span><code><a id="__codelineno-0-19" name="__codelineno-0-19"></a><span class="k">def</span> <span class="nf">clean</span><span class="p">(</span>
|
|
<a id="__codelineno-0-20" name="__codelineno-0-20"></a> <span class="n">text</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span>
|
|
<a id="__codelineno-0-21" name="__codelineno-0-21"></a> <span class="n">ignore_xml</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">False</span><span class="p">,</span>
|
|
<a id="__codelineno-0-22" name="__codelineno-0-22"></a> <span class="n">ignore_latex</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">False</span><span class="p">,</span>
|
|
<a id="__codelineno-0-23" name="__codelineno-0-23"></a> <span class="n">remove_brackets</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">False</span><span class="p">,</span>
|
|
<a id="__codelineno-0-24" name="__codelineno-0-24"></a> <span class="n">convert_to_ascii</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">False</span><span class="p">,</span>
|
|
<a id="__codelineno-0-25" name="__codelineno-0-25"></a><span class="p">)</span> <span class="o">-></span> <span class="nb">str</span><span class="p">:</span>
|
|
<a id="__codelineno-0-26" name="__codelineno-0-26"></a> <span class="sd">"""Clean all XML, LaTeX, PDF-extraction, and Unicode artifacts from the text.</span>
|
|
<a id="__codelineno-0-27" name="__codelineno-0-27"></a>
|
|
<a id="__codelineno-0-28" name="__codelineno-0-28"></a><span class="sd"> The cleaning is quite heavy-weight and can be destructive. However, when working</span>
|
|
<a id="__codelineno-0-29" name="__codelineno-0-29"></a><span class="sd"> with text, this is usually required to achieve sufficient cleanliness before further</span>
|
|
<a id="__codelineno-0-30" name="__codelineno-0-30"></a><span class="sd"> processing.</span>
|
|
<a id="__codelineno-0-31" name="__codelineno-0-31"></a>
|
|
<a id="__codelineno-0-32" name="__codelineno-0-32"></a><span class="sd"> Optionally, the text can be turned into ASCII. Carefully consider whether this is</span>
|
|
<a id="__codelineno-0-33" name="__codelineno-0-33"></a><span class="sd"> absolutely needed for your use-case.</span>
|
|
<a id="__codelineno-0-34" name="__codelineno-0-34"></a>
|
|
<a id="__codelineno-0-35" name="__codelineno-0-35"></a><span class="sd"> Examples:</span>
|
|
<a id="__codelineno-0-36" name="__codelineno-0-36"></a><span class="sd"> >>> clean('<h2 color="red">Bj\\\\"{o}rn is \\t \\\\textit{happy} 🙂 &lt;3</h2>')</span>
|
|
<a id="__codelineno-0-37" name="__codelineno-0-37"></a><span class="sd"> 'Björn is happy 🙂 <3'</span>
|
|
<a id="__codelineno-0-38" name="__codelineno-0-38"></a>
|
|
<a id="__codelineno-0-39" name="__codelineno-0-39"></a><span class="sd"> >>> clean(</span>
|
|
<a id="__codelineno-0-40" name="__codelineno-0-40"></a><span class="sd"> ... '<h2 color="red">Bj\\\\"{o}rn is \\t \\\\textit{happy} 🙂 &lt;3</h2>',</span>
|
|
<a id="__codelineno-0-41" name="__codelineno-0-41"></a><span class="sd"> ... convert_to_ascii=True</span>
|
|
<a id="__codelineno-0-42" name="__codelineno-0-42"></a><span class="sd"> ... )</span>
|
|
<a id="__codelineno-0-43" name="__codelineno-0-43"></a><span class="sd"> 'Bjorn is happy <3'</span>
|
|
<a id="__codelineno-0-44" name="__codelineno-0-44"></a>
|
|
<a id="__codelineno-0-45" name="__codelineno-0-45"></a><span class="sd"> >>> clean(</span>
|
|
<a id="__codelineno-0-46" name="__codelineno-0-46"></a><span class="sd"> ... '<h2 color="red">Bj\\\\"{o}rn is \\t \\\\textit{happy} 🙂 &lt;3</h2>',</span>
|
|
<a id="__codelineno-0-47" name="__codelineno-0-47"></a><span class="sd"> ... ignore_xml=True</span>
|
|
<a id="__codelineno-0-48" name="__codelineno-0-48"></a><span class="sd"> ... )</span>
|
|
<a id="__codelineno-0-49" name="__codelineno-0-49"></a><span class="sd"> '<h2 color="red">Björn is happy 🙂 lt;3</h2>'</span>
|
|
<a id="__codelineno-0-50" name="__codelineno-0-50"></a>
|
|
<a id="__codelineno-0-51" name="__codelineno-0-51"></a><span class="sd"> Args:</span>
|
|
<a id="__codelineno-0-52" name="__codelineno-0-52"></a><span class="sd"> text: Text to be cleaned.</span>
|
|
<a id="__codelineno-0-53" name="__codelineno-0-53"></a><span class="sd"> ignore_xml: Do not process/remove XML-tags.</span>
|
|
<a id="__codelineno-0-54" name="__codelineno-0-54"></a><span class="sd"> ignore_latex: Do not process/remove LaTeX-tags.</span>
|
|
<a id="__codelineno-0-55" name="__codelineno-0-55"></a><span class="sd"> remove_brackets: Do not remove brackets ([])</span>
|
|
<a id="__codelineno-0-56" name="__codelineno-0-56"></a><span class="sd"> convert_to_ascii: Strip (or convert) non-ascii characters.</span>
|
|
<a id="__codelineno-0-57" name="__codelineno-0-57"></a>
|
|
<a id="__codelineno-0-58" name="__codelineno-0-58"></a><span class="sd"> Returns:</span>
|
|
<a id="__codelineno-0-59" name="__codelineno-0-59"></a><span class="sd"> The cleaned input text with sensibly collapsed whitespace and optionally no</span>
|
|
<a id="__codelineno-0-60" name="__codelineno-0-60"></a><span class="sd"> markup.</span>
|
|
<a id="__codelineno-0-61" name="__codelineno-0-61"></a><span class="sd"> """</span>
|
|
<a id="__codelineno-0-62" name="__codelineno-0-62"></a>
|
|
<a id="__codelineno-0-63" name="__codelineno-0-63"></a> <span class="k">if</span> <span class="ow">not</span> <span class="n">ignore_xml</span><span class="p">:</span>
|
|
<a id="__codelineno-0-64" name="__codelineno-0-64"></a> <span class="n">text</span> <span class="o">=</span> <span class="n">re</span><span class="o">.</span><span class="n">sub</span><span class="p">(</span><span class="sa">r</span><span class="s2">"<[^>]*>"</span><span class="p">,</span> <span class="s2">" "</span><span class="p">,</span> <span class="n">text</span><span class="p">)</span>
|
|
<a id="__codelineno-0-65" name="__codelineno-0-65"></a> <span class="n">text</span> <span class="o">=</span> <span class="n">html</span><span class="o">.</span><span class="n">unescape</span><span class="p">(</span><span class="n">text</span><span class="p">)</span>
|
|
<a id="__codelineno-0-66" name="__codelineno-0-66"></a>
|
|
<a id="__codelineno-0-67" name="__codelineno-0-67"></a> <span class="k">if</span> <span class="ow">not</span> <span class="n">ignore_latex</span><span class="p">:</span>
|
|
<a id="__codelineno-0-68" name="__codelineno-0-68"></a> <span class="n">text</span> <span class="o">=</span> <span class="n">text</span><span class="o">.</span><span class="n">replace</span><span class="p">(</span><span class="s2">"%"</span><span class="p">,</span> <span class="s2">"</span><span class="se">\\</span><span class="s2">%"</span><span class="p">)</span> <span class="c1"># escape LaTeX comments before parsing as LaTeX</span>
|
|
<a id="__codelineno-0-69" name="__codelineno-0-69"></a>
|
|
<a id="__codelineno-0-70" name="__codelineno-0-70"></a> <span class="k">try</span><span class="p">:</span>
|
|
<a id="__codelineno-0-71" name="__codelineno-0-71"></a> <span class="n">text</span> <span class="o">=</span> <span class="n">latex</span><span class="o">.</span><span class="n">latex_to_text</span><span class="p">(</span><span class="n">text</span><span class="p">,</span> <span class="n">tolerant_parsing</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span> <span class="n">strict_braces</span><span class="o">=</span><span class="kc">False</span><span class="p">)</span>
|
|
<a id="__codelineno-0-72" name="__codelineno-0-72"></a> <span class="n">text</span> <span class="o">=</span> <span class="n">text</span><span class="o">.</span><span class="n">replace</span><span class="p">(</span><span class="s2">"</span><span class="si">%s</span><span class="s2">"</span><span class="p">,</span> <span class="s2">" "</span><span class="p">)</span>
|
|
<a id="__codelineno-0-73" name="__codelineno-0-73"></a> <span class="k">except</span><span class="p">:</span>
|
|
<a id="__codelineno-0-74" name="__codelineno-0-74"></a> <span class="n">logger</span><span class="o">.</span><span class="n">exception</span><span class="p">(</span><span class="s2">"Latex parsing error"</span><span class="p">)</span>
|
|
<a id="__codelineno-0-75" name="__codelineno-0-75"></a>
|
|
<a id="__codelineno-0-76" name="__codelineno-0-76"></a> <span class="k">if</span> <span class="n">convert_to_ascii</span><span class="p">:</span>
|
|
<a id="__codelineno-0-77" name="__codelineno-0-77"></a> <span class="n">text</span> <span class="o">=</span> <span class="n">unicodedata</span><span class="o">.</span><span class="n">normalize</span><span class="p">(</span><span class="s2">"NFKD"</span><span class="p">,</span> <span class="n">text</span><span class="p">)</span>
|
|
<a id="__codelineno-0-78" name="__codelineno-0-78"></a>
|
|
<a id="__codelineno-0-79" name="__codelineno-0-79"></a> <span class="k">try</span><span class="p">:</span>
|
|
<a id="__codelineno-0-80" name="__codelineno-0-80"></a> <span class="n">text</span><span class="o">.</span><span class="n">encode</span><span class="p">(</span><span class="s2">"ASCII"</span><span class="p">,</span> <span class="n">errors</span><span class="o">=</span><span class="s2">"strict"</span><span class="p">)</span>
|
|
<a id="__codelineno-0-81" name="__codelineno-0-81"></a> <span class="k">except</span> <span class="ne">UnicodeEncodeError</span><span class="p">:</span>
|
|
<a id="__codelineno-0-82" name="__codelineno-0-82"></a> <span class="n">text</span> <span class="o">=</span> <span class="s2">""</span><span class="o">.</span><span class="n">join</span><span class="p">([</span><span class="n">c</span> <span class="k">for</span> <span class="n">c</span> <span class="ow">in</span> <span class="n">text</span> <span class="k">if</span> <span class="ow">not</span> <span class="n">unicodedata</span><span class="o">.</span><span class="n">combining</span><span class="p">(</span><span class="n">c</span><span class="p">)])</span>
|
|
<a id="__codelineno-0-83" name="__codelineno-0-83"></a> <span class="n">text</span> <span class="o">=</span> <span class="n">unidecode</span><span class="o">.</span><span class="n">unidecode</span><span class="p">(</span><span class="n">text</span><span class="p">)</span>
|
|
<a id="__codelineno-0-84" name="__codelineno-0-84"></a>
|
|
<a id="__codelineno-0-85" name="__codelineno-0-85"></a> <span class="n">text</span> <span class="o">=</span> <span class="n">re</span><span class="o">.</span><span class="n">sub</span><span class="p">(</span>
|
|
<a id="__codelineno-0-86" name="__codelineno-0-86"></a> <span class="sa">r</span><span class="s2">"\b[a-zA-Z](?:[\t ]+[a-zA-Z]\b)+"</span><span class="p">,</span> <span class="k">lambda</span> <span class="n">m</span><span class="p">:</span> <span class="n">re</span><span class="o">.</span><span class="n">sub</span><span class="p">(</span><span class="sa">r</span><span class="s2">"[\t ]"</span><span class="p">,</span> <span class="s2">""</span><span class="p">,</span> <span class="n">m</span><span class="p">[</span><span class="mi">0</span><span class="p">]),</span> <span class="n">text</span>
|
|
<a id="__codelineno-0-87" name="__codelineno-0-87"></a> <span class="p">)</span> <span class="c1"># A R T I C L E => ARTICLE</span>
|
|
<a id="__codelineno-0-88" name="__codelineno-0-88"></a>
|
|
<a id="__codelineno-0-89" name="__codelineno-0-89"></a> <span class="k">if</span> <span class="n">remove_brackets</span><span class="p">:</span>
|
|
<a id="__codelineno-0-90" name="__codelineno-0-90"></a> <span class="n">text</span> <span class="o">=</span> <span class="n">re</span><span class="o">.</span><span class="n">sub</span><span class="p">(</span><span class="sa">r</span><span class="s2">"\[[^\]]*\]"</span><span class="p">,</span> <span class="s2">" "</span><span class="p">,</span> <span class="n">text</span><span class="p">)</span>
|
|
<a id="__codelineno-0-91" name="__codelineno-0-91"></a>
|
|
<a id="__codelineno-0-92" name="__codelineno-0-92"></a> <span class="c1"># fix hypens: break- word => break-word</span>
|
|
<a id="__codelineno-0-93" name="__codelineno-0-93"></a> <span class="n">text</span> <span class="o">=</span> <span class="n">re</span><span class="o">.</span><span class="n">sub</span><span class="p">(</span><span class="sa">r</span><span class="s2">"(\S)-\s+"</span><span class="p">,</span> <span class="sa">r</span><span class="s2">"\1-"</span><span class="p">,</span> <span class="n">text</span><span class="p">)</span>
|
|
<a id="__codelineno-0-94" name="__codelineno-0-94"></a> <span class="n">text</span> <span class="o">=</span> <span class="n">re</span><span class="o">.</span><span class="n">sub</span><span class="p">(</span><span class="sa">r</span><span class="s2">"\s+-(\S)"</span><span class="p">,</span> <span class="sa">r</span><span class="s2">"-\1"</span><span class="p">,</span> <span class="n">text</span><span class="p">)</span>
|
|
<a id="__codelineno-0-95" name="__codelineno-0-95"></a>
|
|
<a id="__codelineno-0-96" name="__codelineno-0-96"></a> <span class="c1"># collapse whitespace</span>
|
|
<a id="__codelineno-0-97" name="__codelineno-0-97"></a> <span class="n">text</span> <span class="o">=</span> <span class="n">re</span><span class="o">.</span><span class="n">sub</span><span class="p">(</span><span class="sa">r</span><span class="s2">"\s+"</span><span class="p">,</span> <span class="s2">" "</span><span class="p">,</span> <span class="n">text</span><span class="p">)</span>
|
|
<a id="__codelineno-0-98" name="__codelineno-0-98"></a>
|
|
<a id="__codelineno-0-99" name="__codelineno-0-99"></a> <span class="c1"># fix punctuation</span>
|
|
<a id="__codelineno-0-100" name="__codelineno-0-100"></a> <span class="n">text</span> <span class="o">=</span> <span class="n">re</span><span class="o">.</span><span class="n">sub</span><span class="p">(</span><span class="sa">rf</span><span class="s2">" ([</span><span class="si">{</span><span class="n">joined_left_punctuations</span><span class="si">}</span><span class="s2">])"</span><span class="p">,</span> <span class="sa">r</span><span class="s2">"\1"</span><span class="p">,</span> <span class="n">text</span><span class="p">)</span>
|
|
<a id="__codelineno-0-101" name="__codelineno-0-101"></a> <span class="n">text</span> <span class="o">=</span> <span class="n">re</span><span class="o">.</span><span class="n">sub</span><span class="p">(</span><span class="sa">rf</span><span class="s2">"([</span><span class="si">{</span><span class="n">joined_right_punctuations</span><span class="si">}</span><span class="s2">]) "</span><span class="p">,</span> <span class="sa">r</span><span class="s2">"\1"</span><span class="p">,</span> <span class="n">text</span><span class="p">)</span>
|
|
<a id="__codelineno-0-102" name="__codelineno-0-102"></a>
|
|
<a id="__codelineno-0-103" name="__codelineno-0-103"></a> <span class="n">text</span> <span class="o">=</span> <span class="n">text</span><span class="o">.</span><span class="n">strip</span><span class="p">()</span>
|
|
<a id="__codelineno-0-104" name="__codelineno-0-104"></a>
|
|
<a id="__codelineno-0-105" name="__codelineno-0-105"></a> <span class="k">return</span> <span class="n">text</span>
|
|
</code></pre></div></td></tr></table></div>
|
|
</details>
|
|
</div>
|
|
|
|
</div>
|
|
|
|
|
|
|
|
</div>
|
|
|
|
</div>
|
|
|
|
</div>
|
|
|
|
<div class="doc doc-object doc-module">
|
|
|
|
|
|
|
|
<div class="doc doc-contents first">
|
|
|
|
|
|
|
|
<div class="doc doc-children">
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
<div class="doc doc-object doc-function">
|
|
|
|
|
|
|
|
<h3 id="great_ai.utilities.get_sentences.get_sentences" class="doc doc-heading">
|
|
<code class="highlight language-python"><span class="n">get_sentences</span><span class="p">(</span><span class="n">text</span><span class="p">,</span> <span class="n">ignore_partial</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span> <span class="n">true_case</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span> <span class="n">remove_punctuation</span><span class="o">=</span><span class="kc">False</span><span class="p">)</span></code>
|
|
|
|
<a href="#great_ai.utilities.get_sentences.get_sentences" class="headerlink" title="Permanent link">#</a></h3>
|
|
|
|
|
|
<div class="doc doc-contents ">
|
|
|
|
<p>Return the list of sentences found in the input text.</p>
|
|
<p>Use <a href="https://github.com/fnl/syntok">syntok</a> to segment the sentences. Further
|
|
processing can be enabled with optional arguments.</p>
|
|
|
|
<p><strong>Examples:</strong></p>
|
|
<div class="highlight"><pre><span></span><code><a id="__codelineno-0-1" name="__codelineno-0-1" href="#__codelineno-0-1"></a><span class="gp">>>> </span><span class="n">get_sentences</span><span class="p">(</span><span class="s1">'This is a sentence. This is a half'</span><span class="p">)</span>
|
|
<a id="__codelineno-0-2" name="__codelineno-0-2" href="#__codelineno-0-2"></a><span class="go">['This is a sentence.', 'This is a half']</span>
|
|
</code></pre></div>
|
|
<div class="highlight"><pre><span></span><code><a id="__codelineno-0-1" name="__codelineno-0-1" href="#__codelineno-0-1"></a><span class="gp">>>> </span><span class="n">get_sentences</span><span class="p">(</span><span class="s1">'This is a sentence. This is a half'</span><span class="p">,</span> <span class="n">ignore_partial</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
|
|
<a id="__codelineno-0-2" name="__codelineno-0-2" href="#__codelineno-0-2"></a><span class="go">['This is a sentence.']</span>
|
|
</code></pre></div>
|
|
<div class="highlight"><pre><span></span><code><a id="__codelineno-0-1" name="__codelineno-0-1" href="#__codelineno-0-1"></a><span class="gp">>>> </span><span class="n">get_sentences</span><span class="p">(</span><span class="s1">'I like Apple.'</span><span class="p">,</span> <span class="n">true_case</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span> <span class="n">remove_punctuation</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
|
|
<a id="__codelineno-0-2" name="__codelineno-0-2" href="#__codelineno-0-2"></a><span class="go">['i like Apple']</span>
|
|
</code></pre></div>
|
|
|
|
<p><strong>Parameters:</strong></p>
|
|
<table>
|
|
<thead>
|
|
<tr>
|
|
<th>Name</th>
|
|
<th>Type</th>
|
|
<th>Description</th>
|
|
<th>Default</th>
|
|
</tr>
|
|
</thead>
|
|
<tbody>
|
|
<tr>
|
|
<td><code>text</code></td>
|
|
<td>
|
|
<code>str</code>
|
|
</td>
|
|
<td><p>Text to be segmented into sentences.</p></td>
|
|
<td>
|
|
<em>required</em>
|
|
</td>
|
|
</tr>
|
|
<tr>
|
|
<td><code>ignore_partial</code></td>
|
|
<td>
|
|
<code>bool</code>
|
|
</td>
|
|
<td><p>Filter out sentences that are not capitalised/don't end with a</p></td>
|
|
<td>
|
|
<code>False</code>
|
|
</td>
|
|
</tr>
|
|
<tr>
|
|
<td><code>true_case</code></td>
|
|
<td>
|
|
<code>bool</code>
|
|
</td>
|
|
<td><p>Crude method: lowercase the first word of each sentence.</p></td>
|
|
<td>
|
|
<code>False</code>
|
|
</td>
|
|
</tr>
|
|
<tr>
|
|
<td><code>remove_punctuation</code></td>
|
|
<td>
|
|
<code>bool</code>
|
|
</td>
|
|
<td><p>Remove all kinds of punctuation.</p></td>
|
|
<td>
|
|
<code>False</code>
|
|
</td>
|
|
</tr>
|
|
</tbody>
|
|
</table>
|
|
|
|
<p><strong>Returns:</strong></p>
|
|
<table>
|
|
<thead>
|
|
<tr>
|
|
<th>Type</th>
|
|
<th>Description</th>
|
|
</tr>
|
|
</thead>
|
|
<tbody>
|
|
<tr>
|
|
<td>
|
|
<code><span title="typing.List">List</span>[str]</code>
|
|
</td>
|
|
<td><p>The found sentences (with partial sentences optionally filtered out).</p></td>
|
|
</tr>
|
|
</tbody>
|
|
</table>
|
|
|
|
<details class="quote">
|
|
<summary>Source code in <code>great_ai/utilities/get_sentences.py</code></summary>
|
|
<div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"><a href="#__codelineno-0-13">13</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-14">14</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-15">15</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-16">16</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-17">17</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-18">18</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-19">19</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-20">20</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-21">21</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-22">22</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-23">23</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-24">24</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-25">25</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-26">26</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-27">27</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-28">28</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-29">29</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-30">30</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-31">31</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-32">32</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-33">33</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-34">34</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-35">35</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-36">36</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-37">37</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-38">38</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-39">39</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-40">40</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-41">41</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-42">42</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-43">43</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-44">44</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-45">45</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-46">46</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-47">47</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-48">48</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-49">49</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-50">50</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-51">51</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-52">52</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-53">53</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-54">54</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-55">55</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-56">56</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-57">57</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-58">58</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-59">59</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-60">60</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-61">61</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-62">62</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-63">63</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-64">64</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-65">65</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-66">66</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-67">67</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-68">68</a></span></pre></div></td><td class="code"><div><pre><span></span><code><a id="__codelineno-0-13" name="__codelineno-0-13"></a><span class="k">def</span> <span class="nf">get_sentences</span><span class="p">(</span>
|
|
<a id="__codelineno-0-14" name="__codelineno-0-14"></a> <span class="n">text</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span>
|
|
<a id="__codelineno-0-15" name="__codelineno-0-15"></a> <span class="n">ignore_partial</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">False</span><span class="p">,</span>
|
|
<a id="__codelineno-0-16" name="__codelineno-0-16"></a> <span class="n">true_case</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">False</span><span class="p">,</span>
|
|
<a id="__codelineno-0-17" name="__codelineno-0-17"></a> <span class="n">remove_punctuation</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">False</span><span class="p">,</span>
|
|
<a id="__codelineno-0-18" name="__codelineno-0-18"></a><span class="p">)</span> <span class="o">-></span> <span class="n">List</span><span class="p">[</span><span class="nb">str</span><span class="p">]:</span>
|
|
<a id="__codelineno-0-19" name="__codelineno-0-19"></a> <span class="sd">"""Return the list of sentences found in the input text.</span>
|
|
<a id="__codelineno-0-20" name="__codelineno-0-20"></a>
|
|
<a id="__codelineno-0-21" name="__codelineno-0-21"></a><span class="sd"> Use [syntok](https://github.com/fnl/syntok) to segment the sentences. Further</span>
|
|
<a id="__codelineno-0-22" name="__codelineno-0-22"></a><span class="sd"> processing can be enabled with optional arguments.</span>
|
|
<a id="__codelineno-0-23" name="__codelineno-0-23"></a>
|
|
<a id="__codelineno-0-24" name="__codelineno-0-24"></a><span class="sd"> Examples:</span>
|
|
<a id="__codelineno-0-25" name="__codelineno-0-25"></a><span class="sd"> >>> get_sentences('This is a sentence. This is a half')</span>
|
|
<a id="__codelineno-0-26" name="__codelineno-0-26"></a><span class="sd"> ['This is a sentence.', 'This is a half']</span>
|
|
<a id="__codelineno-0-27" name="__codelineno-0-27"></a>
|
|
<a id="__codelineno-0-28" name="__codelineno-0-28"></a><span class="sd"> >>> get_sentences('This is a sentence. This is a half', ignore_partial=True)</span>
|
|
<a id="__codelineno-0-29" name="__codelineno-0-29"></a><span class="sd"> ['This is a sentence.']</span>
|
|
<a id="__codelineno-0-30" name="__codelineno-0-30"></a>
|
|
<a id="__codelineno-0-31" name="__codelineno-0-31"></a><span class="sd"> >>> get_sentences('I like Apple.', true_case=True, remove_punctuation=True)</span>
|
|
<a id="__codelineno-0-32" name="__codelineno-0-32"></a><span class="sd"> ['i like Apple']</span>
|
|
<a id="__codelineno-0-33" name="__codelineno-0-33"></a>
|
|
<a id="__codelineno-0-34" name="__codelineno-0-34"></a><span class="sd"> Args:</span>
|
|
<a id="__codelineno-0-35" name="__codelineno-0-35"></a><span class="sd"> text: Text to be segmented into sentences.</span>
|
|
<a id="__codelineno-0-36" name="__codelineno-0-36"></a><span class="sd"> ignore_partial: Filter out sentences that are not capitalised/don't end with a</span>
|
|
<a id="__codelineno-0-37" name="__codelineno-0-37"></a><span class="sd"> punctuation.</span>
|
|
<a id="__codelineno-0-38" name="__codelineno-0-38"></a><span class="sd"> true_case: Crude method: lowercase the first word of each sentence.</span>
|
|
<a id="__codelineno-0-39" name="__codelineno-0-39"></a><span class="sd"> remove_punctuation: Remove all kinds of punctuation.</span>
|
|
<a id="__codelineno-0-40" name="__codelineno-0-40"></a>
|
|
<a id="__codelineno-0-41" name="__codelineno-0-41"></a><span class="sd"> Returns:</span>
|
|
<a id="__codelineno-0-42" name="__codelineno-0-42"></a><span class="sd"> The found sentences (with partial sentences optionally filtered out).</span>
|
|
<a id="__codelineno-0-43" name="__codelineno-0-43"></a><span class="sd"> """</span>
|
|
<a id="__codelineno-0-44" name="__codelineno-0-44"></a>
|
|
<a id="__codelineno-0-45" name="__codelineno-0-45"></a> <span class="n">tokenizer</span> <span class="o">=</span> <span class="n">Tokenizer</span><span class="p">(</span>
|
|
<a id="__codelineno-0-46" name="__codelineno-0-46"></a> <span class="n">emit_hyphen_or_underscore_sep</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span> <span class="n">replace_not_contraction</span><span class="o">=</span><span class="kc">False</span>
|
|
<a id="__codelineno-0-47" name="__codelineno-0-47"></a> <span class="p">)</span>
|
|
<a id="__codelineno-0-48" name="__codelineno-0-48"></a> <span class="n">token_stream</span> <span class="o">=</span> <span class="n">tokenizer</span><span class="o">.</span><span class="n">tokenize</span><span class="p">(</span><span class="n">text</span><span class="p">)</span>
|
|
<a id="__codelineno-0-49" name="__codelineno-0-49"></a>
|
|
<a id="__codelineno-0-50" name="__codelineno-0-50"></a> <span class="k">def</span> <span class="nf">process</span><span class="p">(</span><span class="n">sentence</span><span class="p">:</span> <span class="nb">str</span><span class="p">)</span> <span class="o">-></span> <span class="nb">str</span><span class="p">:</span>
|
|
<a id="__codelineno-0-51" name="__codelineno-0-51"></a> <span class="k">if</span> <span class="n">true_case</span><span class="p">:</span>
|
|
<a id="__codelineno-0-52" name="__codelineno-0-52"></a> <span class="n">sentence</span> <span class="o">=</span> <span class="n">sentence</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span><span class="o">.</span><span class="n">lower</span><span class="p">()</span> <span class="o">+</span> <span class="n">sentence</span><span class="p">[</span><span class="mi">1</span><span class="p">:]</span> <span class="c1"># very crude method</span>
|
|
<a id="__codelineno-0-53" name="__codelineno-0-53"></a> <span class="k">if</span> <span class="n">remove_punctuation</span><span class="p">:</span>
|
|
<a id="__codelineno-0-54" name="__codelineno-0-54"></a> <span class="n">sentence</span> <span class="o">=</span> <span class="n">re</span><span class="o">.</span><span class="n">sub</span><span class="p">(</span><span class="n">punctuations_pattern</span><span class="p">,</span> <span class="s2">" "</span><span class="p">,</span> <span class="n">sentence</span><span class="p">)</span>
|
|
<a id="__codelineno-0-55" name="__codelineno-0-55"></a> <span class="k">return</span> <span class="n">sentence</span><span class="o">.</span><span class="n">strip</span><span class="p">()</span>
|
|
<a id="__codelineno-0-56" name="__codelineno-0-56"></a>
|
|
<a id="__codelineno-0-57" name="__codelineno-0-57"></a> <span class="n">sentences</span> <span class="o">=</span> <span class="p">[</span>
|
|
<a id="__codelineno-0-58" name="__codelineno-0-58"></a> <span class="n">process</span><span class="p">(</span><span class="n">tokenizer</span><span class="o">.</span><span class="n">to_text</span><span class="p">(</span><span class="n">sentence</span><span class="p">))</span> <span class="k">for</span> <span class="n">sentence</span> <span class="ow">in</span> <span class="n">segment</span><span class="p">(</span><span class="n">token_stream</span><span class="p">)</span>
|
|
<a id="__codelineno-0-59" name="__codelineno-0-59"></a> <span class="p">]</span>
|
|
<a id="__codelineno-0-60" name="__codelineno-0-60"></a>
|
|
<a id="__codelineno-0-61" name="__codelineno-0-61"></a> <span class="k">if</span> <span class="n">ignore_partial</span><span class="p">:</span>
|
|
<a id="__codelineno-0-62" name="__codelineno-0-62"></a> <span class="n">sentences</span> <span class="o">=</span> <span class="p">[</span>
|
|
<a id="__codelineno-0-63" name="__codelineno-0-63"></a> <span class="n">sentence</span>
|
|
<a id="__codelineno-0-64" name="__codelineno-0-64"></a> <span class="k">for</span> <span class="n">sentence</span> <span class="ow">in</span> <span class="n">sentences</span>
|
|
<a id="__codelineno-0-65" name="__codelineno-0-65"></a> <span class="k">if</span> <span class="n">sentence</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span><span class="o">.</span><span class="n">isupper</span><span class="p">()</span> <span class="ow">and</span> <span class="n">sentence</span><span class="p">[</span><span class="o">-</span><span class="mi">1</span><span class="p">]</span> <span class="ow">in</span> <span class="n">sentence_ending_punctuations</span>
|
|
<a id="__codelineno-0-66" name="__codelineno-0-66"></a> <span class="p">]</span>
|
|
<a id="__codelineno-0-67" name="__codelineno-0-67"></a>
|
|
<a id="__codelineno-0-68" name="__codelineno-0-68"></a> <span class="k">return</span> <span class="n">sentences</span>
|
|
</code></pre></div></td></tr></table></div>
|
|
</details>
|
|
</div>
|
|
|
|
</div>
|
|
|
|
|
|
|
|
</div>
|
|
|
|
</div>
|
|
|
|
</div>
|
|
|
|
<div class="doc doc-object doc-function">
|
|
|
|
|
|
|
|
<div class="doc doc-contents first">
|
|
|
|
<p>Predict the language code from text.</p>
|
|
<p>A thin wrapper over <a href="https://github.com/rspeer/langcodes">langcodes</a> for convenient
|
|
language tagging.</p>
|
|
|
|
<p><strong>Examples:</strong></p>
|
|
<div class="highlight"><pre><span></span><code><a id="__codelineno-0-1" name="__codelineno-0-1" href="#__codelineno-0-1"></a><span class="gp">>>> </span><span class="n">predict_language</span><span class="p">(</span><span class="s1">'This is a sentence.'</span><span class="p">)</span>
|
|
<a id="__codelineno-0-2" name="__codelineno-0-2" href="#__codelineno-0-2"></a><span class="go">'en'</span>
|
|
</code></pre></div>
|
|
|
|
<p><strong>Parameters:</strong></p>
|
|
<table>
|
|
<thead>
|
|
<tr>
|
|
<th>Name</th>
|
|
<th>Type</th>
|
|
<th>Description</th>
|
|
<th>Default</th>
|
|
</tr>
|
|
</thead>
|
|
<tbody>
|
|
<tr>
|
|
<td><code>text</code></td>
|
|
<td>
|
|
<code><span title="typing.Optional">Optional</span>[str]</code>
|
|
</td>
|
|
<td><p>Text used for prediction.</p></td>
|
|
<td>
|
|
<em>required</em>
|
|
</td>
|
|
</tr>
|
|
</tbody>
|
|
</table>
|
|
|
|
<p><strong>Returns:</strong></p>
|
|
<table>
|
|
<thead>
|
|
<tr>
|
|
<th>Type</th>
|
|
<th>Description</th>
|
|
</tr>
|
|
</thead>
|
|
<tbody>
|
|
<tr>
|
|
<td>
|
|
<code>str</code>
|
|
</td>
|
|
<td><p>The predicted language code (en, en-US) or <code>und</code> if a prediction could not be
|
|
made.</p></td>
|
|
</tr>
|
|
</tbody>
|
|
</table>
|
|
|
|
<details class="quote">
|
|
<summary>Source code in <code>great_ai/utilities/language/predict_language.py</code></summary>
|
|
<div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"><a href="#__codelineno-0-7"> 7</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-8"> 8</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-9"> 9</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-10">10</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-11">11</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-12">12</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-13">13</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-14">14</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-15">15</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-16">16</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-17">17</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-18">18</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-19">19</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-20">20</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-21">21</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-22">22</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-23">23</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-24">24</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-25">25</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-26">26</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-27">27</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-28">28</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-29">29</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-30">30</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-31">31</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-32">32</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-33">33</a></span></pre></div></td><td class="code"><div><pre><span></span><code><a id="__codelineno-0-7" name="__codelineno-0-7"></a><span class="k">def</span> <span class="nf">predict_language</span><span class="p">(</span><span class="n">text</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">str</span><span class="p">])</span> <span class="o">-></span> <span class="nb">str</span><span class="p">:</span>
|
|
<a id="__codelineno-0-8" name="__codelineno-0-8"></a> <span class="sd">"""Predict the language code from text.</span>
|
|
<a id="__codelineno-0-9" name="__codelineno-0-9"></a>
|
|
<a id="__codelineno-0-10" name="__codelineno-0-10"></a><span class="sd"> A thin wrapper over [langcodes](https://github.com/rspeer/langcodes) for convenient</span>
|
|
<a id="__codelineno-0-11" name="__codelineno-0-11"></a><span class="sd"> language tagging.</span>
|
|
<a id="__codelineno-0-12" name="__codelineno-0-12"></a>
|
|
<a id="__codelineno-0-13" name="__codelineno-0-13"></a><span class="sd"> Examples:</span>
|
|
<a id="__codelineno-0-14" name="__codelineno-0-14"></a><span class="sd"> >>> predict_language('This is a sentence.')</span>
|
|
<a id="__codelineno-0-15" name="__codelineno-0-15"></a><span class="sd"> 'en'</span>
|
|
<a id="__codelineno-0-16" name="__codelineno-0-16"></a>
|
|
<a id="__codelineno-0-17" name="__codelineno-0-17"></a><span class="sd"> Args:</span>
|
|
<a id="__codelineno-0-18" name="__codelineno-0-18"></a><span class="sd"> text: Text used for prediction.</span>
|
|
<a id="__codelineno-0-19" name="__codelineno-0-19"></a>
|
|
<a id="__codelineno-0-20" name="__codelineno-0-20"></a><span class="sd"> Returns:</span>
|
|
<a id="__codelineno-0-21" name="__codelineno-0-21"></a><span class="sd"> The predicted language code (en, en-US) or `und` if a prediction could not be</span>
|
|
<a id="__codelineno-0-22" name="__codelineno-0-22"></a><span class="sd"> made.</span>
|
|
<a id="__codelineno-0-23" name="__codelineno-0-23"></a><span class="sd"> """</span>
|
|
<a id="__codelineno-0-24" name="__codelineno-0-24"></a>
|
|
<a id="__codelineno-0-25" name="__codelineno-0-25"></a> <span class="k">if</span> <span class="ow">not</span> <span class="n">text</span><span class="p">:</span>
|
|
<a id="__codelineno-0-26" name="__codelineno-0-26"></a> <span class="k">return</span> <span class="n">Language</span><span class="o">.</span><span class="n">make</span><span class="p">()</span><span class="o">.</span><span class="n">to_tag</span><span class="p">()</span>
|
|
<a id="__codelineno-0-27" name="__codelineno-0-27"></a>
|
|
<a id="__codelineno-0-28" name="__codelineno-0-28"></a> <span class="k">try</span><span class="p">:</span>
|
|
<a id="__codelineno-0-29" name="__codelineno-0-29"></a> <span class="n">language_code</span> <span class="o">=</span> <span class="n">detect</span><span class="p">(</span><span class="n">text</span><span class="p">)</span>
|
|
<a id="__codelineno-0-30" name="__codelineno-0-30"></a> <span class="k">except</span> <span class="n">LangDetectException</span><span class="p">:</span>
|
|
<a id="__codelineno-0-31" name="__codelineno-0-31"></a> <span class="k">return</span> <span class="n">Language</span><span class="o">.</span><span class="n">make</span><span class="p">()</span><span class="o">.</span><span class="n">to_tag</span><span class="p">()</span>
|
|
<a id="__codelineno-0-32" name="__codelineno-0-32"></a>
|
|
<a id="__codelineno-0-33" name="__codelineno-0-33"></a> <span class="k">return</span> <span class="n">Language</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">language_code</span><span class="p">)</span><span class="o">.</span><span class="n">to_tag</span><span class="p">()</span>
|
|
</code></pre></div></td></tr></table></div>
|
|
</details>
|
|
</div>
|
|
|
|
</div>
|
|
|
|
<div class="doc doc-object doc-function">
|
|
|
|
|
|
|
|
<div class="doc doc-contents first">
|
|
|
|
<p>Decide whether the <code>language_code</code> is of an English language.</p>
|
|
<p>A thin wrapper over <a href="https://github.com/rspeer/langcodes">langcodes</a> for convenient
|
|
language tagging.</p>
|
|
|
|
<p><strong>Examples:</strong></p>
|
|
<div class="highlight"><pre><span></span><code><a id="__codelineno-0-1" name="__codelineno-0-1" href="#__codelineno-0-1"></a><span class="gp">>>> </span><span class="n">is_english</span><span class="p">(</span><span class="s1">'en-US'</span><span class="p">)</span>
|
|
<a id="__codelineno-0-2" name="__codelineno-0-2" href="#__codelineno-0-2"></a><span class="go">True</span>
|
|
</code></pre></div>
|
|
<div class="highlight"><pre><span></span><code><a id="__codelineno-0-1" name="__codelineno-0-1" href="#__codelineno-0-1"></a><span class="gp">>>> </span><span class="n">is_english</span><span class="p">(</span><span class="kc">None</span><span class="p">)</span>
|
|
<a id="__codelineno-0-2" name="__codelineno-0-2" href="#__codelineno-0-2"></a><span class="go">False</span>
|
|
</code></pre></div>
|
|
<div class="highlight"><pre><span></span><code><a id="__codelineno-0-1" name="__codelineno-0-1" href="#__codelineno-0-1"></a><span class="gp">>>> </span><span class="n">is_english</span><span class="p">(</span><span class="s1">'und'</span><span class="p">)</span>
|
|
<a id="__codelineno-0-2" name="__codelineno-0-2" href="#__codelineno-0-2"></a><span class="go">False</span>
|
|
</code></pre></div>
|
|
|
|
<p><strong>Parameters:</strong></p>
|
|
<table>
|
|
<thead>
|
|
<tr>
|
|
<th>Name</th>
|
|
<th>Type</th>
|
|
<th>Description</th>
|
|
<th>Default</th>
|
|
</tr>
|
|
</thead>
|
|
<tbody>
|
|
<tr>
|
|
<td><code>language_code</code></td>
|
|
<td>
|
|
<code><span title="typing.Optional">Optional</span>[str]</code>
|
|
</td>
|
|
<td><p>Language code, for example, returned by
|
|
`[great_ai.utilities.predict_language][].</p></td>
|
|
<td>
|
|
<em>required</em>
|
|
</td>
|
|
</tr>
|
|
</tbody>
|
|
</table>
|
|
|
|
<p><strong>Returns:</strong></p>
|
|
<table>
|
|
<thead>
|
|
<tr>
|
|
<th>Type</th>
|
|
<th>Description</th>
|
|
</tr>
|
|
</thead>
|
|
<tbody>
|
|
<tr>
|
|
<td>
|
|
<code>bool</code>
|
|
</td>
|
|
<td><p>Boolean indicating whether it's English.</p></td>
|
|
</tr>
|
|
</tbody>
|
|
</table>
|
|
|
|
<details class="quote">
|
|
<summary>Source code in <code>great_ai/utilities/language/is_english.py</code></summary>
|
|
<div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"><a href="#__codelineno-0-6"> 6</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-7"> 7</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-8"> 8</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-9"> 9</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-10">10</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-11">11</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-12">12</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-13">13</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-14">14</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-15">15</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-16">16</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-17">17</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-18">18</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-19">19</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-20">20</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-21">21</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-22">22</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-23">23</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-24">24</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-25">25</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-26">26</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-27">27</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-28">28</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-29">29</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-30">30</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-31">31</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-32">32</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-33">33</a></span></pre></div></td><td class="code"><div><pre><span></span><code><a id="__codelineno-0-6" name="__codelineno-0-6"></a><span class="k">def</span> <span class="nf">is_english</span><span class="p">(</span><span class="n">language_code</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">str</span><span class="p">])</span> <span class="o">-></span> <span class="nb">bool</span><span class="p">:</span>
|
|
<a id="__codelineno-0-7" name="__codelineno-0-7"></a> <span class="sd">"""Decide whether the `language_code` is of an English language.</span>
|
|
<a id="__codelineno-0-8" name="__codelineno-0-8"></a>
|
|
<a id="__codelineno-0-9" name="__codelineno-0-9"></a><span class="sd"> A thin wrapper over [langcodes](https://github.com/rspeer/langcodes) for convenient</span>
|
|
<a id="__codelineno-0-10" name="__codelineno-0-10"></a><span class="sd"> language tagging.</span>
|
|
<a id="__codelineno-0-11" name="__codelineno-0-11"></a>
|
|
<a id="__codelineno-0-12" name="__codelineno-0-12"></a><span class="sd"> Examples:</span>
|
|
<a id="__codelineno-0-13" name="__codelineno-0-13"></a><span class="sd"> >>> is_english('en-US')</span>
|
|
<a id="__codelineno-0-14" name="__codelineno-0-14"></a><span class="sd"> True</span>
|
|
<a id="__codelineno-0-15" name="__codelineno-0-15"></a>
|
|
<a id="__codelineno-0-16" name="__codelineno-0-16"></a><span class="sd"> >>> is_english(None)</span>
|
|
<a id="__codelineno-0-17" name="__codelineno-0-17"></a><span class="sd"> False</span>
|
|
<a id="__codelineno-0-18" name="__codelineno-0-18"></a>
|
|
<a id="__codelineno-0-19" name="__codelineno-0-19"></a><span class="sd"> >>> is_english('und')</span>
|
|
<a id="__codelineno-0-20" name="__codelineno-0-20"></a><span class="sd"> False</span>
|
|
<a id="__codelineno-0-21" name="__codelineno-0-21"></a>
|
|
<a id="__codelineno-0-22" name="__codelineno-0-22"></a><span class="sd"> Args:</span>
|
|
<a id="__codelineno-0-23" name="__codelineno-0-23"></a><span class="sd"> language_code: Language code, for example, returned by</span>
|
|
<a id="__codelineno-0-24" name="__codelineno-0-24"></a><span class="sd"> `[great_ai.utilities.predict_language][].</span>
|
|
<a id="__codelineno-0-25" name="__codelineno-0-25"></a>
|
|
<a id="__codelineno-0-26" name="__codelineno-0-26"></a><span class="sd"> Returns:</span>
|
|
<a id="__codelineno-0-27" name="__codelineno-0-27"></a><span class="sd"> Boolean indicating whether it's English.</span>
|
|
<a id="__codelineno-0-28" name="__codelineno-0-28"></a><span class="sd"> """</span>
|
|
<a id="__codelineno-0-29" name="__codelineno-0-29"></a> <span class="k">if</span> <span class="ow">not</span> <span class="n">language_code</span><span class="p">:</span>
|
|
<a id="__codelineno-0-30" name="__codelineno-0-30"></a> <span class="n">language_code</span> <span class="o">=</span> <span class="s2">"und"</span>
|
|
<a id="__codelineno-0-31" name="__codelineno-0-31"></a>
|
|
<a id="__codelineno-0-32" name="__codelineno-0-32"></a> <span class="n">language_code</span> <span class="o">=</span> <span class="n">standardize_tag</span><span class="p">(</span><span class="n">language_code</span><span class="p">)</span>
|
|
<a id="__codelineno-0-33" name="__codelineno-0-33"></a> <span class="k">return</span> <span class="n">tag_distance</span><span class="p">(</span><span class="n">language_code</span><span class="p">,</span> <span class="s2">"en"</span><span class="p">)</span> <span class="o"><</span> <span class="mi">15</span>
|
|
</code></pre></div></td></tr></table></div>
|
|
</details>
|
|
</div>
|
|
|
|
</div>
|
|
|
|
<div class="doc doc-object doc-function">
|
|
|
|
|
|
|
|
<div class="doc doc-contents first">
|
|
|
|
<p>Human-friendly English name of language from its <code>language_code</code>.</p>
|
|
<p>A thin wrapper over <a href="https://github.com/rspeer/langcodes">langcodes</a> for convenient
|
|
language tagging.</p>
|
|
|
|
<p><strong>Examples:</strong></p>
|
|
<div class="highlight"><pre><span></span><code><a id="__codelineno-0-1" name="__codelineno-0-1" href="#__codelineno-0-1"></a><span class="gp">>>> </span><span class="n">english_name_of_language</span><span class="p">(</span><span class="s1">'en-US'</span><span class="p">)</span>
|
|
<a id="__codelineno-0-2" name="__codelineno-0-2" href="#__codelineno-0-2"></a><span class="go">'English (United States)'</span>
|
|
</code></pre></div>
|
|
<div class="highlight"><pre><span></span><code><a id="__codelineno-0-1" name="__codelineno-0-1" href="#__codelineno-0-1"></a><span class="gp">>>> </span><span class="n">english_name_of_language</span><span class="p">(</span><span class="s1">'und'</span><span class="p">)</span>
|
|
<a id="__codelineno-0-2" name="__codelineno-0-2" href="#__codelineno-0-2"></a><span class="go">'Unknown language'</span>
|
|
</code></pre></div>
|
|
|
|
<p><strong>Parameters:</strong></p>
|
|
<table>
|
|
<thead>
|
|
<tr>
|
|
<th>Name</th>
|
|
<th>Type</th>
|
|
<th>Description</th>
|
|
<th>Default</th>
|
|
</tr>
|
|
</thead>
|
|
<tbody>
|
|
<tr>
|
|
<td><code>language_code</code></td>
|
|
<td>
|
|
<code><span title="typing.Optional">Optional</span>[str]</code>
|
|
</td>
|
|
<td><p>Language code, for example, returned by
|
|
[great_ai.utilities.predict_language][].</p></td>
|
|
<td>
|
|
<em>required</em>
|
|
</td>
|
|
</tr>
|
|
</tbody>
|
|
</table>
|
|
|
|
<p><strong>Returns:</strong></p>
|
|
<table>
|
|
<thead>
|
|
<tr>
|
|
<th>Type</th>
|
|
<th>Description</th>
|
|
</tr>
|
|
</thead>
|
|
<tbody>
|
|
<tr>
|
|
<td>
|
|
<code>str</code>
|
|
</td>
|
|
<td><p>English name of language.</p></td>
|
|
</tr>
|
|
</tbody>
|
|
</table>
|
|
|
|
<details class="quote">
|
|
<summary>Source code in <code>great_ai/utilities/language/english_name_of_language.py</code></summary>
|
|
<div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"><a href="#__codelineno-0-6"> 6</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-7"> 7</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-8"> 8</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-9"> 9</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-10">10</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-11">11</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-12">12</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-13">13</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-14">14</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-15">15</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-16">16</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-17">17</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-18">18</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-19">19</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-20">20</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-21">21</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-22">22</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-23">23</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-24">24</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-25">25</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-26">26</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-27">27</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-28">28</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-29">29</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-30">30</a></span></pre></div></td><td class="code"><div><pre><span></span><code><a id="__codelineno-0-6" name="__codelineno-0-6"></a><span class="k">def</span> <span class="nf">english_name_of_language</span><span class="p">(</span><span class="n">language_code</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">str</span><span class="p">])</span> <span class="o">-></span> <span class="nb">str</span><span class="p">:</span>
|
|
<a id="__codelineno-0-7" name="__codelineno-0-7"></a> <span class="sd">"""Human-friendly English name of language from its `language_code`.</span>
|
|
<a id="__codelineno-0-8" name="__codelineno-0-8"></a>
|
|
<a id="__codelineno-0-9" name="__codelineno-0-9"></a><span class="sd"> A thin wrapper over [langcodes](https://github.com/rspeer/langcodes) for convenient</span>
|
|
<a id="__codelineno-0-10" name="__codelineno-0-10"></a><span class="sd"> language tagging.</span>
|
|
<a id="__codelineno-0-11" name="__codelineno-0-11"></a>
|
|
<a id="__codelineno-0-12" name="__codelineno-0-12"></a><span class="sd"> Examples:</span>
|
|
<a id="__codelineno-0-13" name="__codelineno-0-13"></a><span class="sd"> >>> english_name_of_language('en-US')</span>
|
|
<a id="__codelineno-0-14" name="__codelineno-0-14"></a><span class="sd"> 'English (United States)'</span>
|
|
<a id="__codelineno-0-15" name="__codelineno-0-15"></a>
|
|
<a id="__codelineno-0-16" name="__codelineno-0-16"></a><span class="sd"> >>> english_name_of_language('und')</span>
|
|
<a id="__codelineno-0-17" name="__codelineno-0-17"></a><span class="sd"> 'Unknown language'</span>
|
|
<a id="__codelineno-0-18" name="__codelineno-0-18"></a>
|
|
<a id="__codelineno-0-19" name="__codelineno-0-19"></a><span class="sd"> Args:</span>
|
|
<a id="__codelineno-0-20" name="__codelineno-0-20"></a><span class="sd"> language_code: Language code, for example, returned by</span>
|
|
<a id="__codelineno-0-21" name="__codelineno-0-21"></a><span class="sd"> [great_ai.utilities.predict_language][].</span>
|
|
<a id="__codelineno-0-22" name="__codelineno-0-22"></a>
|
|
<a id="__codelineno-0-23" name="__codelineno-0-23"></a><span class="sd"> Returns:</span>
|
|
<a id="__codelineno-0-24" name="__codelineno-0-24"></a><span class="sd"> English name of language.</span>
|
|
<a id="__codelineno-0-25" name="__codelineno-0-25"></a><span class="sd"> """</span>
|
|
<a id="__codelineno-0-26" name="__codelineno-0-26"></a>
|
|
<a id="__codelineno-0-27" name="__codelineno-0-27"></a> <span class="k">if</span> <span class="ow">not</span> <span class="n">language_code</span><span class="p">:</span>
|
|
<a id="__codelineno-0-28" name="__codelineno-0-28"></a> <span class="n">language_code</span> <span class="o">=</span> <span class="s2">"und"</span>
|
|
<a id="__codelineno-0-29" name="__codelineno-0-29"></a>
|
|
<a id="__codelineno-0-30" name="__codelineno-0-30"></a> <span class="k">return</span> <span class="n">Language</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">language_code</span><span class="p">)</span><span class="o">.</span><span class="n">display_name</span><span class="p">()</span>
|
|
</code></pre></div></td></tr></table></div>
|
|
</details>
|
|
</div>
|
|
|
|
</div>
|
|
|
|
<div class="doc doc-object doc-module">
|
|
|
|
|
|
|
|
<div class="doc doc-contents first">
|
|
|
|
|
|
|
|
<div class="doc doc-children">
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
</div>
|
|
|
|
</div>
|
|
|
|
</div><h2 id="parallel-processing">Parallel processing<a class="headerlink" href="#parallel-processing" title="Permanent link">#</a></h2>
|
|
<p>Multiprocessing and multithreading-based parallelism with support for <code>async</code> functions. Its main purpose is to implement <a class="autorefs autorefs-internal" href="../#great_ai.deploy.great_ai.GreatAI.process_batch">great_ai.GreatAI.process_batch</a>, however, the parallel processing functions are also convenient for covering other types of mapping needs with a friendlier API than <a href="https://joblib.readthedocs.io/en/latest/parallel.html" target="_blank">joblib</a> or <a href="https://pypi.org/project/multiprocess/" target="_blank">multiprocess</a>.</p>
|
|
|
|
|
|
<div class="doc doc-object doc-function">
|
|
|
|
|
|
|
|
<h3 id="great_ai.utilities.simple_parallel_map" class="doc doc-heading">
|
|
<code class="highlight language-python"><span class="n">simple_parallel_map</span><span class="p">(</span><span class="n">func</span><span class="p">,</span> <span class="n">input_values</span><span class="p">,</span> <span class="o">*</span><span class="p">,</span> <span class="n">chunk_size</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">concurrency</span><span class="o">=</span><span class="kc">None</span><span class="p">)</span></code>
|
|
|
|
<a href="#great_ai.utilities.simple_parallel_map" class="headerlink" title="Permanent link">#</a></h3>
|
|
|
|
|
|
<div class="doc doc-contents first">
|
|
|
|
<p>Execute a map operation on an list mimicking the API of the built-in <code>map()</code>.</p>
|
|
<p>A thin-wrapper over <a class="autorefs autorefs-internal" href="#great_ai.utilities.parallel_map.parallel_map.parallel_map">parallel_map</a>.
|
|
For more options, consult the documentation of
|
|
<a class="autorefs autorefs-internal" href="#great_ai.utilities.parallel_map.parallel_map.parallel_map">parallel_map</a>.</p>
|
|
|
|
<p><strong>Examples:</strong></p>
|
|
<div class="highlight"><pre><span></span><code><a id="__codelineno-0-1" name="__codelineno-0-1" href="#__codelineno-0-1"></a><span class="gp">>>> </span><span class="kn">import</span> <span class="nn">math</span>
|
|
<a id="__codelineno-0-2" name="__codelineno-0-2" href="#__codelineno-0-2"></a><span class="gp">>>> </span><span class="nb">list</span><span class="p">(</span><span class="n">simple_parallel_map</span><span class="p">(</span><span class="n">math</span><span class="o">.</span><span class="n">sqrt</span><span class="p">,</span> <span class="p">[</span><span class="mi">9</span><span class="p">,</span> <span class="mi">4</span><span class="p">,</span> <span class="mi">1</span><span class="p">]))</span>
|
|
<a id="__codelineno-0-3" name="__codelineno-0-3" href="#__codelineno-0-3"></a><span class="go">[3.0, 2.0, 1.0]</span>
|
|
</code></pre></div>
|
|
|
|
<p><strong>Parameters:</strong></p>
|
|
<table>
|
|
<thead>
|
|
<tr>
|
|
<th>Name</th>
|
|
<th>Type</th>
|
|
<th>Description</th>
|
|
<th>Default</th>
|
|
</tr>
|
|
</thead>
|
|
<tbody>
|
|
<tr>
|
|
<td><code>func</code></td>
|
|
<td>
|
|
<code><span title="typing.Callable">Callable</span>[[<span title="great_ai.utilities.parallel_map.simple_parallel_map.T">T</span>], <span title="typing.Union">Union</span>[<span title="great_ai.utilities.parallel_map.simple_parallel_map.V">V</span>, <span title="typing.Awaitable">Awaitable</span>[<span title="great_ai.utilities.parallel_map.simple_parallel_map.V">V</span>]]]</code>
|
|
</td>
|
|
<td><p>The function that should be applied to each element of <code>input_values</code>.
|
|
It can <code>async</code>, in that case, a new event loop is started for each chunk.</p></td>
|
|
<td>
|
|
<em>required</em>
|
|
</td>
|
|
</tr>
|
|
<tr>
|
|
<td><code>input_values</code></td>
|
|
<td>
|
|
<code><span title="typing.Sequence">Sequence</span>[<span title="great_ai.utilities.parallel_map.simple_parallel_map.T">T</span>]</code>
|
|
</td>
|
|
<td><p>An iterable of items that <code>func</code> is applied to.</p></td>
|
|
<td>
|
|
<em>required</em>
|
|
</td>
|
|
</tr>
|
|
<tr>
|
|
<td><code>chunk_size</code></td>
|
|
<td>
|
|
<code><span title="typing.Optional">Optional</span>[int]</code>
|
|
</td>
|
|
<td><p>Tune the number of items processed in each step. Larger numbers
|
|
result in smaller communication overhead but less parallelism at the start
|
|
and end. If <code>chunk_size</code> has a <code>__len__</code> property, the <code>chunk_size</code> is
|
|
calculated automatically if not given.</p></td>
|
|
<td>
|
|
<code>None</code>
|
|
</td>
|
|
</tr>
|
|
<tr>
|
|
<td><code>concurrency</code></td>
|
|
<td>
|
|
<code><span title="typing.Optional">Optional</span>[int]</code>
|
|
</td>
|
|
<td><p>Number of new processes to start. Shouldn't be too much more than
|
|
the number of physical cores.</p></td>
|
|
<td>
|
|
<code>None</code>
|
|
</td>
|
|
</tr>
|
|
</tbody>
|
|
</table>
|
|
|
|
<p><strong>Returns:</strong></p>
|
|
<table>
|
|
<thead>
|
|
<tr>
|
|
<th>Type</th>
|
|
<th>Description</th>
|
|
</tr>
|
|
</thead>
|
|
<tbody>
|
|
<tr>
|
|
<td>
|
|
<code><span title="typing.List">List</span>[<span title="great_ai.utilities.parallel_map.simple_parallel_map.V">V</span>]</code>
|
|
</td>
|
|
<td><p>An iterable of results obtained from applying <code>func</code> to each input value.</p></td>
|
|
</tr>
|
|
</tbody>
|
|
</table>
|
|
|
|
<p><strong>Raises:</strong></p>
|
|
<table>
|
|
<thead>
|
|
<tr>
|
|
<th>Type</th>
|
|
<th>Description</th>
|
|
</tr>
|
|
</thead>
|
|
<tbody>
|
|
<tr>
|
|
<td>
|
|
<code><span title="great_ai.utilities.parallel_map.worker_exception.WorkerException">WorkerException</span></code>
|
|
</td>
|
|
<td><p>If there was an error in the <code>func</code> function in a background
|
|
process.</p></td>
|
|
</tr>
|
|
</tbody>
|
|
</table>
|
|
|
|
<details class="quote">
|
|
<summary>Source code in <code>great_ai/utilities/parallel_map/simple_parallel_map.py</code></summary>
|
|
<div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"><a href="#__codelineno-0-11">11</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-12">12</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-13">13</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-14">14</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-15">15</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-16">16</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-17">17</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-18">18</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-19">19</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-20">20</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-21">21</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-22">22</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-23">23</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-24">24</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-25">25</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-26">26</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-27">27</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-28">28</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-29">29</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-30">30</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-31">31</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-32">32</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-33">33</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-34">34</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-35">35</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-36">36</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-37">37</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-38">38</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-39">39</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-40">40</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-41">41</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-42">42</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-43">43</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-44">44</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-45">45</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-46">46</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-47">47</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-48">48</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-49">49</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-50">50</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-51">51</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-52">52</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-53">53</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-54">54</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-55">55</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-56">56</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-57">57</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-58">58</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-59">59</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-60">60</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-61">61</a></span></pre></div></td><td class="code"><div><pre><span></span><code><a id="__codelineno-0-11" name="__codelineno-0-11"></a><span class="k">def</span> <span class="nf">simple_parallel_map</span><span class="p">(</span>
|
|
<a id="__codelineno-0-12" name="__codelineno-0-12"></a> <span class="n">func</span><span class="p">:</span> <span class="n">Callable</span><span class="p">[[</span><span class="n">T</span><span class="p">],</span> <span class="n">Union</span><span class="p">[</span><span class="n">V</span><span class="p">,</span> <span class="n">Awaitable</span><span class="p">[</span><span class="n">V</span><span class="p">]]],</span>
|
|
<a id="__codelineno-0-13" name="__codelineno-0-13"></a> <span class="n">input_values</span><span class="p">:</span> <span class="n">Sequence</span><span class="p">[</span><span class="n">T</span><span class="p">],</span>
|
|
<a id="__codelineno-0-14" name="__codelineno-0-14"></a> <span class="o">*</span><span class="p">,</span>
|
|
<a id="__codelineno-0-15" name="__codelineno-0-15"></a> <span class="n">chunk_size</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">int</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
|
|
<a id="__codelineno-0-16" name="__codelineno-0-16"></a> <span class="n">concurrency</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">int</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
|
|
<a id="__codelineno-0-17" name="__codelineno-0-17"></a><span class="p">)</span> <span class="o">-></span> <span class="n">List</span><span class="p">[</span><span class="n">V</span><span class="p">]:</span>
|
|
<a id="__codelineno-0-18" name="__codelineno-0-18"></a> <span class="sd">"""Execute a map operation on an list mimicking the API of the built-in `map()`.</span>
|
|
<a id="__codelineno-0-19" name="__codelineno-0-19"></a>
|
|
<a id="__codelineno-0-20" name="__codelineno-0-20"></a><span class="sd"> A thin-wrapper over [parallel_map][great_ai.utilities.parallel_map.parallel_map.parallel_map].</span>
|
|
<a id="__codelineno-0-21" name="__codelineno-0-21"></a><span class="sd"> For more options, consult the documentation of</span>
|
|
<a id="__codelineno-0-22" name="__codelineno-0-22"></a><span class="sd"> [parallel_map][great_ai.utilities.parallel_map.parallel_map.parallel_map].</span>
|
|
<a id="__codelineno-0-23" name="__codelineno-0-23"></a>
|
|
<a id="__codelineno-0-24" name="__codelineno-0-24"></a><span class="sd"> Examples:</span>
|
|
<a id="__codelineno-0-25" name="__codelineno-0-25"></a><span class="sd"> >>> import math</span>
|
|
<a id="__codelineno-0-26" name="__codelineno-0-26"></a><span class="sd"> >>> list(simple_parallel_map(math.sqrt, [9, 4, 1]))</span>
|
|
<a id="__codelineno-0-27" name="__codelineno-0-27"></a><span class="sd"> [3.0, 2.0, 1.0]</span>
|
|
<a id="__codelineno-0-28" name="__codelineno-0-28"></a>
|
|
<a id="__codelineno-0-29" name="__codelineno-0-29"></a><span class="sd"> Args:</span>
|
|
<a id="__codelineno-0-30" name="__codelineno-0-30"></a><span class="sd"> func: The function that should be applied to each element of `input_values`.</span>
|
|
<a id="__codelineno-0-31" name="__codelineno-0-31"></a><span class="sd"> It can `async`, in that case, a new event loop is started for each chunk.</span>
|
|
<a id="__codelineno-0-32" name="__codelineno-0-32"></a><span class="sd"> input_values: An iterable of items that `func` is applied to.</span>
|
|
<a id="__codelineno-0-33" name="__codelineno-0-33"></a><span class="sd"> chunk_size: Tune the number of items processed in each step. Larger numbers</span>
|
|
<a id="__codelineno-0-34" name="__codelineno-0-34"></a><span class="sd"> result in smaller communication overhead but less parallelism at the start</span>
|
|
<a id="__codelineno-0-35" name="__codelineno-0-35"></a><span class="sd"> and end. If `chunk_size` has a `__len__` property, the `chunk_size` is</span>
|
|
<a id="__codelineno-0-36" name="__codelineno-0-36"></a><span class="sd"> calculated automatically if not given.</span>
|
|
<a id="__codelineno-0-37" name="__codelineno-0-37"></a><span class="sd"> concurrency: Number of new processes to start. Shouldn't be too much more than</span>
|
|
<a id="__codelineno-0-38" name="__codelineno-0-38"></a><span class="sd"> the number of physical cores.</span>
|
|
<a id="__codelineno-0-39" name="__codelineno-0-39"></a>
|
|
<a id="__codelineno-0-40" name="__codelineno-0-40"></a><span class="sd"> Returns:</span>
|
|
<a id="__codelineno-0-41" name="__codelineno-0-41"></a><span class="sd"> An iterable of results obtained from applying `func` to each input value.</span>
|
|
<a id="__codelineno-0-42" name="__codelineno-0-42"></a>
|
|
<a id="__codelineno-0-43" name="__codelineno-0-43"></a><span class="sd"> Raises:</span>
|
|
<a id="__codelineno-0-44" name="__codelineno-0-44"></a><span class="sd"> WorkerException: If there was an error in the `func` function in a background</span>
|
|
<a id="__codelineno-0-45" name="__codelineno-0-45"></a><span class="sd"> process.</span>
|
|
<a id="__codelineno-0-46" name="__codelineno-0-46"></a><span class="sd"> """</span>
|
|
<a id="__codelineno-0-47" name="__codelineno-0-47"></a>
|
|
<a id="__codelineno-0-48" name="__codelineno-0-48"></a> <span class="n">input_values</span> <span class="o">=</span> <span class="nb">list</span><span class="p">(</span><span class="n">input_values</span><span class="p">)</span> <span class="c1"># in case the input is mistakenly not a sequence</span>
|
|
<a id="__codelineno-0-49" name="__codelineno-0-49"></a> <span class="n">generator</span> <span class="o">=</span> <span class="n">parallel_map</span><span class="p">(</span>
|
|
<a id="__codelineno-0-50" name="__codelineno-0-50"></a> <span class="n">func</span><span class="o">=</span><span class="n">func</span><span class="p">,</span>
|
|
<a id="__codelineno-0-51" name="__codelineno-0-51"></a> <span class="n">input_values</span><span class="o">=</span><span class="n">input_values</span><span class="p">,</span>
|
|
<a id="__codelineno-0-52" name="__codelineno-0-52"></a> <span class="n">chunk_size</span><span class="o">=</span><span class="n">chunk_size</span><span class="p">,</span>
|
|
<a id="__codelineno-0-53" name="__codelineno-0-53"></a> <span class="n">concurrency</span><span class="o">=</span><span class="n">concurrency</span><span class="p">,</span>
|
|
<a id="__codelineno-0-54" name="__codelineno-0-54"></a> <span class="p">)</span>
|
|
<a id="__codelineno-0-55" name="__codelineno-0-55"></a>
|
|
<a id="__codelineno-0-56" name="__codelineno-0-56"></a> <span class="k">return</span> <span class="nb">list</span><span class="p">(</span>
|
|
<a id="__codelineno-0-57" name="__codelineno-0-57"></a> <span class="n">tqdm</span><span class="p">(</span>
|
|
<a id="__codelineno-0-58" name="__codelineno-0-58"></a> <span class="n">generator</span><span class="p">,</span>
|
|
<a id="__codelineno-0-59" name="__codelineno-0-59"></a> <span class="n">total</span><span class="o">=</span><span class="nb">len</span><span class="p">(</span><span class="n">input_values</span><span class="p">),</span>
|
|
<a id="__codelineno-0-60" name="__codelineno-0-60"></a> <span class="p">)</span>
|
|
<a id="__codelineno-0-61" name="__codelineno-0-61"></a> <span class="p">)</span>
|
|
</code></pre></div></td></tr></table></div>
|
|
</details>
|
|
</div>
|
|
|
|
</div>
|
|
|
|
<div class="doc doc-object doc-module">
|
|
|
|
|
|
|
|
<div class="doc doc-contents first">
|
|
|
|
|
|
|
|
<div class="doc doc-children">
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
<div class="doc doc-object doc-function">
|
|
|
|
|
|
|
|
<h3 id="great_ai.utilities.parallel_map.parallel_map.parallel_map" class="doc doc-heading">
|
|
<code class="highlight language-python"><span class="n">parallel_map</span><span class="p">(</span><span class="n">func</span><span class="p">,</span> <span class="n">input_values</span><span class="p">,</span> <span class="o">*</span><span class="p">,</span> <span class="n">chunk_size</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">ignore_exceptions</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span> <span class="n">concurrency</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">unordered</span><span class="o">=</span><span class="kc">False</span><span class="p">)</span></code>
|
|
|
|
<a href="#great_ai.utilities.parallel_map.parallel_map.parallel_map" class="headerlink" title="Permanent link">#</a></h3>
|
|
|
|
|
|
<div class="doc doc-contents ">
|
|
|
|
<p>Execute a map operation on an iterable stream.</p>
|
|
<p>A custom parallel map operation supporting both synchronous and <code>async</code> map
|
|
functions. The <code>func</code> function is serialised with <code>dill</code>. Exceptions encountered in
|
|
the map function are sent to the host process where they are either raised (default)
|
|
or ignored.</p>
|
|
<p>The new processes are forked if the OS allows it, otherwise, new Python processes
|
|
are bootstrapped which can incur some startup cost. Each process processes a single
|
|
chunk at once.</p>
|
|
|
|
<p><strong>Examples:</strong></p>
|
|
<div class="highlight"><pre><span></span><code><a id="__codelineno-0-1" name="__codelineno-0-1" href="#__codelineno-0-1"></a><span class="gp">>>> </span><span class="kn">import</span> <span class="nn">math</span>
|
|
<a id="__codelineno-0-2" name="__codelineno-0-2" href="#__codelineno-0-2"></a><span class="gp">>>> </span><span class="nb">list</span><span class="p">(</span><span class="n">parallel_map</span><span class="p">(</span><span class="n">math</span><span class="o">.</span><span class="n">sqrt</span><span class="p">,</span> <span class="p">[</span><span class="mi">9</span><span class="p">,</span> <span class="mi">4</span><span class="p">,</span> <span class="mi">1</span><span class="p">],</span> <span class="n">concurrency</span><span class="o">=</span><span class="mi">2</span><span class="p">))</span>
|
|
<a id="__codelineno-0-3" name="__codelineno-0-3" href="#__codelineno-0-3"></a><span class="go">[3.0, 2.0, 1.0]</span>
|
|
</code></pre></div>
|
|
|
|
<p><strong>Parameters:</strong></p>
|
|
<table>
|
|
<thead>
|
|
<tr>
|
|
<th>Name</th>
|
|
<th>Type</th>
|
|
<th>Description</th>
|
|
<th>Default</th>
|
|
</tr>
|
|
</thead>
|
|
<tbody>
|
|
<tr>
|
|
<td><code>func</code></td>
|
|
<td>
|
|
<code><span title="typing.Callable">Callable</span>[[<span title="great_ai.utilities.parallel_map.parallel_map.T">T</span>], <span title="typing.Union">Union</span>[<span title="great_ai.utilities.parallel_map.parallel_map.V">V</span>, <span title="typing.Awaitable">Awaitable</span>[<span title="great_ai.utilities.parallel_map.parallel_map.V">V</span>]]]</code>
|
|
</td>
|
|
<td><p>The function that should be applied to each element of <code>input_values</code>.
|
|
It can <code>async</code>, in that case, a new event loop is started for each chunk.</p></td>
|
|
<td>
|
|
<em>required</em>
|
|
</td>
|
|
</tr>
|
|
<tr>
|
|
<td><code>input_values</code></td>
|
|
<td>
|
|
<code><span title="typing.Union">Union</span>[<span title="typing.Iterable">Iterable</span>[<span title="great_ai.utilities.parallel_map.parallel_map.T">T</span>], <span title="typing.Sequence">Sequence</span>[<span title="great_ai.utilities.parallel_map.parallel_map.T">T</span>]]</code>
|
|
</td>
|
|
<td><p>An iterable of items that <code>func</code> is applied to.</p></td>
|
|
<td>
|
|
<em>required</em>
|
|
</td>
|
|
</tr>
|
|
<tr>
|
|
<td><code>chunk_size</code></td>
|
|
<td>
|
|
<code><span title="typing.Optional">Optional</span>[int]</code>
|
|
</td>
|
|
<td><p>Tune the number of items processed in each step. Larger numbers
|
|
result in smaller communication overhead but less parallelism at the start
|
|
and end. If <code>chunk_size</code> has a <code>__len__</code> property, the <code>chunk_size</code> is
|
|
calculated automatically if not given.</p></td>
|
|
<td>
|
|
<code>None</code>
|
|
</td>
|
|
</tr>
|
|
<tr>
|
|
<td><code>ignore_exceptions</code></td>
|
|
<td>
|
|
<code>bool</code>
|
|
</td>
|
|
<td><p>Ignore chunks if <code>next()</code> raises an exception on
|
|
<code>input_values</code>. And return <code>None</code> if <code>func</code> raised an exception in a worker
|
|
process.</p></td>
|
|
<td>
|
|
<code>False</code>
|
|
</td>
|
|
</tr>
|
|
<tr>
|
|
<td><code>concurrency</code></td>
|
|
<td>
|
|
<code><span title="typing.Optional">Optional</span>[int]</code>
|
|
</td>
|
|
<td><p>Number of new processes to start. Shouldn't be too much more than
|
|
the number of physical cores.</p></td>
|
|
<td>
|
|
<code>None</code>
|
|
</td>
|
|
</tr>
|
|
<tr>
|
|
<td><code>unordered</code></td>
|
|
<td>
|
|
<code>bool</code>
|
|
</td>
|
|
<td><p>Do not preserve the order of the elements, yield them as soon as they
|
|
have been processed. This decreases the latency caused by
|
|
difficult-to-process items.</p></td>
|
|
<td>
|
|
<code>False</code>
|
|
</td>
|
|
</tr>
|
|
</tbody>
|
|
</table>
|
|
|
|
<p><strong>Yields:</strong></p>
|
|
<table>
|
|
<thead>
|
|
<tr>
|
|
<th>Type</th>
|
|
<th>Description</th>
|
|
</tr>
|
|
</thead>
|
|
<tbody>
|
|
<tr>
|
|
<td>
|
|
<code><span title="typing.Iterable">Iterable</span>[<span title="typing.Optional">Optional</span>[<span title="great_ai.utilities.parallel_map.parallel_map.V">V</span>]]</code>
|
|
</td>
|
|
<td><p>The next result obtained from applying <code>func</code> to each input value. May
|
|
contain <code>None</code>-s if <code>ignore_exceptions=True</code>. May have different order than
|
|
the input if <code>unordered=True</code>.</p></td>
|
|
</tr>
|
|
</tbody>
|
|
</table>
|
|
|
|
<p><strong>Raises:</strong></p>
|
|
<table>
|
|
<thead>
|
|
<tr>
|
|
<th>Type</th>
|
|
<th>Description</th>
|
|
</tr>
|
|
</thead>
|
|
<tbody>
|
|
<tr>
|
|
<td>
|
|
<code><span title="great_ai.utilities.parallel_map.worker_exception.WorkerException">WorkerException</span></code>
|
|
</td>
|
|
<td><p>If there was an error in the <code>func</code> function in a background
|
|
process and <code>ignore_exceptions=False</code>.</p></td>
|
|
</tr>
|
|
</tbody>
|
|
</table>
|
|
|
|
<details class="quote">
|
|
<summary>Source code in <code>great_ai/utilities/parallel_map/parallel_map.py</code></summary>
|
|
<div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"><a href="#__codelineno-0-77"> 77</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-78"> 78</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-79"> 79</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-80"> 80</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-81"> 81</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-82"> 82</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-83"> 83</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-84"> 84</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-85"> 85</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-86"> 86</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-87"> 87</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-88"> 88</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-89"> 89</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-90"> 90</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-91"> 91</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-92"> 92</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-93"> 93</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-94"> 94</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-95"> 95</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-96"> 96</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-97"> 97</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-98"> 98</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-99"> 99</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-100">100</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-101">101</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-102">102</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-103">103</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-104">104</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-105">105</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-106">106</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-107">107</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-108">108</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-109">109</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-110">110</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-111">111</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-112">112</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-113">113</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-114">114</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-115">115</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-116">116</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-117">117</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-118">118</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-119">119</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-120">120</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-121">121</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-122">122</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-123">123</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-124">124</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-125">125</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-126">126</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-127">127</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-128">128</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-129">129</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-130">130</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-131">131</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-132">132</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-133">133</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-134">134</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-135">135</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-136">136</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-137">137</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-138">138</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-139">139</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-140">140</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-141">141</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-142">142</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-143">143</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-144">144</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-145">145</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-146">146</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-147">147</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-148">148</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-149">149</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-150">150</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-151">151</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-152">152</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-153">153</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-154">154</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-155">155</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-156">156</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-157">157</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-158">158</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-159">159</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-160">160</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-161">161</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-162">162</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-163">163</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-164">164</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-165">165</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-166">166</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-167">167</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-168">168</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-169">169</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-170">170</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-171">171</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-172">172</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-173">173</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-174">174</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-175">175</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-176">176</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-177">177</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-178">178</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-179">179</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-180">180</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-181">181</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-182">182</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-183">183</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-184">184</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-185">185</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-186">186</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-187">187</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-188">188</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-189">189</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-190">190</a></span></pre></div></td><td class="code"><div><pre><span></span><code><a id="__codelineno-0-77" name="__codelineno-0-77"></a><span class="k">def</span> <span class="nf">parallel_map</span><span class="p">(</span>
|
|
<a id="__codelineno-0-78" name="__codelineno-0-78"></a> <span class="n">func</span><span class="p">:</span> <span class="n">Callable</span><span class="p">[[</span><span class="n">T</span><span class="p">],</span> <span class="n">Union</span><span class="p">[</span><span class="n">V</span><span class="p">,</span> <span class="n">Awaitable</span><span class="p">[</span><span class="n">V</span><span class="p">]]],</span>
|
|
<a id="__codelineno-0-79" name="__codelineno-0-79"></a> <span class="n">input_values</span><span class="p">:</span> <span class="n">Union</span><span class="p">[</span><span class="n">Iterable</span><span class="p">[</span><span class="n">T</span><span class="p">],</span> <span class="n">Sequence</span><span class="p">[</span><span class="n">T</span><span class="p">]],</span>
|
|
<a id="__codelineno-0-80" name="__codelineno-0-80"></a> <span class="o">*</span><span class="p">,</span>
|
|
<a id="__codelineno-0-81" name="__codelineno-0-81"></a> <span class="n">chunk_size</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">int</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
|
|
<a id="__codelineno-0-82" name="__codelineno-0-82"></a> <span class="n">ignore_exceptions</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">False</span><span class="p">,</span>
|
|
<a id="__codelineno-0-83" name="__codelineno-0-83"></a> <span class="n">concurrency</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">int</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
|
|
<a id="__codelineno-0-84" name="__codelineno-0-84"></a> <span class="n">unordered</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">False</span><span class="p">,</span>
|
|
<a id="__codelineno-0-85" name="__codelineno-0-85"></a><span class="p">)</span> <span class="o">-></span> <span class="n">Iterable</span><span class="p">[</span><span class="n">Optional</span><span class="p">[</span><span class="n">V</span><span class="p">]]:</span>
|
|
<a id="__codelineno-0-86" name="__codelineno-0-86"></a> <span class="sd">"""Execute a map operation on an iterable stream.</span>
|
|
<a id="__codelineno-0-87" name="__codelineno-0-87"></a>
|
|
<a id="__codelineno-0-88" name="__codelineno-0-88"></a><span class="sd"> A custom parallel map operation supporting both synchronous and `async` map</span>
|
|
<a id="__codelineno-0-89" name="__codelineno-0-89"></a><span class="sd"> functions. The `func` function is serialised with `dill`. Exceptions encountered in</span>
|
|
<a id="__codelineno-0-90" name="__codelineno-0-90"></a><span class="sd"> the map function are sent to the host process where they are either raised (default)</span>
|
|
<a id="__codelineno-0-91" name="__codelineno-0-91"></a><span class="sd"> or ignored.</span>
|
|
<a id="__codelineno-0-92" name="__codelineno-0-92"></a>
|
|
<a id="__codelineno-0-93" name="__codelineno-0-93"></a><span class="sd"> The new processes are forked if the OS allows it, otherwise, new Python processes</span>
|
|
<a id="__codelineno-0-94" name="__codelineno-0-94"></a><span class="sd"> are bootstrapped which can incur some startup cost. Each process processes a single</span>
|
|
<a id="__codelineno-0-95" name="__codelineno-0-95"></a><span class="sd"> chunk at once.</span>
|
|
<a id="__codelineno-0-96" name="__codelineno-0-96"></a>
|
|
<a id="__codelineno-0-97" name="__codelineno-0-97"></a><span class="sd"> Examples:</span>
|
|
<a id="__codelineno-0-98" name="__codelineno-0-98"></a><span class="sd"> >>> import math</span>
|
|
<a id="__codelineno-0-99" name="__codelineno-0-99"></a><span class="sd"> >>> list(parallel_map(math.sqrt, [9, 4, 1], concurrency=2))</span>
|
|
<a id="__codelineno-0-100" name="__codelineno-0-100"></a><span class="sd"> [3.0, 2.0, 1.0]</span>
|
|
<a id="__codelineno-0-101" name="__codelineno-0-101"></a>
|
|
<a id="__codelineno-0-102" name="__codelineno-0-102"></a><span class="sd"> Args:</span>
|
|
<a id="__codelineno-0-103" name="__codelineno-0-103"></a><span class="sd"> func: The function that should be applied to each element of `input_values`.</span>
|
|
<a id="__codelineno-0-104" name="__codelineno-0-104"></a><span class="sd"> It can `async`, in that case, a new event loop is started for each chunk.</span>
|
|
<a id="__codelineno-0-105" name="__codelineno-0-105"></a><span class="sd"> input_values: An iterable of items that `func` is applied to.</span>
|
|
<a id="__codelineno-0-106" name="__codelineno-0-106"></a><span class="sd"> chunk_size: Tune the number of items processed in each step. Larger numbers</span>
|
|
<a id="__codelineno-0-107" name="__codelineno-0-107"></a><span class="sd"> result in smaller communication overhead but less parallelism at the start</span>
|
|
<a id="__codelineno-0-108" name="__codelineno-0-108"></a><span class="sd"> and end. If `chunk_size` has a `__len__` property, the `chunk_size` is</span>
|
|
<a id="__codelineno-0-109" name="__codelineno-0-109"></a><span class="sd"> calculated automatically if not given.</span>
|
|
<a id="__codelineno-0-110" name="__codelineno-0-110"></a><span class="sd"> ignore_exceptions: Ignore chunks if `next()` raises an exception on</span>
|
|
<a id="__codelineno-0-111" name="__codelineno-0-111"></a><span class="sd"> `input_values`. And return `None` if `func` raised an exception in a worker</span>
|
|
<a id="__codelineno-0-112" name="__codelineno-0-112"></a><span class="sd"> process.</span>
|
|
<a id="__codelineno-0-113" name="__codelineno-0-113"></a><span class="sd"> concurrency: Number of new processes to start. Shouldn't be too much more than</span>
|
|
<a id="__codelineno-0-114" name="__codelineno-0-114"></a><span class="sd"> the number of physical cores.</span>
|
|
<a id="__codelineno-0-115" name="__codelineno-0-115"></a><span class="sd"> unordered: Do not preserve the order of the elements, yield them as soon as they</span>
|
|
<a id="__codelineno-0-116" name="__codelineno-0-116"></a><span class="sd"> have been processed. This decreases the latency caused by</span>
|
|
<a id="__codelineno-0-117" name="__codelineno-0-117"></a><span class="sd"> difficult-to-process items.</span>
|
|
<a id="__codelineno-0-118" name="__codelineno-0-118"></a>
|
|
<a id="__codelineno-0-119" name="__codelineno-0-119"></a><span class="sd"> Yields:</span>
|
|
<a id="__codelineno-0-120" name="__codelineno-0-120"></a><span class="sd"> The next result obtained from applying `func` to each input value. May</span>
|
|
<a id="__codelineno-0-121" name="__codelineno-0-121"></a><span class="sd"> contain `None`-s if `ignore_exceptions=True`. May have different order than</span>
|
|
<a id="__codelineno-0-122" name="__codelineno-0-122"></a><span class="sd"> the input if `unordered=True`.</span>
|
|
<a id="__codelineno-0-123" name="__codelineno-0-123"></a>
|
|
<a id="__codelineno-0-124" name="__codelineno-0-124"></a><span class="sd"> Raises:</span>
|
|
<a id="__codelineno-0-125" name="__codelineno-0-125"></a><span class="sd"> WorkerException: If there was an error in the `func` function in a background</span>
|
|
<a id="__codelineno-0-126" name="__codelineno-0-126"></a><span class="sd"> process and `ignore_exceptions=False`.</span>
|
|
<a id="__codelineno-0-127" name="__codelineno-0-127"></a><span class="sd"> """</span>
|
|
<a id="__codelineno-0-128" name="__codelineno-0-128"></a>
|
|
<a id="__codelineno-0-129" name="__codelineno-0-129"></a> <span class="n">config</span> <span class="o">=</span> <span class="n">get_config</span><span class="p">(</span>
|
|
<a id="__codelineno-0-130" name="__codelineno-0-130"></a> <span class="n">function</span><span class="o">=</span><span class="n">func</span><span class="p">,</span>
|
|
<a id="__codelineno-0-131" name="__codelineno-0-131"></a> <span class="n">input_values</span><span class="o">=</span><span class="n">input_values</span><span class="p">,</span>
|
|
<a id="__codelineno-0-132" name="__codelineno-0-132"></a> <span class="n">chunk_size</span><span class="o">=</span><span class="n">chunk_size</span><span class="p">,</span>
|
|
<a id="__codelineno-0-133" name="__codelineno-0-133"></a> <span class="n">concurrency</span><span class="o">=</span><span class="n">concurrency</span><span class="p">,</span>
|
|
<a id="__codelineno-0-134" name="__codelineno-0-134"></a> <span class="p">)</span>
|
|
<a id="__codelineno-0-135" name="__codelineno-0-135"></a>
|
|
<a id="__codelineno-0-136" name="__codelineno-0-136"></a> <span class="n">ctx</span> <span class="o">=</span> <span class="p">(</span>
|
|
<a id="__codelineno-0-137" name="__codelineno-0-137"></a> <span class="n">mp</span><span class="o">.</span><span class="n">get_context</span><span class="p">(</span><span class="s2">"fork"</span><span class="p">)</span>
|
|
<a id="__codelineno-0-138" name="__codelineno-0-138"></a> <span class="k">if</span> <span class="s2">"fork"</span> <span class="ow">in</span> <span class="n">mp</span><span class="o">.</span><span class="n">get_all_start_methods</span><span class="p">()</span>
|
|
<a id="__codelineno-0-139" name="__codelineno-0-139"></a> <span class="k">else</span> <span class="n">mp</span><span class="o">.</span><span class="n">get_context</span><span class="p">(</span><span class="s2">"spawn"</span><span class="p">)</span>
|
|
<a id="__codelineno-0-140" name="__codelineno-0-140"></a> <span class="p">)</span>
|
|
<a id="__codelineno-0-141" name="__codelineno-0-141"></a> <span class="n">ctx</span><span class="o">.</span><span class="n">freeze_support</span><span class="p">()</span>
|
|
<a id="__codelineno-0-142" name="__codelineno-0-142"></a> <span class="n">manager</span> <span class="o">=</span> <span class="n">ctx</span><span class="o">.</span><span class="n">Manager</span><span class="p">()</span>
|
|
<a id="__codelineno-0-143" name="__codelineno-0-143"></a> <span class="n">input_queue</span> <span class="o">=</span> <span class="n">manager</span><span class="o">.</span><span class="n">Queue</span><span class="p">(</span><span class="n">config</span><span class="o">.</span><span class="n">concurrency</span> <span class="o">*</span> <span class="mi">2</span><span class="p">)</span>
|
|
<a id="__codelineno-0-144" name="__codelineno-0-144"></a> <span class="n">output_queue</span> <span class="o">=</span> <span class="n">manager</span><span class="o">.</span><span class="n">Queue</span><span class="p">(</span><span class="n">config</span><span class="o">.</span><span class="n">concurrency</span> <span class="o">*</span> <span class="mi">2</span><span class="p">)</span>
|
|
<a id="__codelineno-0-145" name="__codelineno-0-145"></a>
|
|
<a id="__codelineno-0-146" name="__codelineno-0-146"></a> <span class="n">should_stop</span> <span class="o">=</span> <span class="n">ctx</span><span class="o">.</span><span class="n">Event</span><span class="p">()</span>
|
|
<a id="__codelineno-0-147" name="__codelineno-0-147"></a> <span class="n">serialized_map_function</span> <span class="o">=</span> <span class="n">dill</span><span class="o">.</span><span class="n">dumps</span><span class="p">(</span><span class="n">func</span><span class="p">,</span> <span class="n">byref</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span> <span class="n">recurse</span><span class="o">=</span><span class="kc">False</span><span class="p">)</span>
|
|
<a id="__codelineno-0-148" name="__codelineno-0-148"></a>
|
|
<a id="__codelineno-0-149" name="__codelineno-0-149"></a> <span class="n">processes</span> <span class="o">=</span> <span class="p">[</span>
|
|
<a id="__codelineno-0-150" name="__codelineno-0-150"></a> <span class="n">ctx</span><span class="o">.</span><span class="n">Process</span><span class="p">(</span>
|
|
<a id="__codelineno-0-151" name="__codelineno-0-151"></a> <span class="n">name</span><span class="o">=</span><span class="sa">f</span><span class="s2">"parallel_map_</span><span class="si">{</span><span class="n">config</span><span class="o">.</span><span class="n">function_name</span><span class="si">}</span><span class="s2">_</span><span class="si">{</span><span class="n">i</span><span class="si">}</span><span class="s2">"</span><span class="p">,</span>
|
|
<a id="__codelineno-0-152" name="__codelineno-0-152"></a> <span class="n">target</span><span class="o">=</span><span class="n">mapper_function</span><span class="p">,</span>
|
|
<a id="__codelineno-0-153" name="__codelineno-0-153"></a> <span class="n">daemon</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span>
|
|
<a id="__codelineno-0-154" name="__codelineno-0-154"></a> <span class="n">kwargs</span><span class="o">=</span><span class="nb">dict</span><span class="p">(</span>
|
|
<a id="__codelineno-0-155" name="__codelineno-0-155"></a> <span class="n">input_queue</span><span class="o">=</span><span class="n">input_queue</span><span class="p">,</span>
|
|
<a id="__codelineno-0-156" name="__codelineno-0-156"></a> <span class="n">output_queue</span><span class="o">=</span><span class="n">output_queue</span><span class="p">,</span>
|
|
<a id="__codelineno-0-157" name="__codelineno-0-157"></a> <span class="n">should_stop</span><span class="o">=</span><span class="n">should_stop</span><span class="p">,</span>
|
|
<a id="__codelineno-0-158" name="__codelineno-0-158"></a> <span class="n">func</span><span class="o">=</span><span class="n">serialized_map_function</span><span class="p">,</span>
|
|
<a id="__codelineno-0-159" name="__codelineno-0-159"></a> <span class="p">),</span>
|
|
<a id="__codelineno-0-160" name="__codelineno-0-160"></a> <span class="p">)</span>
|
|
<a id="__codelineno-0-161" name="__codelineno-0-161"></a> <span class="k">for</span> <span class="n">i</span> <span class="ow">in</span> <span class="nb">range</span><span class="p">(</span><span class="n">config</span><span class="o">.</span><span class="n">concurrency</span><span class="p">)</span>
|
|
<a id="__codelineno-0-162" name="__codelineno-0-162"></a> <span class="p">]</span>
|
|
<a id="__codelineno-0-163" name="__codelineno-0-163"></a>
|
|
<a id="__codelineno-0-164" name="__codelineno-0-164"></a> <span class="k">for</span> <span class="n">p</span> <span class="ow">in</span> <span class="n">processes</span><span class="p">:</span>
|
|
<a id="__codelineno-0-165" name="__codelineno-0-165"></a> <span class="n">p</span><span class="o">.</span><span class="n">start</span><span class="p">()</span>
|
|
<a id="__codelineno-0-166" name="__codelineno-0-166"></a>
|
|
<a id="__codelineno-0-167" name="__codelineno-0-167"></a> <span class="k">try</span><span class="p">:</span>
|
|
<a id="__codelineno-0-168" name="__codelineno-0-168"></a> <span class="k">yield from</span> <span class="n">manage_communication</span><span class="p">(</span>
|
|
<a id="__codelineno-0-169" name="__codelineno-0-169"></a> <span class="n">input_values</span><span class="o">=</span><span class="n">input_values</span><span class="p">,</span>
|
|
<a id="__codelineno-0-170" name="__codelineno-0-170"></a> <span class="n">chunk_size</span><span class="o">=</span><span class="n">config</span><span class="o">.</span><span class="n">chunk_size</span><span class="p">,</span>
|
|
<a id="__codelineno-0-171" name="__codelineno-0-171"></a> <span class="n">input_queue</span><span class="o">=</span><span class="n">input_queue</span><span class="p">,</span>
|
|
<a id="__codelineno-0-172" name="__codelineno-0-172"></a> <span class="n">output_queue</span><span class="o">=</span><span class="n">output_queue</span><span class="p">,</span>
|
|
<a id="__codelineno-0-173" name="__codelineno-0-173"></a> <span class="n">unordered</span><span class="o">=</span><span class="n">unordered</span><span class="p">,</span>
|
|
<a id="__codelineno-0-174" name="__codelineno-0-174"></a> <span class="n">ignore_exceptions</span><span class="o">=</span><span class="n">ignore_exceptions</span><span class="p">,</span>
|
|
<a id="__codelineno-0-175" name="__codelineno-0-175"></a> <span class="p">)</span>
|
|
<a id="__codelineno-0-176" name="__codelineno-0-176"></a> <span class="n">should_stop</span><span class="o">.</span><span class="n">set</span><span class="p">()</span>
|
|
<a id="__codelineno-0-177" name="__codelineno-0-177"></a> <span class="k">except</span> <span class="n">WorkerException</span><span class="p">:</span>
|
|
<a id="__codelineno-0-178" name="__codelineno-0-178"></a> <span class="n">should_stop</span><span class="o">.</span><span class="n">set</span><span class="p">()</span>
|
|
<a id="__codelineno-0-179" name="__codelineno-0-179"></a> <span class="k">raise</span>
|
|
<a id="__codelineno-0-180" name="__codelineno-0-180"></a> <span class="k">except</span> <span class="ne">Exception</span><span class="p">:</span>
|
|
<a id="__codelineno-0-181" name="__codelineno-0-181"></a> <span class="k">for</span> <span class="n">p</span> <span class="ow">in</span> <span class="n">processes</span><span class="p">:</span>
|
|
<a id="__codelineno-0-182" name="__codelineno-0-182"></a> <span class="n">p</span><span class="o">.</span><span class="n">terminate</span><span class="p">()</span>
|
|
<a id="__codelineno-0-183" name="__codelineno-0-183"></a> <span class="n">p</span><span class="o">.</span><span class="n">kill</span><span class="p">()</span>
|
|
<a id="__codelineno-0-184" name="__codelineno-0-184"></a> <span class="k">raise</span>
|
|
<a id="__codelineno-0-185" name="__codelineno-0-185"></a> <span class="k">finally</span><span class="p">:</span>
|
|
<a id="__codelineno-0-186" name="__codelineno-0-186"></a> <span class="k">for</span> <span class="n">p</span> <span class="ow">in</span> <span class="n">processes</span><span class="p">:</span>
|
|
<a id="__codelineno-0-187" name="__codelineno-0-187"></a> <span class="n">p</span><span class="o">.</span><span class="n">join</span><span class="p">()</span> <span class="c1"># terminated processes have to be joined else they remain zombies</span>
|
|
<a id="__codelineno-0-188" name="__codelineno-0-188"></a> <span class="n">p</span><span class="o">.</span><span class="n">close</span><span class="p">()</span>
|
|
<a id="__codelineno-0-189" name="__codelineno-0-189"></a>
|
|
<a id="__codelineno-0-190" name="__codelineno-0-190"></a> <span class="n">manager</span><span class="o">.</span><span class="n">shutdown</span><span class="p">()</span>
|
|
</code></pre></div></td></tr></table></div>
|
|
</details>
|
|
</div>
|
|
|
|
</div>
|
|
|
|
|
|
|
|
</div>
|
|
|
|
</div>
|
|
|
|
</div>
|
|
|
|
<div class="doc doc-object doc-function">
|
|
|
|
|
|
|
|
<h3 id="great_ai.utilities.threaded_parallel_map" class="doc doc-heading">
|
|
<code class="highlight language-python"><span class="n">threaded_parallel_map</span><span class="p">(</span><span class="n">func</span><span class="p">,</span> <span class="n">input_values</span><span class="p">,</span> <span class="o">*</span><span class="p">,</span> <span class="n">chunk_size</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">ignore_exceptions</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span> <span class="n">concurrency</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">unordered</span><span class="o">=</span><span class="kc">False</span><span class="p">)</span></code>
|
|
|
|
<a href="#great_ai.utilities.threaded_parallel_map" class="headerlink" title="Permanent link">#</a></h3>
|
|
|
|
|
|
<div class="doc doc-contents first">
|
|
|
|
<p>Execute a map operation on an iterable stream.</p>
|
|
<p>Similar to <a class="autorefs autorefs-internal" href="#great_ai.utilities.parallel_map.parallel_map.parallel_map">parallel_map</a>
|
|
but uses threads instead of processes. Hence, it is not helpful in CPU-bound
|
|
situations.</p>
|
|
<p>A custom parallel map operation supporting both synchronous and <code>async</code> map
|
|
functions. Exceptions encountered in the map function are sent to the host thread
|
|
where they are either raised (default) or ignored. Each process processes a single
|
|
chunk at once.</p>
|
|
|
|
<p><strong>Examples:</strong></p>
|
|
<div class="highlight"><pre><span></span><code><a id="__codelineno-0-1" name="__codelineno-0-1" href="#__codelineno-0-1"></a><span class="gp">>>> </span><span class="nb">list</span><span class="p">(</span><span class="n">threaded_parallel_map</span><span class="p">(</span><span class="k">lambda</span> <span class="n">x</span><span class="p">:</span> <span class="n">x</span> <span class="o">**</span> <span class="mi">2</span><span class="p">,</span> <span class="p">[</span><span class="mi">1</span><span class="p">,</span> <span class="mi">2</span><span class="p">,</span> <span class="mi">3</span><span class="p">]))</span>
|
|
<a id="__codelineno-0-2" name="__codelineno-0-2" href="#__codelineno-0-2"></a><span class="go">[1, 4, 9]</span>
|
|
</code></pre></div>
|
|
|
|
<p><strong>Parameters:</strong></p>
|
|
<table>
|
|
<thead>
|
|
<tr>
|
|
<th>Name</th>
|
|
<th>Type</th>
|
|
<th>Description</th>
|
|
<th>Default</th>
|
|
</tr>
|
|
</thead>
|
|
<tbody>
|
|
<tr>
|
|
<td><code>func</code></td>
|
|
<td>
|
|
<code><span title="typing.Callable">Callable</span>[[<span title="great_ai.utilities.parallel_map.threaded_parallel_map.T">T</span>], <span title="typing.Union">Union</span>[<span title="great_ai.utilities.parallel_map.threaded_parallel_map.V">V</span>, <span title="typing.Awaitable">Awaitable</span>[<span title="great_ai.utilities.parallel_map.threaded_parallel_map.V">V</span>]]]</code>
|
|
</td>
|
|
<td><p>The function that should be applied to each element of <code>input_values</code>.
|
|
It can <code>async</code>, in that case, a new event loop is started for each chunk.</p></td>
|
|
<td>
|
|
<em>required</em>
|
|
</td>
|
|
</tr>
|
|
<tr>
|
|
<td><code>input_values</code></td>
|
|
<td>
|
|
<code><span title="typing.Union">Union</span>[<span title="typing.Iterable">Iterable</span>[<span title="great_ai.utilities.parallel_map.threaded_parallel_map.T">T</span>], <span title="typing.Sequence">Sequence</span>[<span title="great_ai.utilities.parallel_map.threaded_parallel_map.T">T</span>]]</code>
|
|
</td>
|
|
<td><p>An iterable of items that <code>func</code> is applied to.</p></td>
|
|
<td>
|
|
<em>required</em>
|
|
</td>
|
|
</tr>
|
|
<tr>
|
|
<td><code>chunk_size</code></td>
|
|
<td>
|
|
<code><span title="typing.Optional">Optional</span>[int]</code>
|
|
</td>
|
|
<td><p>Tune the number of items processed in each step. Larger numbers
|
|
result in smaller communication overhead but less parallelism at the start
|
|
and end. If <code>chunk_size</code> has a <code>__len__</code> property, the <code>chunk_size</code> is
|
|
calculated automatically if not given.</p></td>
|
|
<td>
|
|
<code>None</code>
|
|
</td>
|
|
</tr>
|
|
<tr>
|
|
<td><code>ignore_exceptions</code></td>
|
|
<td>
|
|
<code>bool</code>
|
|
</td>
|
|
<td><p>Ignore chunks if <code>next()</code> raises an exception on
|
|
<code>input_values</code>. And return <code>None</code> if <code>func</code> raised an exception in a worker
|
|
process.</p></td>
|
|
<td>
|
|
<code>False</code>
|
|
</td>
|
|
</tr>
|
|
<tr>
|
|
<td><code>concurrency</code></td>
|
|
<td>
|
|
<code><span title="typing.Optional">Optional</span>[int]</code>
|
|
</td>
|
|
<td><p>Number of new threads to start.</p></td>
|
|
<td>
|
|
<code>None</code>
|
|
</td>
|
|
</tr>
|
|
<tr>
|
|
<td><code>unordered</code></td>
|
|
<td>
|
|
<code>bool</code>
|
|
</td>
|
|
<td><p>Do not preserve the order of the elements, yield them as soon as they
|
|
have been processed. This decreases the latency caused by
|
|
difficult-to-process items.</p></td>
|
|
<td>
|
|
<code>False</code>
|
|
</td>
|
|
</tr>
|
|
</tbody>
|
|
</table>
|
|
|
|
<p><strong>Yields:</strong></p>
|
|
<table>
|
|
<thead>
|
|
<tr>
|
|
<th>Type</th>
|
|
<th>Description</th>
|
|
</tr>
|
|
</thead>
|
|
<tbody>
|
|
<tr>
|
|
<td>
|
|
<code><span title="typing.Iterable">Iterable</span>[<span title="typing.Optional">Optional</span>[<span title="great_ai.utilities.parallel_map.threaded_parallel_map.V">V</span>]]</code>
|
|
</td>
|
|
<td><p>The next result obtained from applying <code>func</code> to each input value. May
|
|
contain <code>None</code>-s if <code>ignore_exceptions=True</code>. May have different order than
|
|
the input if <code>unordered=True</code>.</p></td>
|
|
</tr>
|
|
</tbody>
|
|
</table>
|
|
|
|
<p><strong>Raises:</strong></p>
|
|
<table>
|
|
<thead>
|
|
<tr>
|
|
<th>Type</th>
|
|
<th>Description</th>
|
|
</tr>
|
|
</thead>
|
|
<tbody>
|
|
<tr>
|
|
<td>
|
|
<code><span title="great_ai.utilities.parallel_map.worker_exception.WorkerException">WorkerException</span></code>
|
|
</td>
|
|
<td><p>If there was an error in the <code>func</code> function in a background
|
|
thread and <code>ignore_exceptions=False</code>.</p></td>
|
|
</tr>
|
|
</tbody>
|
|
</table>
|
|
|
|
<details class="quote">
|
|
<summary>Source code in <code>great_ai/utilities/parallel_map/threaded_parallel_map.py</code></summary>
|
|
<div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"><a href="#__codelineno-0-75"> 75</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-76"> 76</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-77"> 77</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-78"> 78</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-79"> 79</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-80"> 80</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-81"> 81</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-82"> 82</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-83"> 83</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-84"> 84</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-85"> 85</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-86"> 86</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-87"> 87</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-88"> 88</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-89"> 89</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-90"> 90</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-91"> 91</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-92"> 92</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-93"> 93</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-94"> 94</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-95"> 95</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-96"> 96</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-97"> 97</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-98"> 98</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-99"> 99</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-100">100</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-101">101</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-102">102</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-103">103</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-104">104</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-105">105</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-106">106</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-107">107</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-108">108</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-109">109</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-110">110</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-111">111</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-112">112</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-113">113</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-114">114</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-115">115</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-116">116</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-117">117</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-118">118</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-119">119</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-120">120</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-121">121</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-122">122</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-123">123</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-124">124</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-125">125</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-126">126</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-127">127</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-128">128</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-129">129</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-130">130</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-131">131</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-132">132</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-133">133</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-134">134</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-135">135</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-136">136</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-137">137</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-138">138</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-139">139</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-140">140</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-141">141</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-142">142</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-143">143</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-144">144</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-145">145</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-146">146</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-147">147</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-148">148</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-149">149</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-150">150</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-151">151</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-152">152</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-153">153</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-154">154</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-155">155</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-156">156</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-157">157</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-158">158</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-159">159</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-160">160</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-161">161</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-162">162</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-163">163</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-164">164</a></span></pre></div></td><td class="code"><div><pre><span></span><code><a id="__codelineno-0-75" name="__codelineno-0-75"></a><span class="k">def</span> <span class="nf">threaded_parallel_map</span><span class="p">(</span>
|
|
<a id="__codelineno-0-76" name="__codelineno-0-76"></a> <span class="n">func</span><span class="p">:</span> <span class="n">Callable</span><span class="p">[[</span><span class="n">T</span><span class="p">],</span> <span class="n">Union</span><span class="p">[</span><span class="n">V</span><span class="p">,</span> <span class="n">Awaitable</span><span class="p">[</span><span class="n">V</span><span class="p">]]],</span>
|
|
<a id="__codelineno-0-77" name="__codelineno-0-77"></a> <span class="n">input_values</span><span class="p">:</span> <span class="n">Union</span><span class="p">[</span><span class="n">Iterable</span><span class="p">[</span><span class="n">T</span><span class="p">],</span> <span class="n">Sequence</span><span class="p">[</span><span class="n">T</span><span class="p">]],</span>
|
|
<a id="__codelineno-0-78" name="__codelineno-0-78"></a> <span class="o">*</span><span class="p">,</span>
|
|
<a id="__codelineno-0-79" name="__codelineno-0-79"></a> <span class="n">chunk_size</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">int</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
|
|
<a id="__codelineno-0-80" name="__codelineno-0-80"></a> <span class="n">ignore_exceptions</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">False</span><span class="p">,</span>
|
|
<a id="__codelineno-0-81" name="__codelineno-0-81"></a> <span class="n">concurrency</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">int</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
|
|
<a id="__codelineno-0-82" name="__codelineno-0-82"></a> <span class="n">unordered</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">False</span><span class="p">,</span>
|
|
<a id="__codelineno-0-83" name="__codelineno-0-83"></a><span class="p">)</span> <span class="o">-></span> <span class="n">Iterable</span><span class="p">[</span><span class="n">Optional</span><span class="p">[</span><span class="n">V</span><span class="p">]]:</span>
|
|
<a id="__codelineno-0-84" name="__codelineno-0-84"></a> <span class="sd">"""Execute a map operation on an iterable stream.</span>
|
|
<a id="__codelineno-0-85" name="__codelineno-0-85"></a>
|
|
<a id="__codelineno-0-86" name="__codelineno-0-86"></a><span class="sd"> Similar to [parallel_map][great_ai.utilities.parallel_map.parallel_map.parallel_map]</span>
|
|
<a id="__codelineno-0-87" name="__codelineno-0-87"></a><span class="sd"> but uses threads instead of processes. Hence, it is not helpful in CPU-bound</span>
|
|
<a id="__codelineno-0-88" name="__codelineno-0-88"></a><span class="sd"> situations.</span>
|
|
<a id="__codelineno-0-89" name="__codelineno-0-89"></a>
|
|
<a id="__codelineno-0-90" name="__codelineno-0-90"></a><span class="sd"> A custom parallel map operation supporting both synchronous and `async` map</span>
|
|
<a id="__codelineno-0-91" name="__codelineno-0-91"></a><span class="sd"> functions. Exceptions encountered in the map function are sent to the host thread</span>
|
|
<a id="__codelineno-0-92" name="__codelineno-0-92"></a><span class="sd"> where they are either raised (default) or ignored. Each process processes a single</span>
|
|
<a id="__codelineno-0-93" name="__codelineno-0-93"></a><span class="sd"> chunk at once.</span>
|
|
<a id="__codelineno-0-94" name="__codelineno-0-94"></a>
|
|
<a id="__codelineno-0-95" name="__codelineno-0-95"></a><span class="sd"> Examples:</span>
|
|
<a id="__codelineno-0-96" name="__codelineno-0-96"></a><span class="sd"> >>> list(threaded_parallel_map(lambda x: x ** 2, [1, 2, 3]))</span>
|
|
<a id="__codelineno-0-97" name="__codelineno-0-97"></a><span class="sd"> [1, 4, 9]</span>
|
|
<a id="__codelineno-0-98" name="__codelineno-0-98"></a>
|
|
<a id="__codelineno-0-99" name="__codelineno-0-99"></a><span class="sd"> Args:</span>
|
|
<a id="__codelineno-0-100" name="__codelineno-0-100"></a><span class="sd"> func: The function that should be applied to each element of `input_values`.</span>
|
|
<a id="__codelineno-0-101" name="__codelineno-0-101"></a><span class="sd"> It can `async`, in that case, a new event loop is started for each chunk.</span>
|
|
<a id="__codelineno-0-102" name="__codelineno-0-102"></a><span class="sd"> input_values: An iterable of items that `func` is applied to.</span>
|
|
<a id="__codelineno-0-103" name="__codelineno-0-103"></a><span class="sd"> chunk_size: Tune the number of items processed in each step. Larger numbers</span>
|
|
<a id="__codelineno-0-104" name="__codelineno-0-104"></a><span class="sd"> result in smaller communication overhead but less parallelism at the start</span>
|
|
<a id="__codelineno-0-105" name="__codelineno-0-105"></a><span class="sd"> and end. If `chunk_size` has a `__len__` property, the `chunk_size` is</span>
|
|
<a id="__codelineno-0-106" name="__codelineno-0-106"></a><span class="sd"> calculated automatically if not given.</span>
|
|
<a id="__codelineno-0-107" name="__codelineno-0-107"></a><span class="sd"> ignore_exceptions: Ignore chunks if `next()` raises an exception on</span>
|
|
<a id="__codelineno-0-108" name="__codelineno-0-108"></a><span class="sd"> `input_values`. And return `None` if `func` raised an exception in a worker</span>
|
|
<a id="__codelineno-0-109" name="__codelineno-0-109"></a><span class="sd"> process.</span>
|
|
<a id="__codelineno-0-110" name="__codelineno-0-110"></a><span class="sd"> concurrency: Number of new threads to start.</span>
|
|
<a id="__codelineno-0-111" name="__codelineno-0-111"></a><span class="sd"> unordered: Do not preserve the order of the elements, yield them as soon as they</span>
|
|
<a id="__codelineno-0-112" name="__codelineno-0-112"></a><span class="sd"> have been processed. This decreases the latency caused by</span>
|
|
<a id="__codelineno-0-113" name="__codelineno-0-113"></a><span class="sd"> difficult-to-process items.</span>
|
|
<a id="__codelineno-0-114" name="__codelineno-0-114"></a>
|
|
<a id="__codelineno-0-115" name="__codelineno-0-115"></a><span class="sd"> Yields:</span>
|
|
<a id="__codelineno-0-116" name="__codelineno-0-116"></a><span class="sd"> The next result obtained from applying `func` to each input value. May</span>
|
|
<a id="__codelineno-0-117" name="__codelineno-0-117"></a><span class="sd"> contain `None`-s if `ignore_exceptions=True`. May have different order than</span>
|
|
<a id="__codelineno-0-118" name="__codelineno-0-118"></a><span class="sd"> the input if `unordered=True`.</span>
|
|
<a id="__codelineno-0-119" name="__codelineno-0-119"></a>
|
|
<a id="__codelineno-0-120" name="__codelineno-0-120"></a><span class="sd"> Raises:</span>
|
|
<a id="__codelineno-0-121" name="__codelineno-0-121"></a><span class="sd"> WorkerException: If there was an error in the `func` function in a background</span>
|
|
<a id="__codelineno-0-122" name="__codelineno-0-122"></a><span class="sd"> thread and `ignore_exceptions=False`.</span>
|
|
<a id="__codelineno-0-123" name="__codelineno-0-123"></a><span class="sd"> """</span>
|
|
<a id="__codelineno-0-124" name="__codelineno-0-124"></a>
|
|
<a id="__codelineno-0-125" name="__codelineno-0-125"></a> <span class="n">config</span> <span class="o">=</span> <span class="n">get_config</span><span class="p">(</span>
|
|
<a id="__codelineno-0-126" name="__codelineno-0-126"></a> <span class="n">function</span><span class="o">=</span><span class="n">func</span><span class="p">,</span>
|
|
<a id="__codelineno-0-127" name="__codelineno-0-127"></a> <span class="n">input_values</span><span class="o">=</span><span class="n">input_values</span><span class="p">,</span>
|
|
<a id="__codelineno-0-128" name="__codelineno-0-128"></a> <span class="n">chunk_size</span><span class="o">=</span><span class="n">chunk_size</span><span class="p">,</span>
|
|
<a id="__codelineno-0-129" name="__codelineno-0-129"></a> <span class="n">concurrency</span><span class="o">=</span><span class="n">concurrency</span><span class="p">,</span>
|
|
<a id="__codelineno-0-130" name="__codelineno-0-130"></a> <span class="p">)</span>
|
|
<a id="__codelineno-0-131" name="__codelineno-0-131"></a>
|
|
<a id="__codelineno-0-132" name="__codelineno-0-132"></a> <span class="n">input_queue</span><span class="p">:</span> <span class="n">queue</span><span class="o">.</span><span class="n">Queue</span> <span class="o">=</span> <span class="n">queue</span><span class="o">.</span><span class="n">Queue</span><span class="p">(</span><span class="n">config</span><span class="o">.</span><span class="n">concurrency</span> <span class="o">*</span> <span class="mi">2</span><span class="p">)</span>
|
|
<a id="__codelineno-0-133" name="__codelineno-0-133"></a> <span class="n">output_queue</span><span class="p">:</span> <span class="n">queue</span><span class="o">.</span><span class="n">Queue</span> <span class="o">=</span> <span class="n">queue</span><span class="o">.</span><span class="n">Queue</span><span class="p">(</span><span class="n">config</span><span class="o">.</span><span class="n">concurrency</span> <span class="o">*</span> <span class="mi">2</span><span class="p">)</span>
|
|
<a id="__codelineno-0-134" name="__codelineno-0-134"></a> <span class="n">should_stop</span> <span class="o">=</span> <span class="n">threading</span><span class="o">.</span><span class="n">Event</span><span class="p">()</span>
|
|
<a id="__codelineno-0-135" name="__codelineno-0-135"></a>
|
|
<a id="__codelineno-0-136" name="__codelineno-0-136"></a> <span class="n">threads</span> <span class="o">=</span> <span class="p">[</span>
|
|
<a id="__codelineno-0-137" name="__codelineno-0-137"></a> <span class="n">threading</span><span class="o">.</span><span class="n">Thread</span><span class="p">(</span>
|
|
<a id="__codelineno-0-138" name="__codelineno-0-138"></a> <span class="n">name</span><span class="o">=</span><span class="sa">f</span><span class="s2">"threaded_parallel_map_</span><span class="si">{</span><span class="n">config</span><span class="o">.</span><span class="n">function_name</span><span class="si">}</span><span class="s2">_</span><span class="si">{</span><span class="n">i</span><span class="si">}</span><span class="s2">"</span><span class="p">,</span>
|
|
<a id="__codelineno-0-139" name="__codelineno-0-139"></a> <span class="n">target</span><span class="o">=</span><span class="n">mapper_function</span><span class="p">,</span>
|
|
<a id="__codelineno-0-140" name="__codelineno-0-140"></a> <span class="n">daemon</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span>
|
|
<a id="__codelineno-0-141" name="__codelineno-0-141"></a> <span class="n">kwargs</span><span class="o">=</span><span class="nb">dict</span><span class="p">(</span>
|
|
<a id="__codelineno-0-142" name="__codelineno-0-142"></a> <span class="n">input_queue</span><span class="o">=</span><span class="n">input_queue</span><span class="p">,</span>
|
|
<a id="__codelineno-0-143" name="__codelineno-0-143"></a> <span class="n">output_queue</span><span class="o">=</span><span class="n">output_queue</span><span class="p">,</span>
|
|
<a id="__codelineno-0-144" name="__codelineno-0-144"></a> <span class="n">should_stop</span><span class="o">=</span><span class="n">should_stop</span><span class="p">,</span>
|
|
<a id="__codelineno-0-145" name="__codelineno-0-145"></a> <span class="n">func</span><span class="o">=</span><span class="n">func</span><span class="p">,</span>
|
|
<a id="__codelineno-0-146" name="__codelineno-0-146"></a> <span class="p">),</span>
|
|
<a id="__codelineno-0-147" name="__codelineno-0-147"></a> <span class="p">)</span>
|
|
<a id="__codelineno-0-148" name="__codelineno-0-148"></a> <span class="k">for</span> <span class="n">i</span> <span class="ow">in</span> <span class="nb">range</span><span class="p">(</span><span class="n">config</span><span class="o">.</span><span class="n">concurrency</span><span class="p">)</span>
|
|
<a id="__codelineno-0-149" name="__codelineno-0-149"></a> <span class="p">]</span>
|
|
<a id="__codelineno-0-150" name="__codelineno-0-150"></a>
|
|
<a id="__codelineno-0-151" name="__codelineno-0-151"></a> <span class="k">for</span> <span class="n">t</span> <span class="ow">in</span> <span class="n">threads</span><span class="p">:</span>
|
|
<a id="__codelineno-0-152" name="__codelineno-0-152"></a> <span class="n">t</span><span class="o">.</span><span class="n">start</span><span class="p">()</span>
|
|
<a id="__codelineno-0-153" name="__codelineno-0-153"></a>
|
|
<a id="__codelineno-0-154" name="__codelineno-0-154"></a> <span class="k">yield from</span> <span class="n">manage_communication</span><span class="p">(</span>
|
|
<a id="__codelineno-0-155" name="__codelineno-0-155"></a> <span class="n">input_values</span><span class="o">=</span><span class="n">input_values</span><span class="p">,</span>
|
|
<a id="__codelineno-0-156" name="__codelineno-0-156"></a> <span class="n">chunk_size</span><span class="o">=</span><span class="n">config</span><span class="o">.</span><span class="n">chunk_size</span><span class="p">,</span>
|
|
<a id="__codelineno-0-157" name="__codelineno-0-157"></a> <span class="n">input_queue</span><span class="o">=</span><span class="n">input_queue</span><span class="p">,</span>
|
|
<a id="__codelineno-0-158" name="__codelineno-0-158"></a> <span class="n">output_queue</span><span class="o">=</span><span class="n">output_queue</span><span class="p">,</span>
|
|
<a id="__codelineno-0-159" name="__codelineno-0-159"></a> <span class="n">unordered</span><span class="o">=</span><span class="n">unordered</span><span class="p">,</span>
|
|
<a id="__codelineno-0-160" name="__codelineno-0-160"></a> <span class="n">ignore_exceptions</span><span class="o">=</span><span class="n">ignore_exceptions</span><span class="p">,</span>
|
|
<a id="__codelineno-0-161" name="__codelineno-0-161"></a> <span class="p">)</span>
|
|
<a id="__codelineno-0-162" name="__codelineno-0-162"></a> <span class="n">should_stop</span><span class="o">.</span><span class="n">set</span><span class="p">()</span>
|
|
<a id="__codelineno-0-163" name="__codelineno-0-163"></a> <span class="k">for</span> <span class="n">t</span> <span class="ow">in</span> <span class="n">threads</span><span class="p">:</span>
|
|
<a id="__codelineno-0-164" name="__codelineno-0-164"></a> <span class="n">t</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="mi">1</span><span class="p">)</span>
|
|
</code></pre></div></td></tr></table></div>
|
|
</details>
|
|
</div>
|
|
|
|
</div><h2 id="composable-parallel-processing">Composable parallel processing<a class="headerlink" href="#composable-parallel-processing" title="Permanent link">#</a></h2>
|
|
<p>Because both <a class="autorefs autorefs-internal" href="#great_ai.utilities.threaded_parallel_map">threaded_parallel_map</a> and [parallel_map][great_ai.utilities.parallel_map.parallel_map] have a streaming interface, it is easy to compose them and end up with, for example, a process for each CPU core with its own thread-pool or event-loop. Longer pipelines are also easy to imagine. The chunking methods help in these compositions.</p>
|
|
<p>For more info, check-out <a href="/how-to-guides/scraping">the scraping guide</a>.</p>
|
|
|
|
|
|
<div class="doc doc-object doc-module">
|
|
|
|
|
|
|
|
<div class="doc doc-contents first">
|
|
|
|
|
|
|
|
<div class="doc doc-children">
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
<div class="doc doc-object doc-function">
|
|
|
|
|
|
|
|
<h3 id="great_ai.utilities.chunk.chunk" class="doc doc-heading">
|
|
<code class="highlight language-python"><span class="n">chunk</span><span class="p">(</span><span class="n">values</span><span class="p">,</span> <span class="n">chunk_size</span><span class="p">)</span></code>
|
|
|
|
<a href="#great_ai.utilities.chunk.chunk" class="headerlink" title="Permanent link">#</a></h3>
|
|
|
|
|
|
<div class="doc doc-contents ">
|
|
|
|
<p>Turn an iterable of items into an iterable of lists (chunks) of items.</p>
|
|
<p>Each returned chunk is of length <code>chunk_size</code> except the last one the length of
|
|
which is between 1 and <code>chunk_size</code>.</p>
|
|
<p>Useful for parallel processing.</p>
|
|
|
|
<p><strong>Examples:</strong></p>
|
|
<div class="highlight"><pre><span></span><code><a id="__codelineno-0-1" name="__codelineno-0-1" href="#__codelineno-0-1"></a><span class="gp">>>> </span><span class="nb">list</span><span class="p">(</span><span class="n">chunk</span><span class="p">(</span><span class="nb">range</span><span class="p">(</span><span class="mi">10</span><span class="p">),</span> <span class="n">chunk_size</span><span class="o">=</span><span class="mi">3</span><span class="p">))</span>
|
|
<a id="__codelineno-0-2" name="__codelineno-0-2" href="#__codelineno-0-2"></a><span class="go">[[0, 1, 2], [3, 4, 5], [6, 7, 8], [9]]</span>
|
|
</code></pre></div>
|
|
|
|
<p><strong>Parameters:</strong></p>
|
|
<table>
|
|
<thead>
|
|
<tr>
|
|
<th>Name</th>
|
|
<th>Type</th>
|
|
<th>Description</th>
|
|
<th>Default</th>
|
|
</tr>
|
|
</thead>
|
|
<tbody>
|
|
<tr>
|
|
<td><code>values</code></td>
|
|
<td>
|
|
<code><span title="typing.Iterable">Iterable</span>[<span title="great_ai.utilities.chunk.T">T</span>]</code>
|
|
</td>
|
|
<td><p>The stream of items to pack into chunks.</p></td>
|
|
<td>
|
|
<em>required</em>
|
|
</td>
|
|
</tr>
|
|
<tr>
|
|
<td><code>chunk_size</code></td>
|
|
<td>
|
|
<code>int</code>
|
|
</td>
|
|
<td><p>Desired length of each (but the last) chunk.</p></td>
|
|
<td>
|
|
<em>required</em>
|
|
</td>
|
|
</tr>
|
|
</tbody>
|
|
</table>
|
|
|
|
<p><strong>Yields:</strong></p>
|
|
<table>
|
|
<thead>
|
|
<tr>
|
|
<th>Type</th>
|
|
<th>Description</th>
|
|
</tr>
|
|
</thead>
|
|
<tbody>
|
|
<tr>
|
|
<td>
|
|
<code><span title="typing.Iterable">Iterable</span>[<span title="typing.List">List</span>[<span title="great_ai.utilities.chunk.T">T</span>]]</code>
|
|
</td>
|
|
<td><p>The next chunk.</p></td>
|
|
</tr>
|
|
</tbody>
|
|
</table>
|
|
|
|
<details class="quote">
|
|
<summary>Source code in <code>great_ai/utilities/chunk.py</code></summary>
|
|
<div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"><a href="#__codelineno-0-6"> 6</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-7"> 7</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-8"> 8</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-9"> 9</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-10">10</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-11">11</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-12">12</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-13">13</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-14">14</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-15">15</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-16">16</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-17">17</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-18">18</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-19">19</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-20">20</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-21">21</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-22">22</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-23">23</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-24">24</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-25">25</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-26">26</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-27">27</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-28">28</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-29">29</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-30">30</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-31">31</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-32">32</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-33">33</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-34">34</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-35">35</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-36">36</a></span></pre></div></td><td class="code"><div><pre><span></span><code><a id="__codelineno-0-6" name="__codelineno-0-6"></a><span class="k">def</span> <span class="nf">chunk</span><span class="p">(</span><span class="n">values</span><span class="p">:</span> <span class="n">Iterable</span><span class="p">[</span><span class="n">T</span><span class="p">],</span> <span class="n">chunk_size</span><span class="p">:</span> <span class="nb">int</span><span class="p">)</span> <span class="o">-></span> <span class="n">Iterable</span><span class="p">[</span><span class="n">List</span><span class="p">[</span><span class="n">T</span><span class="p">]]:</span>
|
|
<a id="__codelineno-0-7" name="__codelineno-0-7"></a> <span class="sd">"""Turn an iterable of items into an iterable of lists (chunks) of items.</span>
|
|
<a id="__codelineno-0-8" name="__codelineno-0-8"></a>
|
|
<a id="__codelineno-0-9" name="__codelineno-0-9"></a><span class="sd"> Each returned chunk is of length `chunk_size` except the last one the length of</span>
|
|
<a id="__codelineno-0-10" name="__codelineno-0-10"></a><span class="sd"> which is between 1 and `chunk_size`.</span>
|
|
<a id="__codelineno-0-11" name="__codelineno-0-11"></a>
|
|
<a id="__codelineno-0-12" name="__codelineno-0-12"></a><span class="sd"> Useful for parallel processing.</span>
|
|
<a id="__codelineno-0-13" name="__codelineno-0-13"></a>
|
|
<a id="__codelineno-0-14" name="__codelineno-0-14"></a><span class="sd"> Examples:</span>
|
|
<a id="__codelineno-0-15" name="__codelineno-0-15"></a><span class="sd"> >>> list(chunk(range(10), chunk_size=3))</span>
|
|
<a id="__codelineno-0-16" name="__codelineno-0-16"></a><span class="sd"> [[0, 1, 2], [3, 4, 5], [6, 7, 8], [9]]</span>
|
|
<a id="__codelineno-0-17" name="__codelineno-0-17"></a>
|
|
<a id="__codelineno-0-18" name="__codelineno-0-18"></a><span class="sd"> Args:</span>
|
|
<a id="__codelineno-0-19" name="__codelineno-0-19"></a><span class="sd"> values: The stream of items to pack into chunks.</span>
|
|
<a id="__codelineno-0-20" name="__codelineno-0-20"></a><span class="sd"> chunk_size: Desired length of each (but the last) chunk.</span>
|
|
<a id="__codelineno-0-21" name="__codelineno-0-21"></a>
|
|
<a id="__codelineno-0-22" name="__codelineno-0-22"></a><span class="sd"> Yields:</span>
|
|
<a id="__codelineno-0-23" name="__codelineno-0-23"></a><span class="sd"> The next chunk.</span>
|
|
<a id="__codelineno-0-24" name="__codelineno-0-24"></a><span class="sd"> """</span>
|
|
<a id="__codelineno-0-25" name="__codelineno-0-25"></a>
|
|
<a id="__codelineno-0-26" name="__codelineno-0-26"></a> <span class="k">assert</span> <span class="n">chunk_size</span> <span class="o">>=</span> <span class="mi">1</span>
|
|
<a id="__codelineno-0-27" name="__codelineno-0-27"></a>
|
|
<a id="__codelineno-0-28" name="__codelineno-0-28"></a> <span class="n">result</span><span class="p">:</span> <span class="n">List</span><span class="p">[</span><span class="n">T</span><span class="p">]</span> <span class="o">=</span> <span class="p">[]</span>
|
|
<a id="__codelineno-0-29" name="__codelineno-0-29"></a> <span class="k">for</span> <span class="n">v</span> <span class="ow">in</span> <span class="n">values</span><span class="p">:</span>
|
|
<a id="__codelineno-0-30" name="__codelineno-0-30"></a> <span class="n">result</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">v</span><span class="p">)</span>
|
|
<a id="__codelineno-0-31" name="__codelineno-0-31"></a> <span class="k">if</span> <span class="nb">len</span><span class="p">(</span><span class="n">result</span><span class="p">)</span> <span class="o">==</span> <span class="n">chunk_size</span><span class="p">:</span>
|
|
<a id="__codelineno-0-32" name="__codelineno-0-32"></a> <span class="k">yield</span> <span class="n">result</span>
|
|
<a id="__codelineno-0-33" name="__codelineno-0-33"></a> <span class="n">result</span> <span class="o">=</span> <span class="p">[]</span>
|
|
<a id="__codelineno-0-34" name="__codelineno-0-34"></a>
|
|
<a id="__codelineno-0-35" name="__codelineno-0-35"></a> <span class="k">if</span> <span class="nb">len</span><span class="p">(</span><span class="n">result</span><span class="p">)</span> <span class="o">></span> <span class="mi">0</span><span class="p">:</span>
|
|
<a id="__codelineno-0-36" name="__codelineno-0-36"></a> <span class="k">yield</span> <span class="n">result</span>
|
|
</code></pre></div></td></tr></table></div>
|
|
</details>
|
|
</div>
|
|
|
|
</div>
|
|
|
|
|
|
|
|
</div>
|
|
|
|
</div>
|
|
|
|
</div>
|
|
|
|
<div class="doc doc-object doc-module">
|
|
|
|
|
|
|
|
<div class="doc doc-contents first">
|
|
|
|
|
|
|
|
<div class="doc doc-children">
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
<div class="doc doc-object doc-function">
|
|
|
|
|
|
|
|
<h3 id="great_ai.utilities.unchunk.unchunk" class="doc doc-heading">
|
|
<code class="highlight language-python"><span class="n">unchunk</span><span class="p">(</span><span class="n">chunks</span><span class="p">)</span></code>
|
|
|
|
<a href="#great_ai.utilities.unchunk.unchunk" class="headerlink" title="Permanent link">#</a></h3>
|
|
|
|
|
|
<div class="doc doc-contents ">
|
|
|
|
<p>Turn a stream of chunks of items into a stream of items (flatten operation).</p>
|
|
<p>The inverse operation of <a class="autorefs autorefs-internal" href="#great_ai.utilities.chunk.chunk">chunk</a>.
|
|
Useful for parallel processing.</p>
|
|
<p>Similar to itertools.chain but ignores <code>None</code> chunks.</p>
|
|
|
|
<p><strong>Examples:</strong></p>
|
|
<div class="highlight"><pre><span></span><code><a id="__codelineno-0-1" name="__codelineno-0-1" href="#__codelineno-0-1"></a><span class="gp">>>> </span><span class="nb">list</span><span class="p">(</span><span class="n">unchunk</span><span class="p">([[</span><span class="mi">0</span><span class="p">,</span> <span class="mi">1</span><span class="p">,</span> <span class="mi">2</span><span class="p">],</span> <span class="p">[</span><span class="mi">3</span><span class="p">,</span> <span class="mi">4</span><span class="p">,</span> <span class="mi">5</span><span class="p">],</span> <span class="p">[</span><span class="mi">6</span><span class="p">,</span> <span class="mi">7</span><span class="p">,</span> <span class="mi">8</span><span class="p">],</span> <span class="p">[</span><span class="mi">9</span><span class="p">]]))</span>
|
|
<a id="__codelineno-0-2" name="__codelineno-0-2" href="#__codelineno-0-2"></a><span class="go">[0, 1, 2, 3, 4, 5, 6, 7, 8, 9]</span>
|
|
</code></pre></div>
|
|
|
|
<p><strong>Parameters:</strong></p>
|
|
<table>
|
|
<thead>
|
|
<tr>
|
|
<th>Name</th>
|
|
<th>Type</th>
|
|
<th>Description</th>
|
|
<th>Default</th>
|
|
</tr>
|
|
</thead>
|
|
<tbody>
|
|
<tr>
|
|
<td><code>chunks</code></td>
|
|
<td>
|
|
<code><span title="typing.Iterable">Iterable</span>[<span title="typing.Optional">Optional</span>[<span title="typing.Iterable">Iterable</span>[<span title="great_ai.utilities.unchunk.T">T</span>]]]</code>
|
|
</td>
|
|
<td><p>Stream of chunks to unpack.</p></td>
|
|
<td>
|
|
<em>required</em>
|
|
</td>
|
|
</tr>
|
|
</tbody>
|
|
</table>
|
|
|
|
<p><strong>Yields:</strong></p>
|
|
<table>
|
|
<thead>
|
|
<tr>
|
|
<th>Type</th>
|
|
<th>Description</th>
|
|
</tr>
|
|
</thead>
|
|
<tbody>
|
|
<tr>
|
|
<td>
|
|
<code><span title="typing.Iterable">Iterable</span>[<span title="great_ai.utilities.unchunk.T">T</span>]</code>
|
|
</td>
|
|
<td><p>The next item in the flattened iterable.</p></td>
|
|
</tr>
|
|
</tbody>
|
|
</table>
|
|
|
|
<details class="quote">
|
|
<summary>Source code in <code>great_ai/utilities/unchunk.py</code></summary>
|
|
<div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"><a href="#__codelineno-0-6"> 6</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-7"> 7</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-8"> 8</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-9"> 9</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-10">10</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-11">11</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-12">12</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-13">13</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-14">14</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-15">15</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-16">16</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-17">17</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-18">18</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-19">19</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-20">20</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-21">21</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-22">22</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-23">23</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-24">24</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-25">25</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-26">26</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-27">27</a></span></pre></div></td><td class="code"><div><pre><span></span><code><a id="__codelineno-0-6" name="__codelineno-0-6"></a><span class="k">def</span> <span class="nf">unchunk</span><span class="p">(</span><span class="n">chunks</span><span class="p">:</span> <span class="n">Iterable</span><span class="p">[</span><span class="n">Optional</span><span class="p">[</span><span class="n">Iterable</span><span class="p">[</span><span class="n">T</span><span class="p">]]])</span> <span class="o">-></span> <span class="n">Iterable</span><span class="p">[</span><span class="n">T</span><span class="p">]:</span>
|
|
<a id="__codelineno-0-7" name="__codelineno-0-7"></a> <span class="sd">"""Turn a stream of chunks of items into a stream of items (flatten operation).</span>
|
|
<a id="__codelineno-0-8" name="__codelineno-0-8"></a>
|
|
<a id="__codelineno-0-9" name="__codelineno-0-9"></a><span class="sd"> The inverse operation of [chunk][great_ai.utilities.chunk.chunk].</span>
|
|
<a id="__codelineno-0-10" name="__codelineno-0-10"></a><span class="sd"> Useful for parallel processing.</span>
|
|
<a id="__codelineno-0-11" name="__codelineno-0-11"></a>
|
|
<a id="__codelineno-0-12" name="__codelineno-0-12"></a><span class="sd"> Similar to itertools.chain but ignores `None` chunks.</span>
|
|
<a id="__codelineno-0-13" name="__codelineno-0-13"></a>
|
|
<a id="__codelineno-0-14" name="__codelineno-0-14"></a><span class="sd"> Examples:</span>
|
|
<a id="__codelineno-0-15" name="__codelineno-0-15"></a><span class="sd"> >>> list(unchunk([[0, 1, 2], [3, 4, 5], [6, 7, 8], [9]]))</span>
|
|
<a id="__codelineno-0-16" name="__codelineno-0-16"></a><span class="sd"> [0, 1, 2, 3, 4, 5, 6, 7, 8, 9]</span>
|
|
<a id="__codelineno-0-17" name="__codelineno-0-17"></a>
|
|
<a id="__codelineno-0-18" name="__codelineno-0-18"></a><span class="sd"> Args:</span>
|
|
<a id="__codelineno-0-19" name="__codelineno-0-19"></a><span class="sd"> chunks: Stream of chunks to unpack.</span>
|
|
<a id="__codelineno-0-20" name="__codelineno-0-20"></a>
|
|
<a id="__codelineno-0-21" name="__codelineno-0-21"></a><span class="sd"> Yields:</span>
|
|
<a id="__codelineno-0-22" name="__codelineno-0-22"></a><span class="sd"> The next item in the flattened iterable.</span>
|
|
<a id="__codelineno-0-23" name="__codelineno-0-23"></a><span class="sd"> """</span>
|
|
<a id="__codelineno-0-24" name="__codelineno-0-24"></a>
|
|
<a id="__codelineno-0-25" name="__codelineno-0-25"></a> <span class="k">for</span> <span class="n">chunk</span> <span class="ow">in</span> <span class="n">chunks</span><span class="p">:</span>
|
|
<a id="__codelineno-0-26" name="__codelineno-0-26"></a> <span class="k">if</span> <span class="n">chunk</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
|
|
<a id="__codelineno-0-27" name="__codelineno-0-27"></a> <span class="k">yield from</span> <span class="n">chunk</span>
|
|
</code></pre></div></td></tr></table></div>
|
|
</details>
|
|
</div>
|
|
|
|
</div>
|
|
|
|
|
|
|
|
</div>
|
|
|
|
</div>
|
|
|
|
</div><h2 id="operations">Operations<a class="headerlink" href="#operations" title="Permanent link">#</a></h2>
|
|
|
|
|
|
<div class="doc doc-object doc-class">
|
|
|
|
|
|
|
|
<h3 id="great_ai.utilities.ConfigFile" class="doc doc-heading">
|
|
<code>ConfigFile</code>
|
|
|
|
|
|
<a href="#great_ai.utilities.ConfigFile" class="headerlink" title="Permanent link">#</a></h3>
|
|
|
|
|
|
<div class="doc doc-contents first">
|
|
<p class="doc doc-class-bases">
|
|
Bases: <code><span title="typing.Mapping">Mapping</span>[str, str]</code></p>
|
|
|
|
|
|
<p>A small and safe <code>INI</code>-style configuration loader with <code>dict</code> and <code>ENV</code> support.</p>
|
|
<p>The values can be accessed using both dot- and index-notation. It is compatible
|
|
with the <code>dict</code> interface.</p>
|
|
<p>File format example:</p>
|
|
<div class="highlight"><pre><span></span><code><a id="__codelineno-0-1" name="__codelineno-0-1" href="#__codelineno-0-1"></a><span class="c1"># comments are allowed everywhere</span><span class="w"></span>
|
|
<a id="__codelineno-0-2" name="__codelineno-0-2" href="#__codelineno-0-2"></a>
|
|
<a id="__codelineno-0-3" name="__codelineno-0-3" href="#__codelineno-0-3"></a><span class="n">key</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="n">value</span><span class="w"> </span><span class="c1"># you can leave or omit whitespace around the equal-sign</span><span class="w"></span>
|
|
<a id="__codelineno-0-4" name="__codelineno-0-4" href="#__codelineno-0-4"></a><span class="n">my_hashtag</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="s">"#great_ai"</span><span class="w"> </span><span class="c1"># the r-value can be quoted with " or ' or `.</span><span class="w"></span>
|
|
<a id="__codelineno-0-5" name="__codelineno-0-5" href="#__codelineno-0-5"></a>
|
|
<a id="__codelineno-0-6" name="__codelineno-0-6" href="#__codelineno-0-6"></a><span class="n">my_var</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="n">my_default_value</span><span class="w"> </span><span class="c1"># Default values can be given to env-vars,</span><span class="w"></span>
|
|
<a id="__codelineno-0-7" name="__codelineno-0-7" href="#__codelineno-0-7"></a><span class="w"> </span><span class="c1"># see next line. The default value must come first.</span><span class="w"></span>
|
|
<a id="__codelineno-0-8" name="__codelineno-0-8" href="#__codelineno-0-8"></a>
|
|
<a id="__codelineno-0-9" name="__codelineno-0-9" href="#__codelineno-0-9"></a><span class="n">my_var</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="n">ENV</span><span class="p">:</span><span class="n">MY_ENV_VAR</span><span class="w"> </span><span class="c1"># If the value starts with the `ENV:` prefix,</span><span class="w"></span>
|
|
<a id="__codelineno-0-10" name="__codelineno-0-10" href="#__codelineno-0-10"></a><span class="w"> </span><span class="c1"># it is looked up from the environment variables.</span><span class="w"></span>
|
|
</code></pre></div>
|
|
|
|
<p><strong>Examples:</strong></p>
|
|
<div class="highlight"><pre><span></span><code><a id="__codelineno-0-1" name="__codelineno-0-1" href="#__codelineno-0-1"></a><span class="gp">>>> </span><span class="n">ConfigFile</span><span class="p">(</span><span class="s1">'tests/utilities/data/simple.conf'</span><span class="p">)</span>
|
|
<a id="__codelineno-0-2" name="__codelineno-0-2" href="#__codelineno-0-2"></a><span class="go">ConfigFile(path=tests/utilities/data/simple.conf) {'zeroth_key': 'test', 'first_key': 'András'}</span>
|
|
</code></pre></div>
|
|
<div class="highlight"><pre><span></span><code><a id="__codelineno-0-1" name="__codelineno-0-1" href="#__codelineno-0-1"></a><span class="gp">>>> </span><span class="n">ConfigFile</span><span class="p">(</span><span class="s1">'tests/utilities/data/simple.conf'</span><span class="p">)</span><span class="o">.</span><span class="n">zeroth_key</span>
|
|
<a id="__codelineno-0-2" name="__codelineno-0-2" href="#__codelineno-0-2"></a><span class="go">'test'</span>
|
|
</code></pre></div>
|
|
<div class="highlight"><pre><span></span><code><a id="__codelineno-0-1" name="__codelineno-0-1" href="#__codelineno-0-1"></a><span class="gp">>>> </span><span class="n">ConfigFile</span><span class="p">(</span><span class="s1">'tests/utilities/data/simple.conf'</span><span class="p">)</span><span class="o">.</span><span class="n">second_key</span>
|
|
<a id="__codelineno-0-2" name="__codelineno-0-2" href="#__codelineno-0-2"></a><span class="gt">Traceback (most recent call last):</span>
|
|
<a id="__codelineno-0-3" name="__codelineno-0-3" href="#__codelineno-0-3"></a><span class="c">...</span>
|
|
<a id="__codelineno-0-4" name="__codelineno-0-4" href="#__codelineno-0-4"></a><span class="gr">KeyError</span>: <span class="n">'Key `second_key` is not found in configuration file ...</span>
|
|
</code></pre></div>
|
|
<div class="highlight"><pre><span></span><code><a id="__codelineno-0-1" name="__codelineno-0-1" href="#__codelineno-0-1"></a><span class="gp">>>> </span><span class="n">a</span> <span class="o">=</span> <span class="n">ConfigFile</span><span class="p">(</span><span class="s1">'tests/utilities/data/simple.conf'</span><span class="p">)</span>
|
|
<a id="__codelineno-0-2" name="__codelineno-0-2" href="#__codelineno-0-2"></a><span class="gp">>>> </span><span class="p">{</span><span class="o">**</span><span class="n">a</span><span class="p">}</span>
|
|
<a id="__codelineno-0-3" name="__codelineno-0-3" href="#__codelineno-0-3"></a><span class="go">{'zeroth_key': 'test', 'first_key': 'András'}</span>
|
|
</code></pre></div>
|
|
|
|
|
|
<details class="quote">
|
|
<summary>Source code in <code>great_ai/utilities/config_file/config_file.py</code></summary>
|
|
<div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"><a href="#__codelineno-0-12"> 12</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-13"> 13</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-14"> 14</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-15"> 15</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-16"> 16</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-17"> 17</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-18"> 18</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-19"> 19</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-20"> 20</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-21"> 21</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-22"> 22</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-23"> 23</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-24"> 24</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-25"> 25</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-26"> 26</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-27"> 27</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-28"> 28</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-29"> 29</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-30"> 30</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-31"> 31</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-32"> 32</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-33"> 33</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-34"> 34</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-35"> 35</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-36"> 36</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-37"> 37</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-38"> 38</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-39"> 39</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-40"> 40</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-41"> 41</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-42"> 42</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-43"> 43</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-44"> 44</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-45"> 45</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-46"> 46</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-47"> 47</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-48"> 48</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-49"> 49</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-50"> 50</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-51"> 51</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-52"> 52</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-53"> 53</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-54"> 54</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-55"> 55</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-56"> 56</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-57"> 57</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-58"> 58</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-59"> 59</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-60"> 60</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-61"> 61</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-62"> 62</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-63"> 63</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-64"> 64</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-65"> 65</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-66"> 66</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-67"> 67</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-68"> 68</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-69"> 69</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-70"> 70</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-71"> 71</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-72"> 72</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-73"> 73</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-74"> 74</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-75"> 75</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-76"> 76</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-77"> 77</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-78"> 78</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-79"> 79</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-80"> 80</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-81"> 81</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-82"> 82</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-83"> 83</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-84"> 84</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-85"> 85</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-86"> 86</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-87"> 87</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-88"> 88</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-89"> 89</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-90"> 90</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-91"> 91</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-92"> 92</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-93"> 93</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-94"> 94</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-95"> 95</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-96"> 96</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-97"> 97</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-98"> 98</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-99"> 99</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-100">100</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-101">101</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-102">102</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-103">103</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-104">104</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-105">105</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-106">106</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-107">107</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-108">108</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-109">109</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-110">110</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-111">111</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-112">112</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-113">113</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-114">114</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-115">115</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-116">116</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-117">117</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-118">118</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-119">119</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-120">120</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-121">121</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-122">122</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-123">123</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-124">124</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-125">125</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-126">126</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-127">127</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-128">128</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-129">129</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-130">130</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-131">131</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-132">132</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-133">133</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-134">134</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-135">135</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-136">136</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-137">137</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-138">138</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-139">139</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-140">140</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-141">141</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-142">142</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-143">143</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-144">144</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-145">145</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-146">146</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-147">147</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-148">148</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-149">149</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-150">150</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-151">151</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-152">152</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-153">153</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-154">154</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-155">155</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-156">156</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-157">157</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-158">158</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-159">159</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-160">160</a></span></pre></div></td><td class="code"><div><pre><span></span><code><a id="__codelineno-0-12" name="__codelineno-0-12"></a><span class="k">class</span> <span class="nc">ConfigFile</span><span class="p">(</span><span class="n">Mapping</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="nb">str</span><span class="p">]):</span>
|
|
<a id="__codelineno-0-13" name="__codelineno-0-13"></a> <span class="sd">"""A small and safe `INI`-style configuration loader with `dict` and `ENV` support.</span>
|
|
<a id="__codelineno-0-14" name="__codelineno-0-14"></a>
|
|
<a id="__codelineno-0-15" name="__codelineno-0-15"></a><span class="sd"> The values can be accessed using both dot- and index-notation. It is compatible</span>
|
|
<a id="__codelineno-0-16" name="__codelineno-0-16"></a><span class="sd"> with the `dict` interface.</span>
|
|
<a id="__codelineno-0-17" name="__codelineno-0-17"></a>
|
|
<a id="__codelineno-0-18" name="__codelineno-0-18"></a><span class="sd"> File format example:</span>
|
|
<a id="__codelineno-0-19" name="__codelineno-0-19"></a>
|
|
<a id="__codelineno-0-20" name="__codelineno-0-20"></a><span class="sd"> ```toml</span>
|
|
<a id="__codelineno-0-21" name="__codelineno-0-21"></a><span class="sd"> # comments are allowed everywhere</span>
|
|
<a id="__codelineno-0-22" name="__codelineno-0-22"></a>
|
|
<a id="__codelineno-0-23" name="__codelineno-0-23"></a><span class="sd"> key = value # you can leave or omit whitespace around the equal-sign</span>
|
|
<a id="__codelineno-0-24" name="__codelineno-0-24"></a><span class="sd"> my_hashtag = "#great_ai" # the r-value can be quoted with " or ' or `.</span>
|
|
<a id="__codelineno-0-25" name="__codelineno-0-25"></a>
|
|
<a id="__codelineno-0-26" name="__codelineno-0-26"></a><span class="sd"> my_var = my_default_value # Default values can be given to env-vars,</span>
|
|
<a id="__codelineno-0-27" name="__codelineno-0-27"></a><span class="sd"> # see next line. The default value must come first.</span>
|
|
<a id="__codelineno-0-28" name="__codelineno-0-28"></a>
|
|
<a id="__codelineno-0-29" name="__codelineno-0-29"></a><span class="sd"> my_var = ENV:MY_ENV_VAR # If the value starts with the `ENV:` prefix,</span>
|
|
<a id="__codelineno-0-30" name="__codelineno-0-30"></a><span class="sd"> # it is looked up from the environment variables.</span>
|
|
<a id="__codelineno-0-31" name="__codelineno-0-31"></a><span class="sd"> ```</span>
|
|
<a id="__codelineno-0-32" name="__codelineno-0-32"></a>
|
|
<a id="__codelineno-0-33" name="__codelineno-0-33"></a><span class="sd"> Examples:</span>
|
|
<a id="__codelineno-0-34" name="__codelineno-0-34"></a><span class="sd"> >>> ConfigFile('tests/utilities/data/simple.conf')</span>
|
|
<a id="__codelineno-0-35" name="__codelineno-0-35"></a><span class="sd"> ConfigFile(path=tests/utilities/data/simple.conf) {'zeroth_key': 'test', 'first_key': 'András'}</span>
|
|
<a id="__codelineno-0-36" name="__codelineno-0-36"></a>
|
|
<a id="__codelineno-0-37" name="__codelineno-0-37"></a><span class="sd"> >>> ConfigFile('tests/utilities/data/simple.conf').zeroth_key</span>
|
|
<a id="__codelineno-0-38" name="__codelineno-0-38"></a><span class="sd"> 'test'</span>
|
|
<a id="__codelineno-0-39" name="__codelineno-0-39"></a>
|
|
<a id="__codelineno-0-40" name="__codelineno-0-40"></a><span class="sd"> >>> ConfigFile('tests/utilities/data/simple.conf').second_key</span>
|
|
<a id="__codelineno-0-41" name="__codelineno-0-41"></a><span class="sd"> Traceback (most recent call last):</span>
|
|
<a id="__codelineno-0-42" name="__codelineno-0-42"></a><span class="sd"> ...</span>
|
|
<a id="__codelineno-0-43" name="__codelineno-0-43"></a><span class="sd"> KeyError: 'Key `second_key` is not found in configuration file ...</span>
|
|
<a id="__codelineno-0-44" name="__codelineno-0-44"></a>
|
|
<a id="__codelineno-0-45" name="__codelineno-0-45"></a><span class="sd"> >>> a = ConfigFile('tests/utilities/data/simple.conf')</span>
|
|
<a id="__codelineno-0-46" name="__codelineno-0-46"></a><span class="sd"> >>> {**a}</span>
|
|
<a id="__codelineno-0-47" name="__codelineno-0-47"></a><span class="sd"> {'zeroth_key': 'test', 'first_key': 'András'}</span>
|
|
<a id="__codelineno-0-48" name="__codelineno-0-48"></a>
|
|
<a id="__codelineno-0-49" name="__codelineno-0-49"></a><span class="sd"> """</span>
|
|
<a id="__codelineno-0-50" name="__codelineno-0-50"></a>
|
|
<a id="__codelineno-0-51" name="__codelineno-0-51"></a> <span class="n">ENVIRONMENT_VARIABLE_KEY_PREFIX</span> <span class="o">=</span> <span class="s2">"ENV"</span>
|
|
<a id="__codelineno-0-52" name="__codelineno-0-52"></a>
|
|
<a id="__codelineno-0-53" name="__codelineno-0-53"></a> <span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">path</span><span class="p">:</span> <span class="n">Union</span><span class="p">[</span><span class="n">Path</span><span class="p">,</span> <span class="nb">str</span><span class="p">],</span> <span class="o">*</span><span class="p">,</span> <span class="n">ignore_missing</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">False</span><span class="p">)</span> <span class="o">-></span> <span class="kc">None</span><span class="p">:</span>
|
|
<a id="__codelineno-0-54" name="__codelineno-0-54"></a> <span class="sd">"""Load and parse a configuration file.</span>
|
|
<a id="__codelineno-0-55" name="__codelineno-0-55"></a>
|
|
<a id="__codelineno-0-56" name="__codelineno-0-56"></a><span class="sd"> Everything is eager-loaded, thus, exceptions may be thrown here.</span>
|
|
<a id="__codelineno-0-57" name="__codelineno-0-57"></a>
|
|
<a id="__codelineno-0-58" name="__codelineno-0-58"></a><span class="sd"> Args:</span>
|
|
<a id="__codelineno-0-59" name="__codelineno-0-59"></a><span class="sd"> path: Local path of the configuration file.</span>
|
|
<a id="__codelineno-0-60" name="__codelineno-0-60"></a><span class="sd"> ignore_missing: Don't raise an exception on missing environment variables.</span>
|
|
<a id="__codelineno-0-61" name="__codelineno-0-61"></a>
|
|
<a id="__codelineno-0-62" name="__codelineno-0-62"></a><span class="sd"> Raises:</span>
|
|
<a id="__codelineno-0-63" name="__codelineno-0-63"></a><span class="sd"> FileNotFoundError: If there is no file at the specified path.</span>
|
|
<a id="__codelineno-0-64" name="__codelineno-0-64"></a><span class="sd"> ParseError: If the provided file does not conform to the expected format.</span>
|
|
<a id="__codelineno-0-65" name="__codelineno-0-65"></a><span class="sd"> KeyError: If there is duplication in the keys.</span>
|
|
<a id="__codelineno-0-66" name="__codelineno-0-66"></a><span class="sd"> ValueError: If an environment variable is referenced but it is not set in</span>
|
|
<a id="__codelineno-0-67" name="__codelineno-0-67"></a><span class="sd"> the system and `ignore_missing=False`.</span>
|
|
<a id="__codelineno-0-68" name="__codelineno-0-68"></a><span class="sd"> """</span>
|
|
<a id="__codelineno-0-69" name="__codelineno-0-69"></a>
|
|
<a id="__codelineno-0-70" name="__codelineno-0-70"></a> <span class="k">if</span> <span class="ow">not</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">path</span><span class="p">,</span> <span class="n">Path</span><span class="p">):</span>
|
|
<a id="__codelineno-0-71" name="__codelineno-0-71"></a> <span class="n">path</span> <span class="o">=</span> <span class="n">Path</span><span class="p">(</span><span class="n">path</span><span class="p">)</span>
|
|
<a id="__codelineno-0-72" name="__codelineno-0-72"></a>
|
|
<a id="__codelineno-0-73" name="__codelineno-0-73"></a> <span class="k">if</span> <span class="ow">not</span> <span class="n">path</span><span class="o">.</span><span class="n">exists</span><span class="p">():</span>
|
|
<a id="__codelineno-0-74" name="__codelineno-0-74"></a> <span class="k">raise</span> <span class="ne">FileNotFoundError</span><span class="p">(</span><span class="n">path</span><span class="o">.</span><span class="n">absolute</span><span class="p">())</span>
|
|
<a id="__codelineno-0-75" name="__codelineno-0-75"></a>
|
|
<a id="__codelineno-0-76" name="__codelineno-0-76"></a> <span class="bp">self</span><span class="o">.</span><span class="n">_ignore_missing</span> <span class="o">=</span> <span class="n">ignore_missing</span>
|
|
<a id="__codelineno-0-77" name="__codelineno-0-77"></a>
|
|
<a id="__codelineno-0-78" name="__codelineno-0-78"></a> <span class="bp">self</span><span class="o">.</span><span class="n">_path</span> <span class="o">=</span> <span class="n">path</span>
|
|
<a id="__codelineno-0-79" name="__codelineno-0-79"></a> <span class="bp">self</span><span class="o">.</span><span class="n">_key_values</span><span class="p">:</span> <span class="n">Dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="p">{}</span>
|
|
<a id="__codelineno-0-80" name="__codelineno-0-80"></a>
|
|
<a id="__codelineno-0-81" name="__codelineno-0-81"></a> <span class="bp">self</span><span class="o">.</span><span class="n">_parse</span><span class="p">()</span>
|
|
<a id="__codelineno-0-82" name="__codelineno-0-82"></a>
|
|
<a id="__codelineno-0-83" name="__codelineno-0-83"></a> <span class="nd">@property</span>
|
|
<a id="__codelineno-0-84" name="__codelineno-0-84"></a> <span class="k">def</span> <span class="nf">path</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span> <span class="o">-></span> <span class="n">Path</span><span class="p">:</span>
|
|
<a id="__codelineno-0-85" name="__codelineno-0-85"></a> <span class="sd">"""Original path from where the configuration was loaded."""</span>
|
|
<a id="__codelineno-0-86" name="__codelineno-0-86"></a> <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">_path</span>
|
|
<a id="__codelineno-0-87" name="__codelineno-0-87"></a>
|
|
<a id="__codelineno-0-88" name="__codelineno-0-88"></a> <span class="k">def</span> <span class="nf">_parse</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span> <span class="o">-></span> <span class="kc">None</span><span class="p">:</span>
|
|
<a id="__codelineno-0-89" name="__codelineno-0-89"></a> <span class="k">with</span> <span class="nb">open</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">_path</span><span class="p">,</span> <span class="n">encoding</span><span class="o">=</span><span class="s2">"utf-8"</span><span class="p">)</span> <span class="k">as</span> <span class="n">f</span><span class="p">:</span>
|
|
<a id="__codelineno-0-90" name="__codelineno-0-90"></a> <span class="n">lines</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="n">f</span><span class="o">.</span><span class="n">read</span><span class="p">()</span>
|
|
<a id="__codelineno-0-91" name="__codelineno-0-91"></a>
|
|
<a id="__codelineno-0-92" name="__codelineno-0-92"></a> <span class="n">matches</span> <span class="o">=</span> <span class="n">pattern</span><span class="o">.</span><span class="n">findall</span><span class="p">(</span><span class="n">lines</span><span class="p">)</span>
|
|
<a id="__codelineno-0-93" name="__codelineno-0-93"></a> <span class="k">for</span> <span class="n">key</span><span class="p">,</span> <span class="o">*</span><span class="n">values</span> <span class="ow">in</span> <span class="n">matches</span><span class="p">:</span>
|
|
<a id="__codelineno-0-94" name="__codelineno-0-94"></a> <span class="k">try</span><span class="p">:</span>
|
|
<a id="__codelineno-0-95" name="__codelineno-0-95"></a> <span class="n">value</span> <span class="o">=</span> <span class="nb">next</span><span class="p">(</span><span class="n">v</span> <span class="k">for</span> <span class="n">v</span> <span class="ow">in</span> <span class="n">values</span> <span class="k">if</span> <span class="n">v</span><span class="p">)</span>
|
|
<a id="__codelineno-0-96" name="__codelineno-0-96"></a> <span class="k">except</span> <span class="ne">StopIteration</span><span class="p">:</span>
|
|
<a id="__codelineno-0-97" name="__codelineno-0-97"></a> <span class="k">raise</span> <span class="n">ParseError</span><span class="p">(</span>
|
|
<a id="__codelineno-0-98" name="__codelineno-0-98"></a> <span class="sa">f</span><span class="s2">"""Cannot parse config file (</span><span class="si">{</span>
|
|
<a id="__codelineno-0-99" name="__codelineno-0-99"></a> <span class="bp">self</span><span class="o">.</span><span class="n">_path</span><span class="o">.</span><span class="n">absolute</span><span class="p">()</span>
|
|
<a id="__codelineno-0-100" name="__codelineno-0-100"></a> <span class="si">}</span><span class="s2">), error at key `</span><span class="si">{</span><span class="n">key</span><span class="si">}</span><span class="s2">`"""</span>
|
|
<a id="__codelineno-0-101" name="__codelineno-0-101"></a> <span class="p">)</span>
|
|
<a id="__codelineno-0-102" name="__codelineno-0-102"></a>
|
|
<a id="__codelineno-0-103" name="__codelineno-0-103"></a> <span class="n">already_exists</span> <span class="o">=</span> <span class="n">key</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">_key_values</span>
|
|
<a id="__codelineno-0-104" name="__codelineno-0-104"></a> <span class="k">if</span> <span class="n">already_exists</span> <span class="ow">and</span> <span class="ow">not</span> <span class="n">value</span><span class="o">.</span><span class="n">startswith</span><span class="p">(</span>
|
|
<a id="__codelineno-0-105" name="__codelineno-0-105"></a> <span class="sa">f</span><span class="s2">"</span><span class="si">{</span><span class="bp">self</span><span class="o">.</span><span class="n">ENVIRONMENT_VARIABLE_KEY_PREFIX</span><span class="si">}</span><span class="s2">:"</span>
|
|
<a id="__codelineno-0-106" name="__codelineno-0-106"></a> <span class="p">):</span>
|
|
<a id="__codelineno-0-107" name="__codelineno-0-107"></a> <span class="k">raise</span> <span class="ne">KeyError</span><span class="p">(</span>
|
|
<a id="__codelineno-0-108" name="__codelineno-0-108"></a> <span class="sa">f</span><span class="s2">"Key `</span><span class="si">{</span><span class="n">key</span><span class="si">}</span><span class="s2">` has been already defined and its value is `</span><span class="si">{</span><span class="bp">self</span><span class="o">.</span><span class="n">_key_values</span><span class="p">[</span><span class="n">key</span><span class="p">]</span><span class="si">}</span><span class="s2">`"</span>
|
|
<a id="__codelineno-0-109" name="__codelineno-0-109"></a> <span class="p">)</span>
|
|
<a id="__codelineno-0-110" name="__codelineno-0-110"></a>
|
|
<a id="__codelineno-0-111" name="__codelineno-0-111"></a> <span class="k">if</span> <span class="n">value</span><span class="o">.</span><span class="n">startswith</span><span class="p">(</span><span class="sa">f</span><span class="s2">"</span><span class="si">{</span><span class="bp">self</span><span class="o">.</span><span class="n">ENVIRONMENT_VARIABLE_KEY_PREFIX</span><span class="si">}</span><span class="s2">:"</span><span class="p">):</span>
|
|
<a id="__codelineno-0-112" name="__codelineno-0-112"></a> <span class="n">_</span><span class="p">,</span> <span class="n">value</span> <span class="o">=</span> <span class="n">value</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="s2">":"</span><span class="p">)</span>
|
|
<a id="__codelineno-0-113" name="__codelineno-0-113"></a> <span class="k">if</span> <span class="n">value</span> <span class="ow">not</span> <span class="ow">in</span> <span class="n">os</span><span class="o">.</span><span class="n">environ</span><span class="p">:</span>
|
|
<a id="__codelineno-0-114" name="__codelineno-0-114"></a> <span class="n">issue</span> <span class="o">=</span> <span class="sa">f</span><span class="s2">"""The value of `</span><span class="si">{</span><span class="n">key</span><span class="si">}</span><span class="s2">` contains the "</span><span class="si">{</span>
|
|
<a id="__codelineno-0-115" name="__codelineno-0-115"></a> <span class="bp">self</span><span class="o">.</span><span class="n">ENVIRONMENT_VARIABLE_KEY_PREFIX</span>
|
|
<a id="__codelineno-0-116" name="__codelineno-0-116"></a> <span class="si">}</span><span class="s2">` prefix but `</span><span class="si">{</span><span class="n">value</span><span class="si">}</span><span class="s2">` is not defined as an environment variable"""</span>
|
|
<a id="__codelineno-0-117" name="__codelineno-0-117"></a> <span class="k">if</span> <span class="n">already_exists</span><span class="p">:</span>
|
|
<a id="__codelineno-0-118" name="__codelineno-0-118"></a> <span class="n">logger</span><span class="o">.</span><span class="n">warning</span><span class="p">(</span>
|
|
<a id="__codelineno-0-119" name="__codelineno-0-119"></a> <span class="sa">f</span><span class="s2">"""</span><span class="si">{</span><span class="n">issue</span><span class="si">}</span><span class="s2">, using the default value defined above (`</span><span class="si">{</span>
|
|
<a id="__codelineno-0-120" name="__codelineno-0-120"></a> <span class="bp">self</span><span class="o">.</span><span class="n">_key_values</span><span class="p">[</span><span class="n">key</span><span class="p">]</span>
|
|
<a id="__codelineno-0-121" name="__codelineno-0-121"></a> <span class="si">}</span><span class="s2">`)"""</span>
|
|
<a id="__codelineno-0-122" name="__codelineno-0-122"></a> <span class="p">)</span>
|
|
<a id="__codelineno-0-123" name="__codelineno-0-123"></a> <span class="k">continue</span>
|
|
<a id="__codelineno-0-124" name="__codelineno-0-124"></a> <span class="k">elif</span> <span class="bp">self</span><span class="o">.</span><span class="n">_ignore_missing</span><span class="p">:</span>
|
|
<a id="__codelineno-0-125" name="__codelineno-0-125"></a> <span class="n">logger</span><span class="o">.</span><span class="n">warning</span><span class="p">(</span><span class="n">issue</span><span class="p">)</span>
|
|
<a id="__codelineno-0-126" name="__codelineno-0-126"></a> <span class="k">else</span><span class="p">:</span>
|
|
<a id="__codelineno-0-127" name="__codelineno-0-127"></a> <span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span>
|
|
<a id="__codelineno-0-128" name="__codelineno-0-128"></a> <span class="sa">f</span><span class="s2">"</span><span class="si">{</span><span class="n">issue</span><span class="si">}</span><span class="s2"> and no default value has been provided"</span>
|
|
<a id="__codelineno-0-129" name="__codelineno-0-129"></a> <span class="p">)</span>
|
|
<a id="__codelineno-0-130" name="__codelineno-0-130"></a> <span class="k">else</span><span class="p">:</span>
|
|
<a id="__codelineno-0-131" name="__codelineno-0-131"></a> <span class="n">value</span> <span class="o">=</span> <span class="n">os</span><span class="o">.</span><span class="n">environ</span><span class="p">[</span><span class="n">value</span><span class="p">]</span>
|
|
<a id="__codelineno-0-132" name="__codelineno-0-132"></a>
|
|
<a id="__codelineno-0-133" name="__codelineno-0-133"></a> <span class="bp">self</span><span class="o">.</span><span class="n">_key_values</span><span class="p">[</span><span class="n">key</span><span class="p">]</span> <span class="o">=</span> <span class="n">value</span>
|
|
<a id="__codelineno-0-134" name="__codelineno-0-134"></a>
|
|
<a id="__codelineno-0-135" name="__codelineno-0-135"></a> <span class="k">def</span> <span class="fm">__getattr__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">key</span><span class="p">:</span> <span class="nb">str</span><span class="p">)</span> <span class="o">-></span> <span class="nb">str</span><span class="p">:</span>
|
|
<a id="__codelineno-0-136" name="__codelineno-0-136"></a> <span class="k">if</span> <span class="n">key</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">_key_values</span><span class="p">:</span>
|
|
<a id="__codelineno-0-137" name="__codelineno-0-137"></a> <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">_key_values</span><span class="p">[</span><span class="n">key</span><span class="p">]</span>
|
|
<a id="__codelineno-0-138" name="__codelineno-0-138"></a> <span class="k">raise</span> <span class="ne">KeyError</span><span class="p">(</span>
|
|
<a id="__codelineno-0-139" name="__codelineno-0-139"></a> <span class="sa">f</span><span class="s2">"Key `</span><span class="si">{</span><span class="n">key</span><span class="si">}</span><span class="s2">` is not found in configuration file (</span><span class="si">{</span><span class="bp">self</span><span class="o">.</span><span class="n">_path</span><span class="o">.</span><span class="n">absolute</span><span class="p">()</span><span class="si">}</span><span class="s2">)"</span>
|
|
<a id="__codelineno-0-140" name="__codelineno-0-140"></a> <span class="p">)</span>
|
|
<a id="__codelineno-0-141" name="__codelineno-0-141"></a>
|
|
<a id="__codelineno-0-142" name="__codelineno-0-142"></a> <span class="fm">__getitem__</span> <span class="o">=</span> <span class="fm">__getattr__</span>
|
|
<a id="__codelineno-0-143" name="__codelineno-0-143"></a>
|
|
<a id="__codelineno-0-144" name="__codelineno-0-144"></a> <span class="k">def</span> <span class="fm">__iter__</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span> <span class="o">-></span> <span class="n">Iterator</span><span class="p">[</span><span class="nb">str</span><span class="p">]:</span>
|
|
<a id="__codelineno-0-145" name="__codelineno-0-145"></a> <span class="k">return</span> <span class="nb">iter</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">_key_values</span><span class="p">)</span>
|
|
<a id="__codelineno-0-146" name="__codelineno-0-146"></a>
|
|
<a id="__codelineno-0-147" name="__codelineno-0-147"></a> <span class="k">def</span> <span class="fm">__len__</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span> <span class="o">-></span> <span class="nb">int</span><span class="p">:</span>
|
|
<a id="__codelineno-0-148" name="__codelineno-0-148"></a> <span class="k">return</span> <span class="nb">len</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">_key_values</span><span class="p">)</span>
|
|
<a id="__codelineno-0-149" name="__codelineno-0-149"></a>
|
|
<a id="__codelineno-0-150" name="__codelineno-0-150"></a> <span class="k">def</span> <span class="nf">keys</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span> <span class="o">-></span> <span class="n">KeysView</span><span class="p">[</span><span class="nb">str</span><span class="p">]:</span>
|
|
<a id="__codelineno-0-151" name="__codelineno-0-151"></a> <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">_key_values</span><span class="o">.</span><span class="n">keys</span><span class="p">()</span>
|
|
<a id="__codelineno-0-152" name="__codelineno-0-152"></a>
|
|
<a id="__codelineno-0-153" name="__codelineno-0-153"></a> <span class="k">def</span> <span class="nf">values</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span> <span class="o">-></span> <span class="n">ValuesView</span><span class="p">[</span><span class="nb">str</span><span class="p">]:</span>
|
|
<a id="__codelineno-0-154" name="__codelineno-0-154"></a> <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">_key_values</span><span class="o">.</span><span class="n">values</span><span class="p">()</span>
|
|
<a id="__codelineno-0-155" name="__codelineno-0-155"></a>
|
|
<a id="__codelineno-0-156" name="__codelineno-0-156"></a> <span class="k">def</span> <span class="nf">items</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span> <span class="o">-></span> <span class="n">ItemsView</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="nb">str</span><span class="p">]:</span>
|
|
<a id="__codelineno-0-157" name="__codelineno-0-157"></a> <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">_key_values</span><span class="o">.</span><span class="n">items</span><span class="p">()</span>
|
|
<a id="__codelineno-0-158" name="__codelineno-0-158"></a>
|
|
<a id="__codelineno-0-159" name="__codelineno-0-159"></a> <span class="k">def</span> <span class="fm">__repr__</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span> <span class="o">-></span> <span class="nb">str</span><span class="p">:</span>
|
|
<a id="__codelineno-0-160" name="__codelineno-0-160"></a> <span class="k">return</span> <span class="sa">f</span><span class="s2">"</span><span class="si">{</span><span class="nb">type</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="vm">__name__</span><span class="si">}</span><span class="s2">(path=</span><span class="si">{</span><span class="bp">self</span><span class="o">.</span><span class="n">_path</span><span class="o">.</span><span class="n">as_posix</span><span class="p">()</span><span class="si">}</span><span class="s2">) </span><span class="si">{</span><span class="bp">self</span><span class="o">.</span><span class="n">_key_values</span><span class="si">}</span><span class="s2">"</span>
|
|
</code></pre></div></td></tr></table></div>
|
|
</details>
|
|
|
|
|
|
|
|
<div class="doc doc-children">
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
<div class="doc doc-object doc-function">
|
|
|
|
|
|
|
|
<h4 id="great_ai.utilities.config_file.config_file.ConfigFile.__init__" class="doc doc-heading">
|
|
<code class="highlight language-python"><span class="fm">__init__</span><span class="p">(</span><span class="n">path</span><span class="p">,</span> <span class="o">*</span><span class="p">,</span> <span class="n">ignore_missing</span><span class="o">=</span><span class="kc">False</span><span class="p">)</span></code>
|
|
|
|
<a href="#great_ai.utilities.config_file.config_file.ConfigFile.__init__" class="headerlink" title="Permanent link">#</a></h4>
|
|
|
|
|
|
<div class="doc doc-contents ">
|
|
|
|
<p>Load and parse a configuration file.</p>
|
|
<p>Everything is eager-loaded, thus, exceptions may be thrown here.</p>
|
|
|
|
<p><strong>Parameters:</strong></p>
|
|
<table>
|
|
<thead>
|
|
<tr>
|
|
<th>Name</th>
|
|
<th>Type</th>
|
|
<th>Description</th>
|
|
<th>Default</th>
|
|
</tr>
|
|
</thead>
|
|
<tbody>
|
|
<tr>
|
|
<td><code>path</code></td>
|
|
<td>
|
|
<code><span title="typing.Union">Union</span>[<span title="pathlib.Path">Path</span>, str]</code>
|
|
</td>
|
|
<td><p>Local path of the configuration file.</p></td>
|
|
<td>
|
|
<em>required</em>
|
|
</td>
|
|
</tr>
|
|
<tr>
|
|
<td><code>ignore_missing</code></td>
|
|
<td>
|
|
<code>bool</code>
|
|
</td>
|
|
<td><p>Don't raise an exception on missing environment variables.</p></td>
|
|
<td>
|
|
<code>False</code>
|
|
</td>
|
|
</tr>
|
|
</tbody>
|
|
</table>
|
|
|
|
<p><strong>Raises:</strong></p>
|
|
<table>
|
|
<thead>
|
|
<tr>
|
|
<th>Type</th>
|
|
<th>Description</th>
|
|
</tr>
|
|
</thead>
|
|
<tbody>
|
|
<tr>
|
|
<td>
|
|
<code>FileNotFoundError</code>
|
|
</td>
|
|
<td><p>If there is no file at the specified path.</p></td>
|
|
</tr>
|
|
<tr>
|
|
<td>
|
|
<code><span title="great_ai.utilities.config_file.parse_error.ParseError">ParseError</span></code>
|
|
</td>
|
|
<td><p>If the provided file does not conform to the expected format.</p></td>
|
|
</tr>
|
|
<tr>
|
|
<td>
|
|
<code>KeyError</code>
|
|
</td>
|
|
<td><p>If there is duplication in the keys.</p></td>
|
|
</tr>
|
|
<tr>
|
|
<td>
|
|
<code>ValueError</code>
|
|
</td>
|
|
<td><p>If an environment variable is referenced but it is not set in
|
|
the system and <code>ignore_missing=False</code>.</p></td>
|
|
</tr>
|
|
</tbody>
|
|
</table>
|
|
|
|
<details class="quote">
|
|
<summary>Source code in <code>great_ai/utilities/config_file/config_file.py</code></summary>
|
|
<div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"><a href="#__codelineno-0-53">53</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-54">54</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-55">55</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-56">56</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-57">57</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-58">58</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-59">59</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-60">60</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-61">61</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-62">62</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-63">63</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-64">64</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-65">65</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-66">66</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-67">67</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-68">68</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-69">69</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-70">70</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-71">71</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-72">72</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-73">73</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-74">74</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-75">75</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-76">76</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-77">77</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-78">78</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-79">79</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-80">80</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-81">81</a></span></pre></div></td><td class="code"><div><pre><span></span><code><a id="__codelineno-0-53" name="__codelineno-0-53"></a><span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">path</span><span class="p">:</span> <span class="n">Union</span><span class="p">[</span><span class="n">Path</span><span class="p">,</span> <span class="nb">str</span><span class="p">],</span> <span class="o">*</span><span class="p">,</span> <span class="n">ignore_missing</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">False</span><span class="p">)</span> <span class="o">-></span> <span class="kc">None</span><span class="p">:</span>
|
|
<a id="__codelineno-0-54" name="__codelineno-0-54"></a> <span class="sd">"""Load and parse a configuration file.</span>
|
|
<a id="__codelineno-0-55" name="__codelineno-0-55"></a>
|
|
<a id="__codelineno-0-56" name="__codelineno-0-56"></a><span class="sd"> Everything is eager-loaded, thus, exceptions may be thrown here.</span>
|
|
<a id="__codelineno-0-57" name="__codelineno-0-57"></a>
|
|
<a id="__codelineno-0-58" name="__codelineno-0-58"></a><span class="sd"> Args:</span>
|
|
<a id="__codelineno-0-59" name="__codelineno-0-59"></a><span class="sd"> path: Local path of the configuration file.</span>
|
|
<a id="__codelineno-0-60" name="__codelineno-0-60"></a><span class="sd"> ignore_missing: Don't raise an exception on missing environment variables.</span>
|
|
<a id="__codelineno-0-61" name="__codelineno-0-61"></a>
|
|
<a id="__codelineno-0-62" name="__codelineno-0-62"></a><span class="sd"> Raises:</span>
|
|
<a id="__codelineno-0-63" name="__codelineno-0-63"></a><span class="sd"> FileNotFoundError: If there is no file at the specified path.</span>
|
|
<a id="__codelineno-0-64" name="__codelineno-0-64"></a><span class="sd"> ParseError: If the provided file does not conform to the expected format.</span>
|
|
<a id="__codelineno-0-65" name="__codelineno-0-65"></a><span class="sd"> KeyError: If there is duplication in the keys.</span>
|
|
<a id="__codelineno-0-66" name="__codelineno-0-66"></a><span class="sd"> ValueError: If an environment variable is referenced but it is not set in</span>
|
|
<a id="__codelineno-0-67" name="__codelineno-0-67"></a><span class="sd"> the system and `ignore_missing=False`.</span>
|
|
<a id="__codelineno-0-68" name="__codelineno-0-68"></a><span class="sd"> """</span>
|
|
<a id="__codelineno-0-69" name="__codelineno-0-69"></a>
|
|
<a id="__codelineno-0-70" name="__codelineno-0-70"></a> <span class="k">if</span> <span class="ow">not</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">path</span><span class="p">,</span> <span class="n">Path</span><span class="p">):</span>
|
|
<a id="__codelineno-0-71" name="__codelineno-0-71"></a> <span class="n">path</span> <span class="o">=</span> <span class="n">Path</span><span class="p">(</span><span class="n">path</span><span class="p">)</span>
|
|
<a id="__codelineno-0-72" name="__codelineno-0-72"></a>
|
|
<a id="__codelineno-0-73" name="__codelineno-0-73"></a> <span class="k">if</span> <span class="ow">not</span> <span class="n">path</span><span class="o">.</span><span class="n">exists</span><span class="p">():</span>
|
|
<a id="__codelineno-0-74" name="__codelineno-0-74"></a> <span class="k">raise</span> <span class="ne">FileNotFoundError</span><span class="p">(</span><span class="n">path</span><span class="o">.</span><span class="n">absolute</span><span class="p">())</span>
|
|
<a id="__codelineno-0-75" name="__codelineno-0-75"></a>
|
|
<a id="__codelineno-0-76" name="__codelineno-0-76"></a> <span class="bp">self</span><span class="o">.</span><span class="n">_ignore_missing</span> <span class="o">=</span> <span class="n">ignore_missing</span>
|
|
<a id="__codelineno-0-77" name="__codelineno-0-77"></a>
|
|
<a id="__codelineno-0-78" name="__codelineno-0-78"></a> <span class="bp">self</span><span class="o">.</span><span class="n">_path</span> <span class="o">=</span> <span class="n">path</span>
|
|
<a id="__codelineno-0-79" name="__codelineno-0-79"></a> <span class="bp">self</span><span class="o">.</span><span class="n">_key_values</span><span class="p">:</span> <span class="n">Dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="p">{}</span>
|
|
<a id="__codelineno-0-80" name="__codelineno-0-80"></a>
|
|
<a id="__codelineno-0-81" name="__codelineno-0-81"></a> <span class="bp">self</span><span class="o">.</span><span class="n">_parse</span><span class="p">()</span>
|
|
</code></pre></div></td></tr></table></div>
|
|
</details>
|
|
</div>
|
|
|
|
</div>
|
|
|
|
<div class="doc doc-object doc-function">
|
|
|
|
|
|
|
|
<h4 id="great_ai.utilities.config_file.config_file.ConfigFile.path" class="doc doc-heading">
|
|
<code class="highlight language-python"><span class="n">path</span><span class="p">()</span></code>
|
|
|
|
<span class="doc doc-labels">
|
|
<small class="doc doc-label doc-label-property"><code>property</code></small>
|
|
</span>
|
|
|
|
<a href="#great_ai.utilities.config_file.config_file.ConfigFile.path" class="headerlink" title="Permanent link">#</a></h4>
|
|
|
|
|
|
<div class="doc doc-contents ">
|
|
|
|
<p>Original path from where the configuration was loaded.</p>
|
|
|
|
<details class="quote">
|
|
<summary>Source code in <code>great_ai/utilities/config_file/config_file.py</code></summary>
|
|
<div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"><a href="#__codelineno-0-83">83</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-84">84</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-85">85</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-86">86</a></span></pre></div></td><td class="code"><div><pre><span></span><code><a id="__codelineno-0-83" name="__codelineno-0-83"></a><span class="nd">@property</span>
|
|
<a id="__codelineno-0-84" name="__codelineno-0-84"></a><span class="k">def</span> <span class="nf">path</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span> <span class="o">-></span> <span class="n">Path</span><span class="p">:</span>
|
|
<a id="__codelineno-0-85" name="__codelineno-0-85"></a> <span class="sd">"""Original path from where the configuration was loaded."""</span>
|
|
<a id="__codelineno-0-86" name="__codelineno-0-86"></a> <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">_path</span>
|
|
</code></pre></div></td></tr></table></div>
|
|
</details>
|
|
</div>
|
|
|
|
</div>
|
|
|
|
|
|
|
|
</div>
|
|
|
|
</div>
|
|
|
|
</div>
|
|
|
|
<div class="doc doc-object doc-function">
|
|
|
|
|
|
|
|
<h3 id="great_ai.utilities.get_logger" class="doc doc-heading">
|
|
<code class="highlight language-python"><span class="n">get_logger</span><span class="p">(</span><span class="n">name</span><span class="p">,</span> <span class="n">level</span><span class="o">=</span><span class="n">logging</span><span class="o">.</span><span class="n">INFO</span><span class="p">,</span> <span class="n">disable_colors</span><span class="o">=</span><span class="kc">False</span><span class="p">)</span></code>
|
|
|
|
<a href="#great_ai.utilities.get_logger" class="headerlink" title="Permanent link">#</a></h3>
|
|
|
|
|
|
<div class="doc doc-contents first">
|
|
|
|
<p>Return a customised logger used throughout the GreatAI codebase.</p>
|
|
<p>Uses colors, and only prints timestamps when not running inside notebook.</p>
|
|
|
|
<details class="quote">
|
|
<summary>Source code in <code>great_ai/utilities/logger/get_logger.py</code></summary>
|
|
<div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"><a href="#__codelineno-0-9"> 9</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-10">10</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-11">11</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-12">12</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-13">13</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-14">14</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-15">15</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-16">16</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-17">17</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-18">18</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-19">19</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-20">20</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-21">21</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-22">22</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-23">23</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-24">24</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-25">25</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-26">26</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-27">27</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-28">28</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-29">29</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-30">30</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-31">31</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-32">32</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-33">33</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-34">34</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-35">35</a></span>
|
|
<span class="normal"><a href="#__codelineno-0-36">36</a></span></pre></div></td><td class="code"><div><pre><span></span><code><a id="__codelineno-0-9" name="__codelineno-0-9"></a><span class="k">def</span> <span class="nf">get_logger</span><span class="p">(</span>
|
|
<a id="__codelineno-0-10" name="__codelineno-0-10"></a> <span class="n">name</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span> <span class="n">level</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="n">logging</span><span class="o">.</span><span class="n">INFO</span><span class="p">,</span> <span class="n">disable_colors</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">False</span>
|
|
<a id="__codelineno-0-11" name="__codelineno-0-11"></a><span class="p">)</span> <span class="o">-></span> <span class="n">logging</span><span class="o">.</span><span class="n">Logger</span><span class="p">:</span>
|
|
<a id="__codelineno-0-12" name="__codelineno-0-12"></a> <span class="sd">"""Return a customised logger used throughout the GreatAI codebase.</span>
|
|
<a id="__codelineno-0-13" name="__codelineno-0-13"></a>
|
|
<a id="__codelineno-0-14" name="__codelineno-0-14"></a><span class="sd"> Uses colors, and only prints timestamps when not running inside notebook.</span>
|
|
<a id="__codelineno-0-15" name="__codelineno-0-15"></a><span class="sd"> """</span>
|
|
<a id="__codelineno-0-16" name="__codelineno-0-16"></a>
|
|
<a id="__codelineno-0-17" name="__codelineno-0-17"></a> <span class="k">if</span> <span class="n">name</span> <span class="ow">not</span> <span class="ow">in</span> <span class="n">loggers</span><span class="p">:</span>
|
|
<a id="__codelineno-0-18" name="__codelineno-0-18"></a> <span class="n">logger</span> <span class="o">=</span> <span class="n">logging</span><span class="o">.</span><span class="n">getLogger</span><span class="p">(</span><span class="n">name</span><span class="p">)</span>
|
|
<a id="__codelineno-0-19" name="__codelineno-0-19"></a> <span class="n">logger</span><span class="o">.</span><span class="n">setLevel</span><span class="p">(</span><span class="n">level</span><span class="p">)</span>
|
|
<a id="__codelineno-0-20" name="__codelineno-0-20"></a>
|
|
<a id="__codelineno-0-21" name="__codelineno-0-21"></a> <span class="k">try</span><span class="p">:</span>
|
|
<a id="__codelineno-0-22" name="__codelineno-0-22"></a> <span class="n">get_ipython</span><span class="p">()</span> <span class="c1"># type: ignore</span>
|
|
<a id="__codelineno-0-23" name="__codelineno-0-23"></a> <span class="n">log_format</span> <span class="o">=</span> <span class="s2">"</span><span class="si">%(message)s</span><span class="s2">"</span>
|
|
<a id="__codelineno-0-24" name="__codelineno-0-24"></a> <span class="k">except</span> <span class="ne">NameError</span><span class="p">:</span>
|
|
<a id="__codelineno-0-25" name="__codelineno-0-25"></a> <span class="c1"># will fail outside of a notebook https://ipython.org/</span>
|
|
<a id="__codelineno-0-26" name="__codelineno-0-26"></a> <span class="n">log_format</span> <span class="o">=</span> <span class="s2">"</span><span class="si">%(asctime)s</span><span class="s2"> | </span><span class="si">%(levelname)8s</span><span class="s2"> | </span><span class="si">%(message)s</span><span class="s2">"</span>
|
|
<a id="__codelineno-0-27" name="__codelineno-0-27"></a>
|
|
<a id="__codelineno-0-28" name="__codelineno-0-28"></a> <span class="n">stdout_handler</span> <span class="o">=</span> <span class="n">logging</span><span class="o">.</span><span class="n">StreamHandler</span><span class="p">()</span>
|
|
<a id="__codelineno-0-29" name="__codelineno-0-29"></a> <span class="n">stdout_handler</span><span class="o">.</span><span class="n">setLevel</span><span class="p">(</span><span class="n">level</span><span class="p">)</span>
|
|
<a id="__codelineno-0-30" name="__codelineno-0-30"></a> <span class="k">if</span> <span class="ow">not</span> <span class="n">disable_colors</span><span class="p">:</span>
|
|
<a id="__codelineno-0-31" name="__codelineno-0-31"></a> <span class="n">stdout_handler</span><span class="o">.</span><span class="n">setFormatter</span><span class="p">(</span><span class="n">CustomFormatter</span><span class="p">(</span><span class="n">log_format</span><span class="p">))</span>
|
|
<a id="__codelineno-0-32" name="__codelineno-0-32"></a>
|
|
<a id="__codelineno-0-33" name="__codelineno-0-33"></a> <span class="n">logger</span><span class="o">.</span><span class="n">addHandler</span><span class="p">(</span><span class="n">stdout_handler</span><span class="p">)</span>
|
|
<a id="__codelineno-0-34" name="__codelineno-0-34"></a> <span class="n">loggers</span><span class="p">[</span><span class="n">name</span><span class="p">]</span> <span class="o">=</span> <span class="n">logger</span>
|
|
<a id="__codelineno-0-35" name="__codelineno-0-35"></a>
|
|
<a id="__codelineno-0-36" name="__codelineno-0-36"></a> <span class="k">return</span> <span class="n">loggers</span><span class="p">[</span><span class="n">name</span><span class="p">]</span>
|
|
</code></pre></div></td></tr></table></div>
|
|
</details>
|
|
</div>
|
|
|
|
</div>
|
|
|
|
<hr>
|
|
<div class="md-source-file">
|
|
<small>
|
|
|
|
Last update:
|
|
<span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-date">July 11, 2022</span>
|
|
|
|
|
|
</small>
|
|
</div>
|
|
|
|
|
|
|
|
</article>
|
|
|
|
</div>
|
|
</div>
|
|
|
|
</main>
|
|
|
|
<footer class="md-footer">
|
|
|
|
|
|
<nav class="md-footer__inner md-grid" aria-label="Footer" >
|
|
|
|
|
|
<a href="../" class="md-footer__link md-footer__link--prev" aria-label="Previous: GreatAI reference" rel="prev">
|
|
<div class="md-footer__button md-icon">
|
|
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M20 11v2H8l5.5 5.5-1.42 1.42L4.16 12l7.92-7.92L13.5 5.5 8 11h12Z"/></svg>
|
|
</div>
|
|
<div class="md-footer__title">
|
|
<div class="md-ellipsis">
|
|
<span class="md-footer__direction">
|
|
Previous
|
|
</span>
|
|
GreatAI reference
|
|
</div>
|
|
</div>
|
|
</a>
|
|
|
|
|
|
|
|
<a href="../large-file/" class="md-footer__link md-footer__link--next" aria-label="Next: LargeFile" rel="next">
|
|
<div class="md-footer__title">
|
|
<div class="md-ellipsis">
|
|
<span class="md-footer__direction">
|
|
Next
|
|
</span>
|
|
LargeFile
|
|
</div>
|
|
</div>
|
|
<div class="md-footer__button md-icon">
|
|
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M4 11v2h12l-5.5 5.5 1.42 1.42L19.84 12l-7.92-7.92L10.5 5.5 16 11H4Z"/></svg>
|
|
</div>
|
|
</a>
|
|
|
|
</nav>
|
|
|
|
<div class="md-footer-meta md-typeset">
|
|
<div class="md-footer-meta__inner md-grid">
|
|
<div class="md-copyright">
|
|
|
|
<div class="md-copyright__highlight">
|
|
GNU General Public License v3
|
|
</div>
|
|
|
|
|
|
</div>
|
|
|
|
<div class="md-social">
|
|
|
|
|
|
|
|
<a href="https://schmelczer.dev" target="_blank" rel="noopener" title="about the author" class="md-social__link">
|
|
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 640 512"><!--! Font Awesome Free 6.1.1 by @fontawesome - https://fontawesome.com License - https://fontawesome.com/license/free (Icons: CC BY 4.0, Fonts: SIL OFL 1.1, Code: MIT License) Copyright 2022 Fonticons, Inc.--><path d="m414.8 40.79-128 448.01c-4.9 17-22.6 26.8-39.6 22-17-4.9-26.8-22.6-22-39.6l128-447.99c4.9-16.994 22.6-26.834 39.6-21.978 17 4.855 26.8 22.568 22 39.558zm103.8 80.61 112 112c12.5 12.5 12.5 32.7 0 45.2l-112 112c-12.5 12.5-32.7 12.5-45.2 0s-12.5-32.7 0-45.2l89.3-89.4-89.3-89.4c-12.5-12.5-12.5-32.7 0-45.2s32.7-12.5 45.2 0zm-352 45.2L77.25 256l89.35 89.4c12.5 12.5 12.5 32.7 0 45.2s-32.7 12.5-45.2 0L9.372 278.6c-12.496-12.5-12.496-32.7 0-45.2l112.028-112c12.5-12.5 32.7-12.5 45.2 0s12.5 32.7 0 45.2z"/></svg>
|
|
</a>
|
|
|
|
|
|
|
|
<a href="https://github.com/schmelczer/great-ai" target="_blank" rel="noopener" title="great-ai on Github" class="md-social__link">
|
|
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 496 512"><!--! Font Awesome Free 6.1.1 by @fontawesome - https://fontawesome.com License - https://fontawesome.com/license/free (Icons: CC BY 4.0, Fonts: SIL OFL 1.1, Code: MIT License) Copyright 2022 Fonticons, Inc.--><path d="M165.9 397.4c0 2-2.3 3.6-5.2 3.6-3.3.3-5.6-1.3-5.6-3.6 0-2 2.3-3.6 5.2-3.6 3-.3 5.6 1.3 5.6 3.6zm-31.1-4.5c-.7 2 1.3 4.3 4.3 4.9 2.6 1 5.6 0 6.2-2s-1.3-4.3-4.3-5.2c-2.6-.7-5.5.3-6.2 2.3zm44.2-1.7c-2.9.7-4.9 2.6-4.6 4.9.3 2 2.9 3.3 5.9 2.6 2.9-.7 4.9-2.6 4.6-4.6-.3-1.9-3-3.2-5.9-2.9zM244.8 8C106.1 8 0 113.3 0 252c0 110.9 69.8 205.8 169.5 239.2 12.8 2.3 17.3-5.6 17.3-12.1 0-6.2-.3-40.4-.3-61.4 0 0-70 15-84.7-29.8 0 0-11.4-29.1-27.8-36.6 0 0-22.9-15.7 1.6-15.4 0 0 24.9 2 38.6 25.8 21.9 38.6 58.6 27.5 72.9 20.9 2.3-16 8.8-27.1 16-33.7-55.9-6.2-112.3-14.3-112.3-110.5 0-27.5 7.6-41.3 23.6-58.9-2.6-6.5-11.1-33.3 2.6-67.9 20.9-6.5 69 27 69 27 20-5.6 41.5-8.5 62.8-8.5s42.8 2.9 62.8 8.5c0 0 48.1-33.6 69-27 13.7 34.7 5.2 61.4 2.6 67.9 16 17.7 25.8 31.5 25.8 58.9 0 96.5-58.9 104.2-114.8 110.5 9.2 7.9 17 22.9 17 46.4 0 33.7-.3 75.4-.3 83.6 0 6.5 4.6 14.4 17.3 12.1C428.2 457.8 496 362.9 496 252 496 113.3 383.5 8 244.8 8zM97.2 352.9c-1.3 1-1 3.3.7 5.2 1.6 1.6 3.9 2.3 5.2 1 1.3-1 1-3.3-.7-5.2-1.6-1.6-3.9-2.3-5.2-1zm-10.8-8.1c-.7 1.3.3 2.9 2.3 3.9 1.6 1 3.6.7 4.3-.7.7-1.3-.3-2.9-2.3-3.9-2-.6-3.6-.3-4.3.7zm32.4 35.6c-1.6 1.3-1 4.3 1.3 6.2 2.3 2.3 5.2 2.6 6.5 1 1.3-1.3.7-4.3-1.3-6.2-2.2-2.3-5.2-2.6-6.5-1zm-11.4-14.7c-1.6 1-1.6 3.6 0 5.9 1.6 2.3 4.3 3.3 5.6 2.3 1.6-1.3 1.6-3.9 0-6.2-1.4-2.3-4-3.3-5.6-2z"/></svg>
|
|
</a>
|
|
|
|
|
|
|
|
<a href="https://hub.docker.com/repository/docker/schmelczera/great-ai" target="_blank" rel="noopener" title="great-ai on DockerHub" class="md-social__link">
|
|
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 640 512"><!--! Font Awesome Free 6.1.1 by @fontawesome - https://fontawesome.com License - https://fontawesome.com/license/free (Icons: CC BY 4.0, Fonts: SIL OFL 1.1, Code: MIT License) Copyright 2022 Fonticons, Inc.--><path d="M349.9 236.3h-66.1v-59.4h66.1v59.4zm0-204.3h-66.1v60.7h66.1V32zm78.2 144.8H362v59.4h66.1v-59.4zm-156.3-72.1h-66.1v60.1h66.1v-60.1zm78.1 0h-66.1v60.1h66.1v-60.1zm276.8 100c-14.4-9.7-47.6-13.2-73.1-8.4-3.3-24-16.7-44.9-41.1-63.7l-14-9.3-9.3 14c-18.4 27.8-23.4 73.6-3.7 103.8-8.7 4.7-25.8 11.1-48.4 10.7H2.4c-8.7 50.8 5.8 116.8 44 162.1 37.1 43.9 92.7 66.2 165.4 66.2 157.4 0 273.9-72.5 328.4-204.2 21.4.4 67.6.1 91.3-45.2 1.5-2.5 6.6-13.2 8.5-17.1l-13.3-8.9zm-511.1-27.9h-66v59.4h66.1v-59.4zm78.1 0h-66.1v59.4h66.1v-59.4zm78.1 0h-66.1v59.4h66.1v-59.4zm-78.1-72.1h-66.1v60.1h66.1v-60.1z"/></svg>
|
|
</a>
|
|
|
|
|
|
|
|
<a href="https://pypi.org/project/great-ai" target="_blank" rel="noopener" title="great-ai on PyPI" class="md-social__link">
|
|
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 448 512"><!--! Font Awesome Free 6.1.1 by @fontawesome - https://fontawesome.com License - https://fontawesome.com/license/free (Icons: CC BY 4.0, Fonts: SIL OFL 1.1, Code: MIT License) Copyright 2022 Fonticons, Inc.--><path d="M439.8 200.5c-7.7-30.9-22.3-54.2-53.4-54.2h-40.1v47.4c0 36.8-31.2 67.8-66.8 67.8H172.7c-29.2 0-53.4 25-53.4 54.3v101.8c0 29 25.2 46 53.4 54.3 33.8 9.9 66.3 11.7 106.8 0 26.9-7.8 53.4-23.5 53.4-54.3v-40.7H226.2v-13.6h160.2c31.1 0 42.6-21.7 53.4-54.2 11.2-33.5 10.7-65.7 0-108.6zM286.2 404c11.1 0 20.1 9.1 20.1 20.3 0 11.3-9 20.4-20.1 20.4-11 0-20.1-9.2-20.1-20.4.1-11.3 9.1-20.3 20.1-20.3zM167.8 248.1h106.8c29.7 0 53.4-24.5 53.4-54.3V91.9c0-29-24.4-50.7-53.4-55.6-35.8-5.9-74.7-5.6-106.8.1-45.2 8-53.4 24.7-53.4 55.6v40.7h106.9v13.6h-147c-31.1 0-58.3 18.7-66.8 54.2-9.8 40.7-10.2 66.1 0 108.6 7.6 31.6 25.7 54.2 56.8 54.2H101v-48.8c0-35.3 30.5-66.4 66.8-66.4zm-6.7-142.6c-11.1 0-20.1-9.1-20.1-20.3.1-11.3 9-20.4 20.1-20.4 11 0 20.1 9.2 20.1 20.4s-9 20.3-20.1 20.3z"/></svg>
|
|
</a>
|
|
|
|
</div>
|
|
|
|
</div>
|
|
</div>
|
|
</footer>
|
|
|
|
</div>
|
|
<div class="md-dialog" data-md-component="dialog">
|
|
<div class="md-dialog__inner md-typeset"></div>
|
|
</div>
|
|
<script id="__config" type="application/json">{"base": "../..", "features": ["content.code.annotate", "content.tooltips"], "search": "../../assets/javascripts/workers/search.b97dbffb.min.js", "translations": {"clipboard.copied": "Copied to clipboard", "clipboard.copy": "Copy to clipboard", "search.config.lang": "en", "search.config.pipeline": "trimmer, stopWordFilter", "search.config.separator": "[\\s\\-]+", "search.placeholder": "Search", "search.result.more.one": "1 more on this page", "search.result.more.other": "# more on this page", "search.result.none": "No matching documents", "search.result.one": "1 matching document", "search.result.other": "# matching documents", "search.result.placeholder": "Type to start searching", "search.result.term.missing": "Missing", "select.version.title": "Select version"}}</script>
|
|
|
|
|
|
<script src="../../assets/javascripts/bundle.6c7ad80a.min.js"></script>
|
|
|
|
|
|
</body>
|
|
</html> |