andras/great-ai
1
0
Fork 0
Code Issues Pull requests 1 Projects Releases 4 Packages Wiki Activity Actions

Add colours to logging

Browse source 
Signed-off-by: András Schmelczer <andras@schmelczer.dev>
This commit is contained in:
Andras Schmelczer 2022-04-10 09:41:04 +02:00
parent 85a06096bf
commit 3730a884b0
16 changed files with 84 additions and 268 deletions
Download patch file Download diff file Expand all files Collapse all files

1
.vscode/settings.json vendored
View file

@ -4,6 +4,7 @@
"botocore", "botocore",
"iloc", "iloc",
"inplace", "inplace",
"levelno",
"plotly", "plotly",
"pydantic", "pydantic",
"pyplot", "pyplot",

12
example/train.ipynb
View file

@ -60,11 +60,13 @@
"name": "stderr", "name": "stderr",
"output_type": "stream", "output_type": "stream",
"text": [ "text": [
"INFO:good_ai:Defaults: configured ✅\n", "\u001b[38;5;39m2022-04-10 09:40:33,153 | INFO | Running in production mode ✅\u001b[0m\n",
"INFO:open_s3:Fetching online versions of data\n", "\u001b[38;5;226m2022-04-10 09:40:33,156 | WARNING | The selected persistence driver (TinyDbDriver) is not threadsafe\u001b[0m\n",
"INFO:open_s3:Found versions: [0, 1]\n", "\u001b[38;5;39m2022-04-10 09:40:33,157 | INFO | Defaults: configured ✅\u001b[0m\n",
"INFO:open_s3:Lastest version of data-1 is 1\n", "\u001b[38;5;39m2022-04-10 09:40:33,348 | INFO | Fetching online versions of data\u001b[0m\n",
"INFO:open_s3:File data-1 found in cache\n" "\u001b[38;5;39m2022-04-10 09:40:33,743 | INFO | Found versions: [0, 1]\u001b[0m\n",
"\u001b[38;5;39m2022-04-10 09:40:33,744 | INFO | Lastest version of data-1 is 1\u001b[0m\n",
"\u001b[38;5;39m2022-04-10 09:40:33,745 | INFO | File data-1 found in cache\u001b[0m\n"
] ]
} }
], ],

3
good_ai/src/good_ai/good_ai/context/context.py
View file

@ -1,3 +1,5 @@
from logging import Logger
from pydantic import BaseModel from pydantic import BaseModel
from ..persistence import PersistenceDriver from ..persistence import PersistenceDriver
@ -8,6 +10,7 @@ class Context(BaseModel):
persistence: PersistenceDriver persistence: PersistenceDriver
is_production: bool is_production: bool
is_threadsafe: bool is_threadsafe: bool
logger: Logger
class Config: class Config:
arbitrary_types_allowed = True arbitrary_types_allowed = True

22
good_ai/src/good_ai/good_ai/context/get_context.py
View file

@ -1,17 +1,15 @@
import logging
import os import os
import random import random
from logging import INFO, Logger
from pathlib import Path from pathlib import Path
from typing import Optional, cast from typing import Optional, cast
from good_ai.open_s3 import LargeFile from good_ai.open_s3 import LargeFile
from good_ai.utilities.logger import create_logger
from ..persistence import PersistenceDriver, TinyDbDriver from ..persistence import PersistenceDriver, TinyDbDriver
from .context import Context from .context import Context
logger = logging.getLogger("good_ai")
_context: Optional[Context] = None _context: Optional[Context] = None
PRODUCTION_KEY = "production" PRODUCTION_KEY = "production"
@ -24,17 +22,20 @@ def get_context() -> Context:
def set_default_config( def set_default_config(
log_level: int = logging.INFO, log_level: int = INFO,
s3_config: Path = Path("s3.ini"), s3_config: Path = Path("s3.ini"),
seed: int = 42, seed: int = 42,
persistence_driver: PersistenceDriver = TinyDbDriver(Path("tracing_database.json")), persistence_driver: PersistenceDriver = TinyDbDriver(Path("tracing_database.json")),
is_production_mode_override: Optional[bool] = None, is_production_mode_override: Optional[bool] = None,
) -> None: ) -> None:
global _context global _context
logging.basicConfig(level=log_level)
is_production = _is_in_production_mode(override=is_production_mode_override) logger = create_logger("good_ai", level=log_level)
_initialize_large_file(s3_config)
is_production = _is_in_production_mode(
override=is_production_mode_override, logger=logger
)
_initialize_large_file(s3_config, logger=logger)
_set_seed(seed) _set_seed(seed)
is_threadsafe = not isinstance(persistence_driver, TinyDbDriver) is_threadsafe = not isinstance(persistence_driver, TinyDbDriver)
@ -45,12 +46,13 @@ def set_default_config(
persistence=persistence_driver, persistence=persistence_driver,
is_production=is_production, is_production=is_production,
is_threadsafe=is_threadsafe, is_threadsafe=is_threadsafe,
logger=logger,
) )
logger.info("Defaults: configured ✅") logger.info("Defaults: configured ✅")
def _is_in_production_mode(override: Optional[bool]) -> bool: def _is_in_production_mode(override: Optional[bool], logger: Logger) -> bool:
environment = os.environ.get("ENVIRONMENT", PRODUCTION_KEY).lower() environment = os.environ.get("ENVIRONMENT", PRODUCTION_KEY).lower()
is_production = environment == PRODUCTION_KEY if override is None else override is_production = environment == PRODUCTION_KEY if override is None else override
@ -62,7 +64,7 @@ def _is_in_production_mode(override: Optional[bool]) -> bool:
return is_production return is_production
def _initialize_large_file(s3_config: Path) -> None: def _initialize_large_file(s3_config: Path, logger: Logger) -> None:
if s3_config.exists(): if s3_config.exists():
LargeFile.configure_credentials_from_file(s3_config) LargeFile.configure_credentials_from_file(s3_config)
else: else:

5
good_ai/src/good_ai/good_ai/deploy/process_batch.py
View file

@ -1,4 +1,3 @@
import logging
from typing import Any, Callable, Iterable, Optional, Sequence from typing import Any, Callable, Iterable, Optional, Sequence
from good_ai.utilities.parallel_map import parallel_map from good_ai.utilities.parallel_map import parallel_map
@ -7,8 +6,6 @@ from ..context import get_context
from ..tracing import TracingContext from ..tracing import TracingContext
from ..views import Trace from ..views import Trace
logger = logging.getLogger("good_ai")
def process_batch( def process_batch(
function: Callable[..., Any], function: Callable[..., Any],
@ -23,6 +20,6 @@ def process_batch(
if not get_context().is_threadsafe: if not get_context().is_threadsafe:
concurrency = 1 concurrency = 1
logger.warn("Concurrency is ignored") get_context().logger.warn("Concurrency is ignored")
return parallel_map(inner, batch, concurrency=concurrency) return parallel_map(inner, batch, concurrency=concurrency)

3
good_ai/src/good_ai/good_ai/models/load_model.py
View file

@ -1,4 +1,3 @@
import logging
from typing import Any, Optional, Tuple from typing import Any, Optional, Tuple
from joblib import load from joblib import load
@ -7,8 +6,6 @@ from good_ai.open_s3 import LargeFile
from ..context import get_context from ..context import get_context
logger = logging.getLogger("models")
def load_model( def load_model(
key: str, version: Optional[int] = None, return_path: bool = False key: str, version: Optional[int] = None, return_path: bool = False

6
good_ai/src/good_ai/good_ai/models/save_model.py
View file

@ -1,13 +1,11 @@
import logging
from pathlib import Path from pathlib import Path
from typing import Optional, Union from typing import Optional, Union
from joblib import dump from joblib import dump
from good_ai.good_ai.context.get_context import get_context
from good_ai.open_s3 import LargeFile from good_ai.open_s3 import LargeFile
logger = logging.getLogger("models") from ..context import get_context
def save_model( def save_model(
@ -23,6 +21,6 @@ def save_model(
with file as f: with file as f:
dump(model, f) dump(model, f)
logger.info(f"Model {key} uploaded with version {file.version}") get_context().logger.info(f"Model {key} uploaded with version {file.version}")
return file.version return file.version

5
good_ai/src/good_ai/good_ai/tracing/tracing_context.py
View file

@ -1,4 +1,3 @@
import logging
import threading import threading
from collections import defaultdict from collections import defaultdict
from datetime import datetime from datetime import datetime
@ -8,8 +7,6 @@ from typing import Any, DefaultDict, Dict, List, Optional, Type
from ..context import get_context from ..context import get_context
from ..views import Model, Trace from ..views import Model, Trace
logger = logging.getLogger("good_ai")
class TracingContext: class TracingContext:
_contexts: DefaultDict[int, List["TracingContext"]] = defaultdict(lambda: []) _contexts: DefaultDict[int, List["TracingContext"]] = defaultdict(lambda: [])
@ -64,6 +61,6 @@ class TracingContext:
assert self._trace is not None assert self._trace is not None
get_context().persistence.save_document(self._trace) get_context().persistence.save_document(self._trace)
else: else:
logger.exception(f"Could not finish operation: {exception}") get_context().logger.exception(f"Could not finish operation: {exception}")
return True return True

9
good_ai/src/good_ai/open_s3/__main__.py
View file

@ -1,19 +1,20 @@
#!/usr/bin/env python3 #!/usr/bin/env python3
import logging
from pathlib import Path from pathlib import Path
from good_ai.utilities.logger import create_logger
from .large_file import LargeFile from .large_file import LargeFile
from .parse_arguments import parse_arguments from .parse_arguments import parse_arguments
if __name__ == "__main__": if __name__ == "__main__":
logging.basicConfig(level=logging.INFO) logger = create_logger("open_s3")
parser, args = parse_arguments() parser, args = parse_arguments()
LargeFile.configure_credentials_from_file(args.secrets) LargeFile.configure_credentials_from_file(args.secrets)
if not args.cache and not args.push and not args.delete: if not args.cache and not args.push and not args.delete:
logging.warn("No action required.") logger.warn("No action required.")
parser.print_help() parser.print_help()
try: try:
if args.cache: if args.cache:
@ -32,4 +33,4 @@ if __name__ == "__main__":
for f in args.delete: for f in args.delete:
LargeFile(f).delete() LargeFile(f).delete()
except Exception as e: except Exception as e:
logging.error(e) logger.error(e)

5
good_ai/src/good_ai/open_s3/large_file.py
View file

@ -1,5 +1,4 @@
import configparser import configparser
import logging
import os import os
import shutil import shutil
import tempfile import tempfile
@ -9,9 +8,11 @@ from typing import IO, Any, List, Mapping, Optional, Type, Union, cast
import boto3 import boto3
from good_ai.utilities.logger import create_logger
from .helper import DownloadProgressBar, UploadProgressBar, human_readable_to_byte from .helper import DownloadProgressBar, UploadProgressBar, human_readable_to_byte
logger = logging.getLogger("open_s3") logger = create_logger("open_s3")
class LargeFile: class LargeFile:

5
good_ai/src/good_ai/utilities/clean.py
View file

@ -1,5 +1,4 @@
import html import html
import logging
import re import re
import unicodedata import unicodedata
@ -7,8 +6,10 @@ import unidecode
from .data import left_regular_punctuations, right_regular_punctuations from .data import left_regular_punctuations, right_regular_punctuations
from .external.pylatexenc.latex2text import LatexNodes2Text from .external.pylatexenc.latex2text import LatexNodes2Text
from .logger.create_logger import create_logger
logger = create_logger("clean")
logger = logging.getLogger("clean")
latex = LatexNodes2Text() latex = LatexNodes2Text()

230
good_ai/src/good_ai/utilities/external/pylatexenc/latexwalker/__main__.py vendored
View file

@ -1,230 +0,0 @@
# -*- coding: utf-8 -*-
#
# The MIT License (MIT)
#
# Copyright (c) 2018 Philippe Faist
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
# THE SOFTWARE.
#
import argparse
import fileinput
import json
import logging
import sys
from ..latexwalker import LatexWalker, disp_node, make_json_encoder
from ..version import version_str
def main(argv=None):
if argv is None:
argv = sys.argv[1:]
parser = argparse.ArgumentParser(prog="latexwalker", add_help=False)
parser.add_argument(
"--output-format",
metavar="FORMAT",
dest="output_format",
choices=["human", "json"],
default="human",
help='Requested output format for the node tree ("human" or "json")',
)
parser.add_argument(
"--json-indent",
metavar="NUMSPACES",
dest="json_indent",
type=int,
default=2,
help="Indentation in JSON output (specify number of spaces "
"per indentation level)",
)
parser.add_argument(
"--json-compact",
dest="json_indent",
action="store_const",
const=None,
help="Output compact JSON",
)
parser.add_argument(
"--keep-inline-math",
action="store_const",
const=True,
dest="keep_inline_math",
default=True,
help=argparse.SUPPRESS,
)
parser.add_argument(
"--no-keep-inline-math",
action="store_const",
const=False,
dest="keep_inline_math",
help=argparse.SUPPRESS,
)
parser.add_argument(
"--tolerant-parsing",
action="store_const",
const=True,
dest="tolerant_parsing",
default=True,
)
parser.add_argument(
"--no-tolerant-parsing",
action="store_const",
const=False,
dest="tolerant_parsing",
help="Tolerate syntax errors when parsing, and attempt "
"to continue (default yes)",
)
# I'm not sure this flag is useful and if it should be exposed at all.
# Accept it, but make it hidden.
parser.add_argument(
"--strict-braces",
action="store_const",
const=True,
dest="strict_braces",
default=False,
help=argparse.SUPPRESS,
)
parser.add_argument(
"--no-strict-braces",
action="store_const",
const=False,
dest="strict_braces",
# help="Report errors for mismatching LaTeX braces (default no)"
help=argparse.SUPPRESS,
)
parser.add_argument(
"-q",
"--quiet",
dest="logging_level",
action="store_const",
const=logging.ERROR,
default=logging.INFO,
help="Suppress warning messages",
)
parser.add_argument(
"-v",
"--verbose",
dest="logging_level",
action="store_const",
const=logging.DEBUG,
help="Verbose output",
)
parser.add_argument(
"--version",
action="version",
version="pylatexenc {}".format(version_str),
help="Show version information and exit",
)
parser.add_argument(
"--help", action="help", help="Show this help information and exit"
)
parser.add_argument(
"--code",
"-c",
action="store",
default=None,
metavar="LATEX_CODE",
help="Convert the given LATEX_CODE to unicode text instead of reading "
"from FILE or standard input. You cannot specify FILEs if you use this "
"option, and any standard input is ignored.",
)
parser.add_argument(
"files",
metavar="FILE",
nargs="*",
help="Input files (if none specified, read from stdandard input)",
)
args = parser.parse_args(argv)
logging.basicConfig()
logging.getLogger().setLevel(args.logging_level)
logger = logging.getLogger(__name__)
latex = ""
if args.code:
if args.files:
logger.error(
"Cannot specify both FILEs and --code option. "
"Use --help option for more information."
)
sys.exit(1)
latex = args.code
else:
for line in fileinput.input(files=args.files):
latex += line
latexwalker = LatexWalker(
latex, tolerant_parsing=args.tolerant_parsing, strict_braces=args.strict_braces
)
(nodelist, pos, len_) = latexwalker.get_latex_nodes()
if args.output_format == "human":
print("\n--- NODES ---\n")
for n in nodelist:
disp_node(n)
print("\n-------------\n")
return
if args.output_format == "json":
json.dump(
{
"nodelist": nodelist,
},
sys.stdout,
cls=make_json_encoder(latexwalker),
indent=args.json_indent,
)
sys.stdout.write("\n")
return
raise ValueError("Invalid output format: " + args.output_format)
def run_main():
try:
main()
except SystemExit:
raise
except: # lgtm [py/catch-base-exception]
import pdb
import traceback
traceback.print_exc()
pdb.post_mortem()
if __name__ == "__main__":
# run_main() # debug
main()

1
good_ai/src/good_ai/utilities/logger/__init__.py Normal file
View file

@ -0,0 +1 @@
from .create_logger import create_logger

6
good_ai/src/good_ai/utilities/logger/colors.py Normal file
View file

@ -0,0 +1,6 @@
GREY = "\x1b[38;21m"
BLUE = "\x1b[38;5;39m"
YELLOW = "\x1b[38;5;226m"
RED = "\x1b[38;5;196m"
BOLD_RED = "\x1b[31;1m"
RESET = "\x1b[0m"

18
good_ai/src/good_ai/utilities/logger/create_logger.py Normal file
View file

@ -0,0 +1,18 @@
import logging
from .custom_formatter import CustomFormatter
def create_logger(name: str, level: int = logging.INFO) -> logging.Logger:
logger = logging.getLogger(name)
logger.setLevel(level)
fmt = "%(asctime)s | %(levelname)8s | %(message)s"
stdout_handler = logging.StreamHandler()
stdout_handler.setLevel(logging.DEBUG)
stdout_handler.setFormatter(CustomFormatter(fmt))
logger.addHandler(stdout_handler)
return logger

21
good_ai/src/good_ai/utilities/logger/custom_formatter.py Normal file
View file

@ -0,0 +1,21 @@
import logging
from .colors import BLUE, BOLD_RED, GREY, RED, RESET, YELLOW
class CustomFormatter(logging.Formatter):
def __init__(self, fmt: str):
super().__init__()
self.fmt = fmt
self.FORMATS = {
logging.DEBUG: GREY + self.fmt + RESET,
logging.INFO: BLUE + self.fmt + RESET,
logging.WARNING: YELLOW + self.fmt + RESET,
logging.ERROR: RED + self.fmt + RESET,
logging.CRITICAL: BOLD_RED + self.fmt + RESET,
}
def format(self, record: logging.LogRecord) -> None:
log_fmt = self.FORMATS.get(record.levelno)
formatter = logging.Formatter(log_fmt)
return formatter.format(record)