andras/great-ai
1
0
Fork 0
Code Issues Pull requests 1 Projects Releases 4 Packages Wiki Activity Actions

Add colours to logging

Browse source 
Signed-off-by: András Schmelczer <andras@schmelczer.dev>
This commit is contained in:
Andras Schmelczer 2022-04-10 09:41:04 +02:00
parent 85a06096bf
commit 3730a884b0
16 changed files with 84 additions and 268 deletions
Download patch file Download diff file Expand all files Collapse all files

1
.vscode/settings.json vendored
View file

@ -4,6 +4,7 @@
"botocore",
"iloc",
"inplace",
"levelno",
"plotly",
"pydantic",
"pyplot",

12
example/train.ipynb
View file

@ -60,11 +60,13 @@
"name": "stderr",
"output_type": "stream",
"text": [
"INFO:good_ai:Defaults: configured ✅\n",
"INFO:open_s3:Fetching online versions of data\n",
"INFO:open_s3:Found versions: [0, 1]\n",
"INFO:open_s3:Lastest version of data-1 is 1\n",
"INFO:open_s3:File data-1 found in cache\n"
"\u001b[38;5;39m2022-04-10 09:40:33,153 | INFO | Running in production mode ✅\u001b[0m\n",
"\u001b[38;5;226m2022-04-10 09:40:33,156 | WARNING | The selected persistence driver (TinyDbDriver) is not threadsafe\u001b[0m\n",
"\u001b[38;5;39m2022-04-10 09:40:33,157 | INFO | Defaults: configured ✅\u001b[0m\n",
"\u001b[38;5;39m2022-04-10 09:40:33,348 | INFO | Fetching online versions of data\u001b[0m\n",
"\u001b[38;5;39m2022-04-10 09:40:33,743 | INFO | Found versions: [0, 1]\u001b[0m\n",
"\u001b[38;5;39m2022-04-10 09:40:33,744 | INFO | Lastest version of data-1 is 1\u001b[0m\n",
"\u001b[38;5;39m2022-04-10 09:40:33,745 | INFO | File data-1 found in cache\u001b[0m\n"
]
}
],

3
good_ai/src/good_ai/good_ai/context/context.py
View file

@ -1,3 +1,5 @@
from logging import Logger
from pydantic import BaseModel
from ..persistence import PersistenceDriver
@ -8,6 +10,7 @@ class Context(BaseModel):
persistence: PersistenceDriver
is_production: bool
is_threadsafe: bool
logger: Logger
class Config:
arbitrary_types_allowed = True

22
good_ai/src/good_ai/good_ai/context/get_context.py
View file

@ -1,17 +1,15 @@
import logging
import os
import random
from logging import INFO, Logger
from pathlib import Path
from typing import Optional, cast
from good_ai.open_s3 import LargeFile
from good_ai.utilities.logger import create_logger
from ..persistence import PersistenceDriver, TinyDbDriver
from .context import Context
logger = logging.getLogger("good_ai")
_context: Optional[Context] = None
PRODUCTION_KEY = "production"
@ -24,17 +22,20 @@ def get_context() -> Context:
def set_default_config(
log_level: int = logging.INFO,
log_level: int = INFO,
s3_config: Path = Path("s3.ini"),
seed: int = 42,
persistence_driver: PersistenceDriver = TinyDbDriver(Path("tracing_database.json")),
is_production_mode_override: Optional[bool] = None,
) -> None:
global _context
logging.basicConfig(level=log_level)
is_production = _is_in_production_mode(override=is_production_mode_override)
_initialize_large_file(s3_config)
logger = create_logger("good_ai", level=log_level)
is_production = _is_in_production_mode(
override=is_production_mode_override, logger=logger
)
_initialize_large_file(s3_config, logger=logger)
_set_seed(seed)
is_threadsafe = not isinstance(persistence_driver, TinyDbDriver)
@ -45,12 +46,13 @@ def set_default_config(
persistence=persistence_driver,
is_production=is_production,
is_threadsafe=is_threadsafe,
logger=logger,
)
logger.info("Defaults: configured ✅")
def _is_in_production_mode(override: Optional[bool]) -> bool:
def _is_in_production_mode(override: Optional[bool], logger: Logger) -> bool:
environment = os.environ.get("ENVIRONMENT", PRODUCTION_KEY).lower()
is_production = environment == PRODUCTION_KEY if override is None else override
@ -62,7 +64,7 @@ def _is_in_production_mode(override: Optional[bool]) -> bool:
return is_production
def _initialize_large_file(s3_config: Path) -> None:
def _initialize_large_file(s3_config: Path, logger: Logger) -> None:
if s3_config.exists():
LargeFile.configure_credentials_from_file(s3_config)
else:

5
good_ai/src/good_ai/good_ai/deploy/process_batch.py
View file

@ -1,4 +1,3 @@
import logging
from typing import Any, Callable, Iterable, Optional, Sequence
from good_ai.utilities.parallel_map import parallel_map
@ -7,8 +6,6 @@ from ..context import get_context
from ..tracing import TracingContext
from ..views import Trace
logger = logging.getLogger("good_ai")
def process_batch(
function: Callable[..., Any],
@ -23,6 +20,6 @@ def process_batch(
if not get_context().is_threadsafe:
concurrency = 1
logger.warn("Concurrency is ignored")
get_context().logger.warn("Concurrency is ignored")
return parallel_map(inner, batch, concurrency=concurrency)

3
good_ai/src/good_ai/good_ai/models/load_model.py
View file

@ -1,4 +1,3 @@
import logging
from typing import Any, Optional, Tuple
from joblib import load
@ -7,8 +6,6 @@ from good_ai.open_s3 import LargeFile
from ..context import get_context
logger = logging.getLogger("models")
def load_model(
key: str, version: Optional[int] = None, return_path: bool = False

6
good_ai/src/good_ai/good_ai/models/save_model.py
View file

@ -1,13 +1,11 @@
import logging
from pathlib import Path
from typing import Optional, Union
from joblib import dump
from good_ai.good_ai.context.get_context import get_context
from good_ai.open_s3 import LargeFile
logger = logging.getLogger("models")
from ..context import get_context
def save_model(
@ -23,6 +21,6 @@ def save_model(
with file as f:
dump(model, f)
logger.info(f"Model {key} uploaded with version {file.version}")
get_context().logger.info(f"Model {key} uploaded with version {file.version}")
return file.version

5
good_ai/src/good_ai/good_ai/tracing/tracing_context.py
View file

@ -1,4 +1,3 @@
import logging
import threading
from collections import defaultdict
from datetime import datetime
@ -8,8 +7,6 @@ from typing import Any, DefaultDict, Dict, List, Optional, Type
from ..context import get_context
from ..views import Model, Trace
logger = logging.getLogger("good_ai")
class TracingContext:
_contexts: DefaultDict[int, List["TracingContext"]] = defaultdict(lambda: [])
@ -64,6 +61,6 @@ class TracingContext:
assert self._trace is not None
get_context().persistence.save_document(self._trace)
else:
logger.exception(f"Could not finish operation: {exception}")
get_context().logger.exception(f"Could not finish operation: {exception}")
return True

9
good_ai/src/good_ai/open_s3/__main__.py
View file

@ -1,19 +1,20 @@
#!/usr/bin/env python3
import logging
from pathlib import Path
from good_ai.utilities.logger import create_logger
from .large_file import LargeFile
from .parse_arguments import parse_arguments
if __name__ == "__main__":
logging.basicConfig(level=logging.INFO)
logger = create_logger("open_s3")
parser, args = parse_arguments()
LargeFile.configure_credentials_from_file(args.secrets)
if not args.cache and not args.push and not args.delete:
logging.warn("No action required.")
logger.warn("No action required.")
parser.print_help()
try:
if args.cache:
@ -32,4 +33,4 @@ if __name__ == "__main__":
for f in args.delete:
LargeFile(f).delete()
except Exception as e:
logging.error(e)
logger.error(e)

5
good_ai/src/good_ai/open_s3/large_file.py
View file

@ -1,5 +1,4 @@
import configparser
import logging
import os
import shutil
import tempfile
@ -9,9 +8,11 @@ from typing import IO, Any, List, Mapping, Optional, Type, Union, cast
import boto3
from good_ai.utilities.logger import create_logger
from .helper import DownloadProgressBar, UploadProgressBar, human_readable_to_byte
logger = logging.getLogger("open_s3")
logger = create_logger("open_s3")
class LargeFile:

5
good_ai/src/good_ai/utilities/clean.py
View file

@ -1,5 +1,4 @@
import html
import logging
import re
import unicodedata
@ -7,8 +6,10 @@ import unidecode
from .data import left_regular_punctuations, right_regular_punctuations
from .external.pylatexenc.latex2text import LatexNodes2Text
from .logger.create_logger import create_logger
logger = create_logger("clean")
logger = logging.getLogger("clean")
latex = LatexNodes2Text()

230
good_ai/src/good_ai/utilities/external/pylatexenc/latexwalker/__main__.py vendored
View file

@ -1,230 +0,0 @@
# -*- coding: utf-8 -*-
#
# The MIT License (MIT)
#
# Copyright (c) 2018 Philippe Faist
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
# THE SOFTWARE.
#
import argparse
import fileinput
import json
import logging
import sys
from ..latexwalker import LatexWalker, disp_node, make_json_encoder
from ..version import version_str
def main(argv=None):
if argv is None:
argv = sys.argv[1:]
parser = argparse.ArgumentParser(prog="latexwalker", add_help=False)
parser.add_argument(
"--output-format",
metavar="FORMAT",
dest="output_format",
choices=["human", "json"],
default="human",
help='Requested output format for the node tree ("human" or "json")',
)
parser.add_argument(
"--json-indent",
metavar="NUMSPACES",
dest="json_indent",
type=int,
default=2,
help="Indentation in JSON output (specify number of spaces "
"per indentation level)",
)
parser.add_argument(
"--json-compact",
dest="json_indent",
action="store_const",
const=None,
help="Output compact JSON",
)
parser.add_argument(
"--keep-inline-math",
action="store_const",
const=True,
dest="keep_inline_math",
default=True,
help=argparse.SUPPRESS,
)
parser.add_argument(
"--no-keep-inline-math",
action="store_const",
const=False,
dest="keep_inline_math",
help=argparse.SUPPRESS,
)
parser.add_argument(
"--tolerant-parsing",
action="store_const",
const=True,
dest="tolerant_parsing",
default=True,
)
parser.add_argument(
"--no-tolerant-parsing",
action="store_const",
const=False,
dest="tolerant_parsing",
help="Tolerate syntax errors when parsing, and attempt "
"to continue (default yes)",
)
# I'm not sure this flag is useful and if it should be exposed at all.
# Accept it, but make it hidden.
parser.add_argument(
"--strict-braces",
action="store_const",
const=True,
dest="strict_braces",
default=False,
help=argparse.SUPPRESS,
)
parser.add_argument(
"--no-strict-braces",
action="store_const",
const=False,
dest="strict_braces",
# help="Report errors for mismatching LaTeX braces (default no)"
help=argparse.SUPPRESS,
)
parser.add_argument(
"-q",
"--quiet",
dest="logging_level",
action="store_const",
const=logging.ERROR,
default=logging.INFO,
help="Suppress warning messages",
)
parser.add_argument(
"-v",
"--verbose",
dest="logging_level",
action="store_const",
const=logging.DEBUG,
help="Verbose output",
)
parser.add_argument(
"--version",
action="version",
version="pylatexenc {}".format(version_str),
help="Show version information and exit",
)
parser.add_argument(
"--help", action="help", help="Show this help information and exit"
)
parser.add_argument(
"--code",
"-c",
action="store",
default=None,
metavar="LATEX_CODE",
help="Convert the given LATEX_CODE to unicode text instead of reading "
"from FILE or standard input. You cannot specify FILEs if you use this "
"option, and any standard input is ignored.",
)
parser.add_argument(
"files",
metavar="FILE",
nargs="*",
help="Input files (if none specified, read from stdandard input)",
)
args = parser.parse_args(argv)
logging.basicConfig()
logging.getLogger().setLevel(args.logging_level)
logger = logging.getLogger(__name__)
latex = ""
if args.code:
if args.files:
logger.error(
"Cannot specify both FILEs and --code option. "
"Use --help option for more information."
)
sys.exit(1)
latex = args.code
else:
for line in fileinput.input(files=args.files):
latex += line
latexwalker = LatexWalker(
latex, tolerant_parsing=args.tolerant_parsing, strict_braces=args.strict_braces
)
(nodelist, pos, len_) = latexwalker.get_latex_nodes()
if args.output_format == "human":
print("\n--- NODES ---\n")
for n in nodelist:
disp_node(n)
print("\n-------------\n")
return
if args.output_format == "json":
json.dump(
{
"nodelist": nodelist,
},
sys.stdout,
cls=make_json_encoder(latexwalker),
indent=args.json_indent,
)
sys.stdout.write("\n")
return
raise ValueError("Invalid output format: " + args.output_format)
def run_main():
try:
main()
except SystemExit:
raise
except: # lgtm [py/catch-base-exception]
import pdb
import traceback
traceback.print_exc()
pdb.post_mortem()
if __name__ == "__main__":
# run_main() # debug
main()

1
good_ai/src/good_ai/utilities/logger/__init__.py Normal file
View file

@ -0,0 +1 @@
from .create_logger import create_logger

6
good_ai/src/good_ai/utilities/logger/colors.py Normal file
View file

@ -0,0 +1,6 @@
GREY = "\x1b[38;21m"
BLUE = "\x1b[38;5;39m"
YELLOW = "\x1b[38;5;226m"
RED = "\x1b[38;5;196m"
BOLD_RED = "\x1b[31;1m"
RESET = "\x1b[0m"

18
good_ai/src/good_ai/utilities/logger/create_logger.py Normal file
View file

@ -0,0 +1,18 @@
import logging
from .custom_formatter import CustomFormatter
def create_logger(name: str, level: int = logging.INFO) -> logging.Logger:
logger = logging.getLogger(name)
logger.setLevel(level)
fmt = "%(asctime)s | %(levelname)8s | %(message)s"
stdout_handler = logging.StreamHandler()
stdout_handler.setLevel(logging.DEBUG)
stdout_handler.setFormatter(CustomFormatter(fmt))
logger.addHandler(stdout_handler)
return logger

21
good_ai/src/good_ai/utilities/logger/custom_formatter.py Normal file
View file

@ -0,0 +1,21 @@
import logging
from .colors import BLUE, BOLD_RED, GREY, RED, RESET, YELLOW
class CustomFormatter(logging.Formatter):
def __init__(self, fmt: str):
super().__init__()
self.fmt = fmt
self.FORMATS = {
logging.DEBUG: GREY + self.fmt + RESET,
logging.INFO: BLUE + self.fmt + RESET,
logging.WARNING: YELLOW + self.fmt + RESET,
logging.ERROR: RED + self.fmt + RESET,
logging.CRITICAL: BOLD_RED + self.fmt + RESET,
}
def format(self, record: logging.LogRecord) -> None:
log_fmt = self.FORMATS.get(record.levelno)
formatter = logging.Formatter(log_fmt)
return formatter.format(record)