From d4886273766c4bffd50f437331f7369e2f2b99c3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A1s=20Schmelczer?= Date: Sun, 10 Apr 2022 09:41:04 +0200 Subject: [PATCH] Add colours to logging MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: András Schmelczer --- .vscode/settings.json | 1 + example/train.ipynb | 12 +- .../src/good_ai/good_ai/context/context.py | 3 + .../good_ai/good_ai/context/get_context.py | 22 +- .../good_ai/good_ai/deploy/process_batch.py | 5 +- .../src/good_ai/good_ai/models/load_model.py | 3 - .../src/good_ai/good_ai/models/save_model.py | 6 +- .../good_ai/tracing/tracing_context.py | 5 +- good_ai/src/good_ai/open_s3/__main__.py | 9 +- good_ai/src/good_ai/open_s3/large_file.py | 5 +- good_ai/src/good_ai/utilities/clean.py | 5 +- .../pylatexenc/latexwalker/__main__.py | 230 ------------------ .../src/good_ai/utilities/logger/__init__.py | 1 + .../src/good_ai/utilities/logger/colors.py | 6 + .../good_ai/utilities/logger/create_logger.py | 18 ++ .../utilities/logger/custom_formatter.py | 21 ++ 16 files changed, 84 insertions(+), 268 deletions(-) delete mode 100644 good_ai/src/good_ai/utilities/external/pylatexenc/latexwalker/__main__.py create mode 100644 good_ai/src/good_ai/utilities/logger/__init__.py create mode 100644 good_ai/src/good_ai/utilities/logger/colors.py create mode 100644 good_ai/src/good_ai/utilities/logger/create_logger.py create mode 100644 good_ai/src/good_ai/utilities/logger/custom_formatter.py diff --git a/.vscode/settings.json b/.vscode/settings.json index 61ce14b..c9af87b 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -4,6 +4,7 @@ "botocore", "iloc", "inplace", + "levelno", "plotly", "pydantic", "pyplot", diff --git a/example/train.ipynb b/example/train.ipynb index 79e8072..dc00dfe 100644 --- a/example/train.ipynb +++ b/example/train.ipynb @@ -60,11 +60,13 @@ "name": "stderr", "output_type": "stream", "text": [ - "INFO:good_ai:Defaults: configured ✅\n", - "INFO:open_s3:Fetching online versions of data\n", - "INFO:open_s3:Found versions: [0, 1]\n", - "INFO:open_s3:Lastest version of data-1 is 1\n", - "INFO:open_s3:File data-1 found in cache\n" + "\u001b[38;5;39m2022-04-10 09:40:33,153 | INFO | Running in production mode ✅\u001b[0m\n", + "\u001b[38;5;226m2022-04-10 09:40:33,156 | WARNING | The selected persistence driver (TinyDbDriver) is not threadsafe\u001b[0m\n", + "\u001b[38;5;39m2022-04-10 09:40:33,157 | INFO | Defaults: configured ✅\u001b[0m\n", + "\u001b[38;5;39m2022-04-10 09:40:33,348 | INFO | Fetching online versions of data\u001b[0m\n", + "\u001b[38;5;39m2022-04-10 09:40:33,743 | INFO | Found versions: [0, 1]\u001b[0m\n", + "\u001b[38;5;39m2022-04-10 09:40:33,744 | INFO | Lastest version of data-1 is 1\u001b[0m\n", + "\u001b[38;5;39m2022-04-10 09:40:33,745 | INFO | File data-1 found in cache\u001b[0m\n" ] } ], diff --git a/good_ai/src/good_ai/good_ai/context/context.py b/good_ai/src/good_ai/good_ai/context/context.py index 53e040c..078a7b6 100644 --- a/good_ai/src/good_ai/good_ai/context/context.py +++ b/good_ai/src/good_ai/good_ai/context/context.py @@ -1,3 +1,5 @@ +from logging import Logger + from pydantic import BaseModel from ..persistence import PersistenceDriver @@ -8,6 +10,7 @@ class Context(BaseModel): persistence: PersistenceDriver is_production: bool is_threadsafe: bool + logger: Logger class Config: arbitrary_types_allowed = True diff --git a/good_ai/src/good_ai/good_ai/context/get_context.py b/good_ai/src/good_ai/good_ai/context/get_context.py index f8ae061..1a6d4f7 100644 --- a/good_ai/src/good_ai/good_ai/context/get_context.py +++ b/good_ai/src/good_ai/good_ai/context/get_context.py @@ -1,17 +1,15 @@ -import logging import os import random +from logging import INFO, Logger from pathlib import Path from typing import Optional, cast from good_ai.open_s3 import LargeFile +from good_ai.utilities.logger import create_logger from ..persistence import PersistenceDriver, TinyDbDriver from .context import Context -logger = logging.getLogger("good_ai") - - _context: Optional[Context] = None PRODUCTION_KEY = "production" @@ -24,17 +22,20 @@ def get_context() -> Context: def set_default_config( - log_level: int = logging.INFO, + log_level: int = INFO, s3_config: Path = Path("s3.ini"), seed: int = 42, persistence_driver: PersistenceDriver = TinyDbDriver(Path("tracing_database.json")), is_production_mode_override: Optional[bool] = None, ) -> None: global _context - logging.basicConfig(level=log_level) - is_production = _is_in_production_mode(override=is_production_mode_override) - _initialize_large_file(s3_config) + logger = create_logger("good_ai", level=log_level) + + is_production = _is_in_production_mode( + override=is_production_mode_override, logger=logger + ) + _initialize_large_file(s3_config, logger=logger) _set_seed(seed) is_threadsafe = not isinstance(persistence_driver, TinyDbDriver) @@ -45,12 +46,13 @@ def set_default_config( persistence=persistence_driver, is_production=is_production, is_threadsafe=is_threadsafe, + logger=logger, ) logger.info("Defaults: configured ✅") -def _is_in_production_mode(override: Optional[bool]) -> bool: +def _is_in_production_mode(override: Optional[bool], logger: Logger) -> bool: environment = os.environ.get("ENVIRONMENT", PRODUCTION_KEY).lower() is_production = environment == PRODUCTION_KEY if override is None else override @@ -62,7 +64,7 @@ def _is_in_production_mode(override: Optional[bool]) -> bool: return is_production -def _initialize_large_file(s3_config: Path) -> None: +def _initialize_large_file(s3_config: Path, logger: Logger) -> None: if s3_config.exists(): LargeFile.configure_credentials_from_file(s3_config) else: diff --git a/good_ai/src/good_ai/good_ai/deploy/process_batch.py b/good_ai/src/good_ai/good_ai/deploy/process_batch.py index 92885a4..07bc0bb 100644 --- a/good_ai/src/good_ai/good_ai/deploy/process_batch.py +++ b/good_ai/src/good_ai/good_ai/deploy/process_batch.py @@ -1,4 +1,3 @@ -import logging from typing import Any, Callable, Iterable, Optional, Sequence from good_ai.utilities.parallel_map import parallel_map @@ -7,8 +6,6 @@ from ..context import get_context from ..tracing import TracingContext from ..views import Trace -logger = logging.getLogger("good_ai") - def process_batch( function: Callable[..., Any], @@ -23,6 +20,6 @@ def process_batch( if not get_context().is_threadsafe: concurrency = 1 - logger.warn("Concurrency is ignored") + get_context().logger.warn("Concurrency is ignored") return parallel_map(inner, batch, concurrency=concurrency) diff --git a/good_ai/src/good_ai/good_ai/models/load_model.py b/good_ai/src/good_ai/good_ai/models/load_model.py index 147edb0..a8a142a 100644 --- a/good_ai/src/good_ai/good_ai/models/load_model.py +++ b/good_ai/src/good_ai/good_ai/models/load_model.py @@ -1,4 +1,3 @@ -import logging from typing import Any, Optional, Tuple from joblib import load @@ -7,8 +6,6 @@ from good_ai.open_s3 import LargeFile from ..context import get_context -logger = logging.getLogger("models") - def load_model( key: str, version: Optional[int] = None, return_path: bool = False diff --git a/good_ai/src/good_ai/good_ai/models/save_model.py b/good_ai/src/good_ai/good_ai/models/save_model.py index 8053145..27ed293 100644 --- a/good_ai/src/good_ai/good_ai/models/save_model.py +++ b/good_ai/src/good_ai/good_ai/models/save_model.py @@ -1,13 +1,11 @@ -import logging from pathlib import Path from typing import Optional, Union from joblib import dump -from good_ai.good_ai.context.get_context import get_context from good_ai.open_s3 import LargeFile -logger = logging.getLogger("models") +from ..context import get_context def save_model( @@ -23,6 +21,6 @@ def save_model( with file as f: dump(model, f) - logger.info(f"Model {key} uploaded with version {file.version}") + get_context().logger.info(f"Model {key} uploaded with version {file.version}") return file.version diff --git a/good_ai/src/good_ai/good_ai/tracing/tracing_context.py b/good_ai/src/good_ai/good_ai/tracing/tracing_context.py index bf8840e..ec990f2 100644 --- a/good_ai/src/good_ai/good_ai/tracing/tracing_context.py +++ b/good_ai/src/good_ai/good_ai/tracing/tracing_context.py @@ -1,4 +1,3 @@ -import logging import threading from collections import defaultdict from datetime import datetime @@ -8,8 +7,6 @@ from typing import Any, DefaultDict, Dict, List, Optional, Type from ..context import get_context from ..views import Model, Trace -logger = logging.getLogger("good_ai") - class TracingContext: _contexts: DefaultDict[int, List["TracingContext"]] = defaultdict(lambda: []) @@ -64,6 +61,6 @@ class TracingContext: assert self._trace is not None get_context().persistence.save_document(self._trace) else: - logger.exception(f"Could not finish operation: {exception}") + get_context().logger.exception(f"Could not finish operation: {exception}") return True diff --git a/good_ai/src/good_ai/open_s3/__main__.py b/good_ai/src/good_ai/open_s3/__main__.py index 3b96e7c..fd3f7b9 100644 --- a/good_ai/src/good_ai/open_s3/__main__.py +++ b/good_ai/src/good_ai/open_s3/__main__.py @@ -1,19 +1,20 @@ #!/usr/bin/env python3 -import logging from pathlib import Path +from good_ai.utilities.logger import create_logger + from .large_file import LargeFile from .parse_arguments import parse_arguments if __name__ == "__main__": - logging.basicConfig(level=logging.INFO) + logger = create_logger("open_s3") parser, args = parse_arguments() LargeFile.configure_credentials_from_file(args.secrets) if not args.cache and not args.push and not args.delete: - logging.warn("No action required.") + logger.warn("No action required.") parser.print_help() try: if args.cache: @@ -32,4 +33,4 @@ if __name__ == "__main__": for f in args.delete: LargeFile(f).delete() except Exception as e: - logging.error(e) + logger.error(e) diff --git a/good_ai/src/good_ai/open_s3/large_file.py b/good_ai/src/good_ai/open_s3/large_file.py index 137ffa4..1713c87 100644 --- a/good_ai/src/good_ai/open_s3/large_file.py +++ b/good_ai/src/good_ai/open_s3/large_file.py @@ -1,5 +1,4 @@ import configparser -import logging import os import shutil import tempfile @@ -9,9 +8,11 @@ from typing import IO, Any, List, Mapping, Optional, Type, Union, cast import boto3 +from good_ai.utilities.logger import create_logger + from .helper import DownloadProgressBar, UploadProgressBar, human_readable_to_byte -logger = logging.getLogger("open_s3") +logger = create_logger("open_s3") class LargeFile: diff --git a/good_ai/src/good_ai/utilities/clean.py b/good_ai/src/good_ai/utilities/clean.py index d7b5758..c616171 100644 --- a/good_ai/src/good_ai/utilities/clean.py +++ b/good_ai/src/good_ai/utilities/clean.py @@ -1,5 +1,4 @@ import html -import logging import re import unicodedata @@ -7,8 +6,10 @@ import unidecode from .data import left_regular_punctuations, right_regular_punctuations from .external.pylatexenc.latex2text import LatexNodes2Text +from .logger.create_logger import create_logger + +logger = create_logger("clean") -logger = logging.getLogger("clean") latex = LatexNodes2Text() diff --git a/good_ai/src/good_ai/utilities/external/pylatexenc/latexwalker/__main__.py b/good_ai/src/good_ai/utilities/external/pylatexenc/latexwalker/__main__.py deleted file mode 100644 index fe95f0b..0000000 --- a/good_ai/src/good_ai/utilities/external/pylatexenc/latexwalker/__main__.py +++ /dev/null @@ -1,230 +0,0 @@ -# -*- coding: utf-8 -*- -# -# The MIT License (MIT) -# -# Copyright (c) 2018 Philippe Faist -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in -# all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -# THE SOFTWARE. -# - -import argparse -import fileinput -import json -import logging -import sys - -from ..latexwalker import LatexWalker, disp_node, make_json_encoder -from ..version import version_str - - -def main(argv=None): - - if argv is None: - argv = sys.argv[1:] - - parser = argparse.ArgumentParser(prog="latexwalker", add_help=False) - - parser.add_argument( - "--output-format", - metavar="FORMAT", - dest="output_format", - choices=["human", "json"], - default="human", - help='Requested output format for the node tree ("human" or "json")', - ) - parser.add_argument( - "--json-indent", - metavar="NUMSPACES", - dest="json_indent", - type=int, - default=2, - help="Indentation in JSON output (specify number of spaces " - "per indentation level)", - ) - parser.add_argument( - "--json-compact", - dest="json_indent", - action="store_const", - const=None, - help="Output compact JSON", - ) - - parser.add_argument( - "--keep-inline-math", - action="store_const", - const=True, - dest="keep_inline_math", - default=True, - help=argparse.SUPPRESS, - ) - parser.add_argument( - "--no-keep-inline-math", - action="store_const", - const=False, - dest="keep_inline_math", - help=argparse.SUPPRESS, - ) - - parser.add_argument( - "--tolerant-parsing", - action="store_const", - const=True, - dest="tolerant_parsing", - default=True, - ) - parser.add_argument( - "--no-tolerant-parsing", - action="store_const", - const=False, - dest="tolerant_parsing", - help="Tolerate syntax errors when parsing, and attempt " - "to continue (default yes)", - ) - - # I'm not sure this flag is useful and if it should be exposed at all. - # Accept it, but make it hidden. - parser.add_argument( - "--strict-braces", - action="store_const", - const=True, - dest="strict_braces", - default=False, - help=argparse.SUPPRESS, - ) - parser.add_argument( - "--no-strict-braces", - action="store_const", - const=False, - dest="strict_braces", - # help="Report errors for mismatching LaTeX braces (default no)" - help=argparse.SUPPRESS, - ) - - parser.add_argument( - "-q", - "--quiet", - dest="logging_level", - action="store_const", - const=logging.ERROR, - default=logging.INFO, - help="Suppress warning messages", - ) - parser.add_argument( - "-v", - "--verbose", - dest="logging_level", - action="store_const", - const=logging.DEBUG, - help="Verbose output", - ) - parser.add_argument( - "--version", - action="version", - version="pylatexenc {}".format(version_str), - help="Show version information and exit", - ) - parser.add_argument( - "--help", action="help", help="Show this help information and exit" - ) - - parser.add_argument( - "--code", - "-c", - action="store", - default=None, - metavar="LATEX_CODE", - help="Convert the given LATEX_CODE to unicode text instead of reading " - "from FILE or standard input. You cannot specify FILEs if you use this " - "option, and any standard input is ignored.", - ) - - parser.add_argument( - "files", - metavar="FILE", - nargs="*", - help="Input files (if none specified, read from stdandard input)", - ) - - args = parser.parse_args(argv) - - logging.basicConfig() - logging.getLogger().setLevel(args.logging_level) - logger = logging.getLogger(__name__) - - latex = "" - if args.code: - if args.files: - logger.error( - "Cannot specify both FILEs and --code option. " - "Use --help option for more information." - ) - sys.exit(1) - latex = args.code - else: - for line in fileinput.input(files=args.files): - latex += line - - latexwalker = LatexWalker( - latex, tolerant_parsing=args.tolerant_parsing, strict_braces=args.strict_braces - ) - - (nodelist, pos, len_) = latexwalker.get_latex_nodes() - - if args.output_format == "human": - print("\n--- NODES ---\n") - for n in nodelist: - disp_node(n) - print("\n-------------\n") - return - - if args.output_format == "json": - json.dump( - { - "nodelist": nodelist, - }, - sys.stdout, - cls=make_json_encoder(latexwalker), - indent=args.json_indent, - ) - sys.stdout.write("\n") - return - - raise ValueError("Invalid output format: " + args.output_format) - - -def run_main(): - - try: - - main() - - except SystemExit: - raise - except: # lgtm [py/catch-base-exception] - import pdb - import traceback - - traceback.print_exc() - pdb.post_mortem() - - -if __name__ == "__main__": - - # run_main() # debug - main() diff --git a/good_ai/src/good_ai/utilities/logger/__init__.py b/good_ai/src/good_ai/utilities/logger/__init__.py new file mode 100644 index 0000000..9eb451a --- /dev/null +++ b/good_ai/src/good_ai/utilities/logger/__init__.py @@ -0,0 +1 @@ +from .create_logger import create_logger diff --git a/good_ai/src/good_ai/utilities/logger/colors.py b/good_ai/src/good_ai/utilities/logger/colors.py new file mode 100644 index 0000000..8f99e92 --- /dev/null +++ b/good_ai/src/good_ai/utilities/logger/colors.py @@ -0,0 +1,6 @@ +GREY = "\x1b[38;21m" +BLUE = "\x1b[38;5;39m" +YELLOW = "\x1b[38;5;226m" +RED = "\x1b[38;5;196m" +BOLD_RED = "\x1b[31;1m" +RESET = "\x1b[0m" diff --git a/good_ai/src/good_ai/utilities/logger/create_logger.py b/good_ai/src/good_ai/utilities/logger/create_logger.py new file mode 100644 index 0000000..a5ae26c --- /dev/null +++ b/good_ai/src/good_ai/utilities/logger/create_logger.py @@ -0,0 +1,18 @@ +import logging + +from .custom_formatter import CustomFormatter + + +def create_logger(name: str, level: int = logging.INFO) -> logging.Logger: + logger = logging.getLogger(name) + logger.setLevel(level) + + fmt = "%(asctime)s | %(levelname)8s | %(message)s" + + stdout_handler = logging.StreamHandler() + stdout_handler.setLevel(logging.DEBUG) + stdout_handler.setFormatter(CustomFormatter(fmt)) + + logger.addHandler(stdout_handler) + + return logger diff --git a/good_ai/src/good_ai/utilities/logger/custom_formatter.py b/good_ai/src/good_ai/utilities/logger/custom_formatter.py new file mode 100644 index 0000000..a1bcfd0 --- /dev/null +++ b/good_ai/src/good_ai/utilities/logger/custom_formatter.py @@ -0,0 +1,21 @@ +import logging + +from .colors import BLUE, BOLD_RED, GREY, RED, RESET, YELLOW + + +class CustomFormatter(logging.Formatter): + def __init__(self, fmt: str): + super().__init__() + self.fmt = fmt + self.FORMATS = { + logging.DEBUG: GREY + self.fmt + RESET, + logging.INFO: BLUE + self.fmt + RESET, + logging.WARNING: YELLOW + self.fmt + RESET, + logging.ERROR: RED + self.fmt + RESET, + logging.CRITICAL: BOLD_RED + self.fmt + RESET, + } + + def format(self, record: logging.LogRecord) -> None: + log_fmt = self.FORMATS.get(record.levelno) + formatter = logging.Formatter(log_fmt) + return formatter.format(record)