From 7e0150d9db146648fa87e37a7918c9f8f0ee2e62 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 13 Jul 2021 06:43:15 +0200 Subject: [PATCH 1/5] [pre-commit.ci] pre-commit autoupdate (#333) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit updates: - [github.com/PyCQA/isort: 5.9.1 → 5.9.2](https://github.com/PyCQA/isort/compare/5.9.1...5.9.2) Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 94690045..043863fc 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -7,7 +7,7 @@ repos: language_version: python3.8 - repo: https://github.com/PyCQA/isort - rev: 5.9.1 + rev: 5.9.2 hooks: - id: isort additional_dependencies: [toml] From d110bf9d8dab5581abe1ac23f7ee21ed0d7b94e0 Mon Sep 17 00:00:00 2001 From: charlie Date: Fri, 25 Jun 2021 14:56:54 +0200 Subject: [PATCH 2/5] Add initial logfmt approximation --- src/structlog/processors.py | 21 +++++++++++++++++++++ tests/test_processors.py | 11 +++++++++++ 2 files changed, 32 insertions(+) diff --git a/src/structlog/processors.py b/src/structlog/processors.py index 4fcc074d..7591a233 100644 --- a/src/structlog/processors.py +++ b/src/structlog/processors.py @@ -8,6 +8,7 @@ import datetime import json import operator +import re import sys import time @@ -43,6 +44,7 @@ "format_exc_info", "ExceptionPrettyPrinter", "StackInfoRenderer", + "LogFmtRenderer", ] @@ -446,3 +448,22 @@ def __call__( ) return event_dict + + +class LogFmtRenderer(KeyValueRenderer): + """ + Inherits from KeyValueRenderer to make output be logfmt compatible. + + Useful with logfmt parsers like that of Loki. + """ + + def _logfmt_repr(self, inst: Any): + if isinstance(inst, str): + return json.dumps(inst) if re.search("\\W", inst) else inst + return repr(inst) + + def __call__(self, _: WrappedLogger, __, event_dict: EventDict): + return " ".join( + k + "=" + self._logfmt_repr(v) + for k, v in self._ordered_items(event_dict) + ) diff --git a/tests/test_processors.py b/tests/test_processors.py index 47f2a6cf..5e3b7a7a 100644 --- a/tests/test_processors.py +++ b/tests/test_processors.py @@ -18,6 +18,7 @@ ExceptionPrettyPrinter, JSONRenderer, KeyValueRenderer, + LogFmtRenderer, StackInfoRenderer, TimeStamper, UnicodeDecoder, @@ -193,6 +194,16 @@ def test_simplejson(self, event_dict): } == json.loads(jr(None, None, event_dict)) +class TestLogFmtRenderer: + def test_string_is_not_quoted(self, event_dict): + """ + A string with no special chars should not be quoted. + """ + rv = LogFmtRenderer()(None, None, event_dict) + + assert r"a= b=[3, 4] x=7 y=test z=(1, 2)" == rv + + class TestTimeStamper: def test_disallows_non_utc_unix_timestamps(self): """ From 7d531e4582a10603805154e5bcd747e6834d239f Mon Sep 17 00:00:00 2001 From: charlie Date: Fri, 25 Jun 2021 19:54:25 +0200 Subject: [PATCH 3/5] Injects repr_formatter into KeyValueRenderer --- src/structlog/processors.py | 48 ++++++++++++++++--------------------- tests/test_processors.py | 46 ++++++++++++++++++++++++++--------- 2 files changed, 55 insertions(+), 39 deletions(-) diff --git a/src/structlog/processors.py b/src/structlog/processors.py index 7591a233..35f7ee37 100644 --- a/src/structlog/processors.py +++ b/src/structlog/processors.py @@ -44,10 +44,25 @@ "format_exc_info", "ExceptionPrettyPrinter", "StackInfoRenderer", - "LogFmtRenderer", + "logfmt_repr", ] +def logfmt_repr(inst: Any) -> str: + if inst is None: + return "null" + if isinstance(inst, str): + return json.dumps(inst) if re.search("\\W", inst) else inst + return repr(inst) + + +def _repr(inst: Any) -> str: + if isinstance(inst, str): + return inst + else: + return repr(inst) + + class KeyValueRenderer: """ Render ``event_dict`` as a list of ``Key=repr(Value)`` pairs. @@ -62,10 +77,12 @@ class KeyValueRenderer: to native strings. Setting this to ``False`` is useful if you want to have human-readable non-ASCII output on Python 2. + :param repr_formatter: A callable to use as a custom :func:`repr()`. .. versionadded:: 0.2.0 *key_order* .. versionadded:: 16.1.0 *drop_missing* .. versionadded:: 17.1.0 *repr_native_str* + .. versionadded:: 21.2.0 *repr_formatter* """ def __init__( @@ -74,6 +91,7 @@ def __init__( key_order: Optional[Sequence[str]] = None, drop_missing: bool = False, repr_native_str: bool = True, + repr_formatter: Callable = _repr, ): # Use an optimized version for each case. if key_order and sort_keys is True: @@ -115,14 +133,7 @@ def ordered_items(event_dict: EventDict) -> List[Tuple[str, Any]]: if repr_native_str is True: self._repr = repr else: - - def _repr(inst: Any) -> str: - if isinstance(inst, str): - return inst - else: - return repr(inst) - - self._repr = _repr + self._repr = repr_formatter def __call__( self, _: WrappedLogger, __: str, event_dict: EventDict @@ -448,22 +459,3 @@ def __call__( ) return event_dict - - -class LogFmtRenderer(KeyValueRenderer): - """ - Inherits from KeyValueRenderer to make output be logfmt compatible. - - Useful with logfmt parsers like that of Loki. - """ - - def _logfmt_repr(self, inst: Any): - if isinstance(inst, str): - return json.dumps(inst) if re.search("\\W", inst) else inst - return repr(inst) - - def __call__(self, _: WrappedLogger, __, event_dict: EventDict): - return " ".join( - k + "=" + self._logfmt_repr(v) - for k, v in self._ordered_items(event_dict) - ) diff --git a/tests/test_processors.py b/tests/test_processors.py index 5e3b7a7a..e02623c7 100644 --- a/tests/test_processors.py +++ b/tests/test_processors.py @@ -18,7 +18,6 @@ ExceptionPrettyPrinter, JSONRenderer, KeyValueRenderer, - LogFmtRenderer, StackInfoRenderer, TimeStamper, UnicodeDecoder, @@ -26,6 +25,7 @@ _figure_out_exc_info, _json_fallback_handler, format_exc_info, + logfmt_repr, ) from structlog.threadlocal import wrap_dict @@ -125,6 +125,16 @@ def test_repr_native_str(self, rns): cnt = rv.count("哈") assert 2 == cnt + def test_repr_formatter(self, event_dict): + """ + Injects a custom repr-like formatter when repr_native_str=False + """ + rv = KeyValueRenderer( + repr_formatter=logfmt_repr, repr_native_str=False + )(None, None, event_dict) + + assert r"a= b=[3, 4] x=7 y=test z=(1, 2)" == rv + class TestJSONRenderer: def test_renders_json(self, event_dict): @@ -194,16 +204,6 @@ def test_simplejson(self, event_dict): } == json.loads(jr(None, None, event_dict)) -class TestLogFmtRenderer: - def test_string_is_not_quoted(self, event_dict): - """ - A string with no special chars should not be quoted. - """ - rv = LogFmtRenderer()(None, None, event_dict) - - assert r"a= b=[3, 4] x=7 y=test z=(1, 2)" == rv - - class TestTimeStamper: def test_disallows_non_utc_unix_timestamps(self): """ @@ -562,3 +562,27 @@ def test_py3_exception_no_traceback(self): e = ValueError() assert (e.__class__, e, None) == _figure_out_exc_info(e) + + +class TestLogFmtRepr: + @pytest.mark.parametrize( + "obj,expected", + [ + ("word", "word"), # plain word + ("a sentence", '"a sentence"'), # a sentence with spaces + ( + 'a sentence "with quotes"', + '"a sentence \\"with quotes\\""', + ), # a sentence in double quotes + (42, "42"), # int + (3.14, "3.14"), # float + ("127.0.0.1", '"127.0.0.1"'), # an ip address + ("user@email.test", '"user@email.test"'), # an email + (None, "null"), # NoneType + ], + ) + def test_logfmt_repr(self, obj, expected): + """ + Ensure logfmt_repr produces logfmt-compatible strings + """ + assert expected == logfmt_repr(obj) From 8bcb00e0643f10afb45b2dced8d91db3744d9d7e Mon Sep 17 00:00:00 2001 From: charlie Date: Mon, 28 Jun 2021 15:02:11 +0200 Subject: [PATCH 4/5] Add changelog and documentation --- CHANGELOG.rst | 2 ++ docs/api.rst | 13 +++++++++++++ 2 files changed, 15 insertions(+) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index fa6b7c34..f394f068 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -32,6 +32,8 @@ Changes: - If the `better-exceptions `_ package is present, ``structlog.dev.ConsoleRenderer`` will now pretty-print exceptions using it. Pass ``pretty_exceptions=False`` to disable. This only works if ``format_exc_info`` is **absent** in the processor chain. +- ``KeyValueRenderer`` now accepts an optional ``repr_formatter`` parameter to handle log representation. +- Added logfmt_repr to make KeyValueRenderer output logfmt compatible. ---- diff --git a/docs/api.rst b/docs/api.rst index b1307da5..2feabb41 100644 --- a/docs/api.rst +++ b/docs/api.rst @@ -226,6 +226,19 @@ API Reference >>> TimeStamper(fmt="%Y", key="year")(None, None, {}) # doctest: +SKIP {'year': '2013'} +.. autofunction:: logfmt_repr + + .. doctest:: + + >>> from structlog.processors import logfmt_repr + >>> logfmt_repr("string with withespace") + '"string with whitespace"' + >>> logfmt_repr("string") + 'string' + >>> logfmt_repr(42) + '42' + >>> logfmt_repr(None) + 'null' `structlog.stdlib` Module ------------------------- From 0e9ffd22c707f1df7704ee1ccfd0456e4e0fd289 Mon Sep 17 00:00:00 2001 From: charlie Date: Mon, 12 Jul 2021 13:42:13 +0200 Subject: [PATCH 5/5] Checks only for whitespace for quoting --- src/structlog/processors.py | 2 +- tests/test_processors.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/structlog/processors.py b/src/structlog/processors.py index 35f7ee37..eb3ffe1c 100644 --- a/src/structlog/processors.py +++ b/src/structlog/processors.py @@ -52,7 +52,7 @@ def logfmt_repr(inst: Any) -> str: if inst is None: return "null" if isinstance(inst, str): - return json.dumps(inst) if re.search("\\W", inst) else inst + return json.dumps(inst) if re.search("\\s", inst) else inst return repr(inst) diff --git a/tests/test_processors.py b/tests/test_processors.py index e02623c7..2da8496a 100644 --- a/tests/test_processors.py +++ b/tests/test_processors.py @@ -576,8 +576,8 @@ class TestLogFmtRepr: ), # a sentence in double quotes (42, "42"), # int (3.14, "3.14"), # float - ("127.0.0.1", '"127.0.0.1"'), # an ip address - ("user@email.test", '"user@email.test"'), # an email + ("127.0.0.1", "127.0.0.1"), # an ip address + ("user@email.test", "user@email.test"), # an email (None, "null"), # NoneType ], )