From f929b8df0956574cff269b9799e64d6193374f57 Mon Sep 17 00:00:00 2001
From: Dan Bungert <daniel.bungert@canonical.com>
Date: Tue, 28 Mar 2023 16:17:11 -0600
Subject: [PATCH 1/2] utils: run() handle unicode decode failure

Expand the run() wrapper to handle logging and unicode decode failures
in general.

LP: #2011344
---
 probert/tests/test_utils.py | 23 ++++++++++++++++++++++-
 probert/utils.py            | 24 ++++++++++++++++--------
 2 files changed, 38 insertions(+), 9 deletions(-)

diff --git a/probert/tests/test_utils.py b/probert/tests/test_utils.py
index 5b3fc44..e71d34c 100644
--- a/probert/tests/test_utils.py
+++ b/probert/tests/test_utils.py
@@ -5,7 +5,7 @@
 import tempfile
 import testtools
 import textwrap
-# import unittest
+import unittest
 
 from probert import utils
 from probert.tests.helpers import random_string, simple_mocked_open
@@ -180,3 +180,24 @@ def test_run_failure(self):
             )]
             self.assertIsNone(actual)
             self.assertEqual(expected, m_logs.output)
+
+    def test_run_nonunicode_out(self):
+        with self.assertLogs('probert.utils', level=logging.DEBUG) as m_logs:
+            with unittest.mock.patch('subprocess.run') as m_run:
+                m_run.return_value = unittest.mock.Mock()
+                m_run.return_value.returncode = 0
+                m_run.return_value.stdout = b'r\xe9serv\xe9e'
+                m_run.return_value.stderr = b''
+                actual = utils.run(['cmd'])
+        expected = [self.leader + line for line in (
+            'Command `cmd` exited with result: 0',
+            'UnicodeDecodeError on stdout: '
+            "'utf-8' codec can't decode byte 0xe9 in position 1: "
+            'invalid continuation byte',
+            'stdout: ------------------------------------------',
+            "b'r\\xe9serv\\xe9e'",
+            '<empty stderr>',
+            '--------------------------------------------------',
+        )]
+        self.assertIsNone(actual)
+        self.assertEqual(expected, m_logs.output)
diff --git a/probert/utils.py b/probert/utils.py
index 8b8605f..a6f4c4a 100644
--- a/probert/utils.py
+++ b/probert/utils.py
@@ -42,29 +42,37 @@ def _clean_env(env):
     return env
 
 
-def _log_stream(stream, name):
+def _decode_stream(stream, name):
     if stream:
+        try:
+            log_input = text = stream.decode('utf-8')
+        except UnicodeDecodeError as ude:
+            log.debug(f'UnicodeDecodeError on {name}: {ude}')
+            text = None
+            log_input = stream
         log.debug(f'{name}: ------------------------------------------')
-        for line in stream.splitlines():
+        for line in log_input.splitlines():
             log.debug(line)
+        return text
     else:
         log.debug(f'<empty {name}>')
+        return ''
 
 
 def run(cmdarr, env=None, **kw):
-    """Run the given, with stdout, stderr, and return code always logged.
+    """Run the given command with stdout, stderr, return code always logged.
     Returns the stdout on command success, or None on command failure."""
     env = _clean_env(env)
-    sp = subprocess.run(cmdarr, text=True, env=env,
+    sp = subprocess.run(cmdarr, text=False, env=env,
                         stdout=PIPE, stderr=PIPE, **kw)
     display_cmd = shlex.join(cmdarr)
     rc = sp.returncode
     log.debug(f'Command `{display_cmd}` exited with result: {rc}')
-    _log_stream(sp.stdout, 'stdout')
-    _log_stream(sp.stderr, 'stderr')
+    stdout = _decode_stream(sp.stdout, 'stdout')
+    _decode_stream(sp.stderr, 'stderr')
     log.debug('--------------------------------------------------')
-    if sp.returncode == 0:
-        return sp.stdout
+    if stdout is not None and sp.returncode == 0:
+        return stdout
     return None
 
 

From 59d770146ccfb15d9cf8acb84c46fa139c0a1924 Mon Sep 17 00:00:00 2001
From: Dan Bungert <daniel.bungert@canonical.com>
Date: Wed, 29 Mar 2023 16:29:38 -0600
Subject: [PATCH 2/2] utils: use backslashreplace for errors

---
 probert/tests/test_utils.py | 2 +-
 probert/utils.py            | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/probert/tests/test_utils.py b/probert/tests/test_utils.py
index e71d34c..c51b94b 100644
--- a/probert/tests/test_utils.py
+++ b/probert/tests/test_utils.py
@@ -195,7 +195,7 @@ def test_run_nonunicode_out(self):
             "'utf-8' codec can't decode byte 0xe9 in position 1: "
             'invalid continuation byte',
             'stdout: ------------------------------------------',
-            "b'r\\xe9serv\\xe9e'",
+            'r\\xe9serv\\xe9e',
             '<empty stderr>',
             '--------------------------------------------------',
         )]
diff --git a/probert/utils.py b/probert/utils.py
index a6f4c4a..fe288b2 100644
--- a/probert/utils.py
+++ b/probert/utils.py
@@ -49,7 +49,7 @@ def _decode_stream(stream, name):
         except UnicodeDecodeError as ude:
             log.debug(f'UnicodeDecodeError on {name}: {ude}')
             text = None
-            log_input = stream
+            log_input = stream.decode('utf-8', 'backslashreplace')
         log.debug(f'{name}: ------------------------------------------')
         for line in log_input.splitlines():
             log.debug(line)