Fixed a number of encoding issues (in run context)

.travis.yml

@@ -35,6 +35,7 @@ script:
   # Run unit and integration tests, generating coverage report (skipping html)
   # This will fail if the tests fail.
   - "inv coverage --no-html"
+  - "LC_ALL=C inv coverage --no-html"
   # Websites build OK?
   - inv sites
   # Did we break setup.py?

integration/main.py

@@ -1,10 +1,13 @@
+from __future__ import unicode_literals
+
 import os
 import sys
 
 from spec import Spec, trap, eq_, skip, ok_
 
 from invoke import run
 from invoke._version import __version__
+from invoke.exceptions import Failure
 from invoke.platform import WINDOWS
 
 
@@ -84,18 +87,56 @@ def basic_nonstandard_characters(self):
                 cmd = "type tree.out"
             else:
                 cmd = "cat tree.out"
-            run(cmd, hide='both')
+            run(cmd)
 
         def nonprinting_bytes(self):
             # Seriously non-printing characters (i.e. non UTF8) also don't
             # asplode
-            run("echo '\xff'", hide='both')
+            try:
+                run(b"echo '\xff'")
+            except TypeError:
+                if sys.version_info > (3, 0) and sys.version_info < (3, 3):
+                    # Python 3.2 is known to raise TypeError here
+                    pass
+                else:
+                    raise
 
         def nonprinting_bytes_pty(self):
             if WINDOWS:
                 return
             # PTY use adds another utf-8 decode spot which can also fail.
-            run("echo '\xff'", pty=True, hide='both')
+            run(b"echo '\xff'", pty=True)
+
+        def nonprinting_bytes_in_unicode(self):
+            # Seriously non-printing characters (i.e. non UTF8) should fail as
+            # we cannot just automatically ignore/replace encoding issues in
+            # user commands
+            try:
+                run("echo '\xff'")
+            except (TypeError, UnicodeEncodeError):
+                if sys.stdout.encoding == 'UTF-8':
+                    raise
+            else:
+                if sys.stdout.encoding != 'UTF-8':
+                    raise Exception(
+                        "Non-printing bytes in unicode command must fail"
+                    )
+
+        def nonprinting_bytes_in_unicode_pty(self):
+            if WINDOWS:
+                return
+            # PTY use adds another utf-8 decode spot which should also fail
+            # when user command has non-encodable symbols.
+            try:
+                run("echo '\xff'", pty=True)
+            except Failure:
+                if sys.stdout.encoding == 'UTF-8':
+                    raise
+            else:
+                if sys.stdout.encoding != 'UTF-8':
+                    raise Exception(
+                        "Non-printing bytes in unicode command must fail"
+                    )
 
     def pty_puts_both_streams_in_stdout(self):
         if WINDOWS:

invoke/exceptions.py

@@ -6,6 +6,8 @@
 condition in a way easily told apart from other, truly unexpected errors".
 """
 
+from __future__ import unicode_literals
+
 from collections import namedtuple
 from traceback import format_exception
 from pprint import pformat
@@ -19,6 +21,7 @@ def __init__(self, name, start):
         self.start = start
 
 
+@six.python_2_unicode_compatible
 class Failure(Exception):
     """
     Exception subclass representing failure of a command execution.
@@ -35,15 +38,12 @@ def __str__(self):
         if self.result.pty:
             err_label = "Stdout (pty=True; no stderr possible)"
             err_text = self.result.stdout
-        return """Command execution failure!
-
-Exit code: {0}
-
-{1}:
-
-{2}
-
-""".format(self.result.exited, err_label, err_text)
+        return (
+            "Command execution failure!\n"
+            "Exit code: {0}\n"
+            "{1}:\n"
+            "{2}"
+        ).format(self.result.exited, err_label, err_text)
 
     def __repr__(self):
         return str(self)
@@ -133,6 +133,7 @@ def _printable_kwargs(kwargs):
         printable[key] = item
     return printable
 
+@six.python_2_unicode_compatible
 class ThreadException(Exception):
     """
     One or more exceptions were raised within background (usually I/O) threads.
@@ -175,9 +176,7 @@ def __str__(self):
             ", ".join(x.type.__name__ for x in self.exceptions),
             "\n\n".join(details),
         )
-        return """
-Saw {0} exceptions within threads ({1}):
-
-
-{2}
-""".format(*args)
+        return (
+            "Saw {0} exceptions within threads ({1}):\n"
+            "{2}"
+        ).format(*args)

invoke/runners.py

@@ -1,4 +1,5 @@
 # -*- coding: utf-8 -*-
+from __future__ import unicode_literals
 
 import codecs
 import locale
@@ -254,7 +255,18 @@ def run(self, command, **kwargs):
         env = self.generate_env(opts['env'], opts['replace_env'])
         # Echo running command
         if opts['echo']:
-            print("\033[1;37m{0}\033[0m".format(command))
+            if isinstance(command, six.text_type):
+                unicode_command = command
+            else:
+                if six.PY2:
+                    unicode_command = "b{0}".format(repr(command))
+                else:
+                    unicode_command = repr(command)
+            # Use a wrapped sys.stdout to avoid encoding errors (errors will
+            # be replaced with backslashed unicode codes)
+            self._wrap_output(sys.stdout).write(
+                "\033[1;37m{0}\033[0m\n".format(unicode_command)
+            )
         # Start executing the actual command (runs in background)
         self.start(command, shell, env)
         # Arrive at final encoding if neither config nor kwargs had one
@@ -372,10 +384,10 @@ def _run_opts(self, kwargs):
         # Derive stream objects
         out_stream = opts['out_stream']
         if out_stream is None:
-            out_stream = sys.stdout
+            out_stream = self._wrap_output(sys.stdout)
         err_stream = opts['err_stream']
         if err_stream is None:
-            err_stream = sys.stderr
+            err_stream = self._wrap_output(sys.stderr)
         in_stream = opts['in_stream']
         if in_stream is None:
             in_stream = sys.stdin
@@ -635,6 +647,27 @@ def generate_env(self, env, replace_env):
         """
         return env if replace_env else dict(os.environ, **env)
 
+    def _wrap_output(self, stream):
+        """
+        Wraps a ``stream`` with ``codecs.StreamWriter`` where encoding errors
+        are replaced.
+        """
+        if hasattr(stream, 'buffer'):
+            output_buffer = stream.buffer
+        else:
+            output_buffer = stream
+
+        # NOTE: sys.stdout.encoding returns None on Python 2.x when
+        # C locale is used
+        stream_encoding = getattr(stream, 'encoding', None)
+        if stream_encoding is None:
+            stream_encoding = self.default_encoding()
+
+        return codecs.getwriter(stream_encoding)(
+            output_buffer,
+            errors='backslashreplace'
+        )
+
     def should_use_pty(self, pty, fallback):
         """
         Should execution attempt to use a pseudo-terminal?
@@ -786,22 +819,33 @@ def start(self, command, shell, env):
             # shell, just as subprocess does; this replaces our process - whose
             # pipes are all hooked up to the PTY - with the "real" one.
             if self.pid == 0:
-                # TODO: both pty.spawn() and pexpect.spawn() do a lot of
-                # setup/teardown involving tty.setraw, getrlimit, signal.
-                # Ostensibly we'll want some of that eventually, but if
-                # possible write tests - integration-level if necessary -
-                # before adding it!
-                #
-                # Set pty window size based on what our own controlling
-                # terminal's window size appears to be.
-                # TODO: make subroutine?
-                winsize = struct.pack('HHHH', rows, cols, 0, 0)
-                fcntl.ioctl(sys.stdout.fileno(), termios.TIOCSWINSZ, winsize)
-                # Use execve for bare-minimum "exec w/ variable # args + env"
-                # behavior. No need for the 'p' (use PATH to find executable)
-                # for now.
-                # TODO: see if subprocess is using equivalent of execvp...
-                os.execve(shell, [shell, '-c', command], env)
+                try:
+                    # TODO: both pty.spawn() and pexpect.spawn() do a lot of
+                    # setup/teardown involving tty.setraw, getrlimit, signal.
+                    # Ostensibly we'll want some of that eventually, but if
+                    # possible write tests - integration-level if necessary -
+                    # before adding it!
+                    #
+                    # Set pty window size based on what our own controlling
+                    # terminal's window size appears to be.
+                    # TODO: make subroutine?
+                    winsize = struct.pack(b'HHHH', rows, cols, 0, 0)
+                    fcntl.ioctl(
+                        sys.stdout.fileno(), termios.TIOCSWINSZ, winsize
+                    )
+                    # Use execve for bare-minimum
+                    # "exec w/ variable args + env" behavior. No need for the
+                    # 'p' (use PATH to find executable) # for now.
+                    # TODO: see if subprocess is using equivalent of execvp...
+                    os.execve(shell, [shell, '-c', command], env)
+                except Exception:
+                    # Prevent process hanging when something wrong happened
+                    # here, for example, encoding error in `execve`
+                    import traceback
+                    traceback.print_exc()
+                    # We forcefully terminate this fork branch to simulate
+                    # no-op `execve`
+                    os._exit(1)
         else:
             self.process = Popen(
                 command,
@@ -814,6 +858,20 @@ def start(self, command, shell, env):
             )
 
     def default_encoding(self):
+        # Based on some experiments there is an issue with
+        # `locale.getpreferredencoding(do_setlocale=False)` in Python 2.x on
+        # Linux and OS X, and `locale.getpreferredencoding(do_setlocale=True)`
+        # triggers some global state changes:
+        # https://github.com/pyinvoke/invoke/pull/274/
+        if six.PY2 and not WINDOWS:
+            # This is a workaround, please, report if there is a case when it
+            # doesn't work.
+            default_locale_encoding = locale.getdefaultlocale()[1]
+            # It is known that when default locale is C (POSIX), Python
+            # returns None.
+            if default_locale_encoding is not None:
+                return default_locale_encoding
+        # This is a preferred way of getting a default encoding.
         return locale.getpreferredencoding(False)
 
     def wait(self):
@@ -854,6 +912,7 @@ def returncode(self):
             return self.process.returncode
 
 
+@six.python_2_unicode_compatible
 class Result(object):
     """
     A container for information about the result of a command execution.
@@ -903,9 +962,10 @@ def __str__(self):
         ret = ["Command exited with status {0}.".format(self.exited)]
         for x in ('stdout', 'stderr'):
             val = getattr(self, x)
-            ret.append("""=== {0} ===
-{1}
-""".format(x, val.rstrip()) if val else "(no {0})".format(x))
+            if val:
+                ret.append("=== {0} ===\n{1}".format(x, val.rstrip()))
+            else:
+                ret.append("(no {0})".format(x))
         return "\n".join(ret)
 
     @property