Handling encodings correctly

tests/test_cmdline.py

@@ -1,3 +1,4 @@
+# coding: utf-8
 import sys
 import subprocess
 import virtualenv
@@ -22,6 +23,18 @@ def test_commandline_explicit_interp(tmpdir):
         str(tmpdir.join('venv'))
     ])
 
+def test_commandline_non_ascii_path(tmpdir):
+    target_path = str(tmpdir.join('vénv'))
+    if sys.version_info[0] == 2:
+        target_path = target_path.decode('utf-8').encode(sys.getfilesystemencoding())
+
+    subprocess.check_call([
+        sys.executable,
+        VIRTUALENV_SCRIPT,
+        '-p', sys.executable,
+        target_path
+    ])
+
 # The registry lookups to support the abbreviated "-p 3.5" form of specifying
 # a Python interpreter on Windows don't seem to work with Python 3.5. The
 # registry layout is not well documented, and it's not clear that the feature

virtualenv.py

@@ -356,11 +356,21 @@ def copyfile(src, dest, symlink=True):
         logger.info('Copying to %s', dest)
         copyfileordir(src, dest, symlink)
 
-def writefile(dest, content, overwrite=True):
+def get_system_encoding():
+    if is_win:
+        # .encode('mbcs') is broken on Windows. For example,
+        # u'\u00a3'.encode('mbcs') gives a wrong result in CP437 (en-US), and
+        # u'\uffe1'.encode('mbcs') raises and error in CP950 (zh-TW)
+        import ctypes
+        return 'cp%d' % ctypes.windll.kernel32.GetOEMCP()
+    else:
+        return sys.getfilesystemencoding()
+
+def writefile(dest, content, overwrite=True, use_system_locale=False):
     if not os.path.exists(dest):
         logger.info('Writing %s', dest)
         with open(dest, 'wb') as f:
-            f.write(content.encode('utf-8'))
+            f.write(content.encode(get_system_encoding() if use_system_locale else 'utf-8'))
         return
     else:
         with open(dest, 'rb') as f:
@@ -720,15 +730,15 @@ def call_subprocess(cmd, show_stdout=True,
                     remove_from_env=None, stdin=None):
     cmd_parts = []
     for part in cmd:
-        if len(part) > 45:
-            part = part[:20]+"..."+part[-20:]
-        if ' ' in part or '\n' in part or '"' in part or "'" in part:
-            part = '"%s"' % part.replace('"', '\\"')
         if hasattr(part, 'decode'):
             try:
                 part = part.decode(sys.getdefaultencoding())
             except UnicodeDecodeError:
                 part = part.decode(sys.getfilesystemencoding())
+        if len(part) > 45:
+            part = part[:20]+"..."+part[-20:]
+        if ' ' in part or '\n' in part or '"' in part or "'" in part:
+            part = '"%s"' % part.replace('"', '\\"')
         cmd_parts.append(part)
     cmd_desc = ' '.join(cmd_parts)
     if show_stdout:
@@ -1285,6 +1295,15 @@ def install_python(home_dir, lib_dir, inc_dir, bin_dir, site_packages, clear, sy
                     raise SystemExit(3)
                 logger.info('Copying lib_pypy')
                 copyfile(d, os.path.join(home_dir, 'lib_pypy'), symlink)
+            else:
+                # On Unix-like systems, if PyPy is built with --shared, it
+                # requires libpypy-c.(so|dylib) to run. See
+                # _pypy_install_libs_after_virtualenv() in
+                # lib-python/(2.7|3)/subprocess.py of PyPy
+                for name in ['libpypy-c.so', 'libpypy-c.dylib']:
+                    src = join(prefix, 'bin', name)
+                    if os.path.exists(src):
+                        copyfile(src, join(bin_dir, name), symlink)
 
     if os.path.splitext(os.path.basename(py_executable))[0] != expected_exe:
         secondary_exe = os.path.join(os.path.dirname(py_executable),
@@ -1371,8 +1390,9 @@ def install_python(home_dir, lib_dir, inc_dir, bin_dir, site_packages, clear, sy
             else:
                 copyfile(py_executable, full_pth, symlink)
 
-    cmd = [py_executable, '-c', 'import sys;out=sys.stdout;'
-        'getattr(out, "buffer", out).write(sys.prefix.encode("utf-8"))']
+    cmd = [py_executable, '-c', 'import sys;out=sys.stdout;prefix=sys.prefix;'
+        'prefix=prefix.decode(sys.getfilesystemencoding()) if sys.version_info[0]==2 else prefix;'
+        'getattr(out, "buffer", out).write(prefix.encode("utf-8"))']
     logger.info('Testing executable with %s %s "%s"' % tuple(cmd))
     try:
         proc = subprocess.Popen(cmd,
@@ -1388,9 +1408,11 @@ def install_python(home_dir, lib_dir, inc_dir, bin_dir, site_packages, clear, sy
 
     proc_stdout = proc_stdout.strip().decode("utf-8")
     proc_stdout = os.path.normcase(os.path.abspath(proc_stdout))
-    norm_home_dir = os.path.normcase(os.path.abspath(home_dir))
-    if hasattr(norm_home_dir, 'decode'):
-        norm_home_dir = norm_home_dir.decode(sys.getfilesystemencoding())
+    if hasattr(home_dir, 'decode'):
+        unicode_home_dir = home_dir.decode(sys.getfilesystemencoding())
+    else:
+        unicode_home_dir = home_dir
+    norm_home_dir = os.path.normcase(os.path.abspath(unicode_home_dir))
     if proc_stdout != norm_home_dir:
         logger.fatal(
             'ERROR: The executable %s is not functioning' % py_executable)
@@ -1441,6 +1463,8 @@ def install_activate(home_dir, bin_dir, prompt=None):
         # Run-time conditional enables (basic) Cygwin compatibility
         home_dir_sh = ("""$(if [ "$OSTYPE" "==" "cygwin" ]; then cygpath -u '%s'; else echo '%s'; fi;)""" %
                        (home_dir, home_dir_msys))
+        if majver == 2:
+            home_dir_sh = home_dir_sh.decode(sys.getfilesystemencoding())
         files['activate'] = ACTIVATE_SH.replace('__VIRTUAL_ENV__', home_dir_sh)
 
     else:
@@ -1467,7 +1491,7 @@ def install_files(home_dir, bin_dir, prompt, files):
         content = content.replace('__VIRTUAL_ENV__', home_dir)
         content = content.replace('__VIRTUAL_NAME__', vname)
         content = content.replace('__BIN_NAME__', os.path.basename(bin_dir))
-        writefile(os.path.join(bin_dir, name), content)
+        writefile(os.path.join(bin_dir, name), content, use_system_locale=(name.endswith('.bat')))
 
 def install_python_config(home_dir, bin_dir, prompt=None):
     if sys.platform == 'win32' or is_jython and os._name == 'nt':