[commit: ghc] master: Testsuite: write "\n" instead of "\r\n" when using mingw Python (6f6f515)

[git at git.haskell.org]

Repository : ssh://git@git.haskell.org/ghc

On branch  : master
Link       : http://ghc.haskell.org/trac/ghc/changeset/6f6f515401a29d26eaa5daae308b8e700abd4c04/ghc

>---------------------------------------------------------------

commit 6f6f515401a29d26eaa5daae308b8e700abd4c04
Author: Thomas Miedema <thomasmiedema at gmail.com>
Date:   Fri Jun 17 15:23:34 2016 +0200

    Testsuite: write "\n" instead of "\r\n" when using mingw Python
    
    Mingw style Python uses '\r\n' by default for newlines. This is
    annoying, because it means that when a GHC developer on Windows uses
    mingw Python to `make accept` a test, every single line of the
    .stderr file is touched. This makes it difficult to spot the real
    changes, and it leads to unnecessary git history bloat.
    
    Prevent this from happening by using io.open instead of open.
    See `Note [Universal newlines]`
    
    Reviewed by: Phyx
    
    Differential Revision: https://phabricator.haskell.org/D2342


>---------------------------------------------------------------

6f6f515401a29d26eaa5daae308b8e700abd4c04
 libraries/base/tests/IO/readwrite003.hs |  1 +
 testsuite/driver/runtests.py            |  2 +-
 testsuite/driver/testlib.py             | 43 ++++++++++++++++++++++++---------
 3 files changed, 34 insertions(+), 12 deletions(-)

diff --git a/libraries/base/tests/IO/readwrite003.hs b/libraries/base/tests/IO/readwrite003.hs
index d7ee78d..c8995e3 100644
--- a/libraries/base/tests/IO/readwrite003.hs
+++ b/libraries/base/tests/IO/readwrite003.hs
@@ -9,4 +9,5 @@ main = do
   hPutStrLn h "yz"
   hClose h
   h <- openBinaryFile file ReadMode
+  hSetNewlineMode stdout noNewlineTranslation
   hGetContents h >>= putStr
diff --git a/testsuite/driver/runtests.py b/testsuite/driver/runtests.py
index 33b432f..917003b 100644
--- a/testsuite/driver/runtests.py
+++ b/testsuite/driver/runtests.py
@@ -162,7 +162,7 @@ if windows:
 # Try to use UTF8
 if windows:
     import ctypes
-    # Windows Python provides windll, mingw python provides cdll.
+    # Windows and mingw* Python provide windll, msys2 python provides cdll.
     if hasattr(ctypes, 'windll'):
         mydll = ctypes.windll
     else:
diff --git a/testsuite/driver/testlib.py b/testsuite/driver/testlib.py
index 32b6951..ced16d1 100644
--- a/testsuite/driver/testlib.py
+++ b/testsuite/driver/testlib.py
@@ -5,6 +5,7 @@
 
 from __future__ import print_function
 
+import io
 import shutil
 import sys
 import os
@@ -1466,16 +1467,15 @@ def interpreter_run( name, way, extra_hc_opts, compile_only, top_mod ):
 
 
 def split_file(in_fn, delimiter, out1_fn, out2_fn):
-    infile = open(in_fn)
-    out1 = open(out1_fn, 'w')
-    out2 = open(out2_fn, 'w')
+    # See Note [Universal newlines].
+    infile = io.open(in_fn, 'r', encoding='utf8', newline=None)
+    out1 = io.open(out1_fn, 'w', encoding='utf8', newline='')
+    out2 = io.open(out2_fn, 'w', encoding='utf8', newline='')
 
     line = infile.readline()
-    line = re.sub('\r', '', line) # ignore Windows EOL
     while (re.sub('^\s*','',line) != delimiter and line != ''):
         out1.write(line)
         line = infile.readline()
-        line = re.sub('\r', '', line)
     out1.close()
 
     line = infile.readline()
@@ -1538,20 +1538,41 @@ def dump_stderr( name ):
 def read_no_crs(file):
     str = ''
     try:
-        h = open(file)
+        # See Note [Universal newlines].
+        h = io.open(file, 'r', encoding='utf8', newline=None)
         str = h.read()
         h.close
     except:
         # On Windows, if the program fails very early, it seems the
         # files stdout/stderr are redirected to may not get created
         pass
-    return re.sub('\r', '', str)
+    return str
 
 def write_file(file, str):
-    h = open(file, 'w')
+    # See Note [Universal newlines].
+    h = io.open(file, 'w', encoding='utf8', newline='')
     h.write(str)
     h.close
 
+# Note [Universal newlines]
+#
+# We don't want to write any Windows style line endings ever, because
+# it would mean that `make accept` would touch every line of the file
+# when switching between Linux and Windows.
+#
+# Furthermore, when reading a file, it is convenient to translate all
+# Windows style endings to '\n', as it simplifies searching or massaging
+# the content.
+#
+# Solution: use `io.open` instead of `open`
+#  * when reading: use newline=None to translate '\r\n' to '\n'
+#  * when writing: use newline='' to not translate '\n' to '\r\n'
+#
+# See https://docs.python.org/2/library/io.html#io.open.
+#
+# This should work with both python2 and python3, and with both mingw*
+# as msys2 style Python.
+
 def check_hp_ok(name):
     opts = getTestOpts()
 
@@ -1681,7 +1702,7 @@ def compare_outputs(way, kind, normaliser, expected_file, actual_file,
 
 def normalise_whitespace( str ):
     # Merge contiguous whitespace characters into a single space.
-    return ' '.join(w for w in str.split())
+    return u' '.join(w for w in str.split())
 
 callSite_re = re.compile(r', called at (.+):[\d]+:[\d]+ in [\w\-\.]+:')
 
@@ -1825,7 +1846,7 @@ def normalise_asm( str ):
           out.append(instr[0] + ' ' + instr[1])
         else:
           out.append(instr[0])
-    out = '\n'.join(out)
+    out = u'\n'.join(out)
     return out
 
 def if_verbose( n, s ):
@@ -2115,7 +2136,7 @@ def printFrameworkFailureSummary(file, testInfos):
     file.write('\n')
 
 def modify_lines(s, f):
-    s = '\n'.join([f(l) for l in s.splitlines()])
+    s = u'\n'.join([f(l) for l in s.splitlines()])
     if s and s[-1] != '\n':
         # Prevent '\ No newline at end of file' warnings when diffing.
         s += '\n'