[commit: base] encoding: Dump localeEncoding_ignore in favour of fileSystemEncoding (43ce077)
Max Bolingbroke
batterseapower at hotmail.com
Tue Apr 5 10:05:15 CEST 2011
Repository : ssh://darcs.haskell.org//srv/darcs/packages/base
On branch : encoding
http://hackage.haskell.org/trac/ghc/changeset/43ce077ccdd6c3c22d3d43119c1e9cd8ef3002bb
>---------------------------------------------------------------
commit 43ce077ccdd6c3c22d3d43119c1e9cd8ef3002bb
Author: Max Bolingbroke <batterseapower at hotmail.com>
Date: Mon Apr 4 15:24:04 2011 +0100
Dump localeEncoding_ignore in favour of fileSystemEncoding
I also changed the code so that mkTextEncoding always tries the
native-Haskell decoders in preference to those from iconv, even on
non-Windows. The motivation here is simply that it is better for
compatibility if we do this, and those are the ones you get for
the utf* and latin1* predefined TextEncodings anyway.
>---------------------------------------------------------------
GHC/IO/Encoding.hs | 64 +++++++++++++++++++++++--------------------
GHC/IO/Encoding/Failure.hs | 2 +-
GHC/IO/Encoding/Iconv.hs | 8 +++---
3 files changed, 39 insertions(+), 35 deletions(-)
diff --git a/GHC/IO/Encoding.hs b/GHC/IO/Encoding.hs
index d42934f..8d31e6e 100644
--- a/GHC/IO/Encoding.hs
+++ b/GHC/IO/Encoding.hs
@@ -21,19 +21,20 @@ module GHC.IO.Encoding (
utf8, utf8_bom,
utf16, utf16le, utf16be,
utf32, utf32le, utf32be,
- localeEncoding, localeEncoding_ignore,
+ localeEncoding, fileSystemEncoding,
mkTextEncoding,
) where
import GHC.Base
--import GHC.IO
+import GHC.IO.Exception
import GHC.IO.Buffer
+import GHC.IO.Encoding.Failure
import GHC.IO.Encoding.Types
import GHC.Word
#if !defined(mingw32_HOST_OS)
import qualified GHC.IO.Encoding.Iconv as Iconv
#else
-import GHC.IO.Encoding.Failure
import qualified GHC.IO.Encoding.CodePage as CodePage
import Text.Read (reads)
#endif
@@ -42,10 +43,8 @@ import qualified GHC.IO.Encoding.UTF8 as UTF8
import qualified GHC.IO.Encoding.UTF16 as UTF16
import qualified GHC.IO.Encoding.UTF32 as UTF32
-#if defined(mingw32_HOST_OS)
+import Data.List
import Data.Maybe
-import GHC.IO.Exception
-#endif
-- -----------------------------------------------------------------------------
@@ -100,15 +99,16 @@ utf32be = UTF32.utf32be
-- | The Unicode encoding of the current locale
localeEncoding :: TextEncoding
--- | The Unicode encoding of the current locale, ignoring coding errors
-localeEncoding_ignore :: TextEncoding
+-- | The Unicode encoding of the current locale, but allowing arbitrary
+-- undecodable bytes to be round-tripped through it
+fileSystemEncoding :: TextEncoding
#if !defined(mingw32_HOST_OS)
localeEncoding = Iconv.localeEncoding
-localeEncoding_ignore = Iconv.localeEncoding_ignore
+fileSystemEncoding = Iconv.localeEncodingFailingWith SurrogateEscapeFailure
#else
localeEncoding = CodePage.localeEncoding
-localeEncoding_ignore = CodePage.localeEncoding_ignore
+fileSystemEncoding = CodePage.localeEncodingFailingWith SurrogateEscapeFailure
#endif
-- | Look up the named Unicode encoding. May fail with
@@ -138,31 +138,35 @@ localeEncoding_ignore = CodePage.localeEncoding_ignore
-- @CP@; for example, @\"CP1250\"@.
--
mkTextEncoding :: String -> IO TextEncoding
-#if !defined(mingw32_HOST_OS)
-mkTextEncoding = Iconv.mkTextEncoding
+mkTextEncoding e = case mb_coding_failure_mode of
+ Nothing -> unknown_encoding
+ Just cfm -> case enc of
+ "UTF-8" -> return $ UTF8.utf8FailingWith cfm
+ "UTF-16" -> return $ UTF16.utf16FailingWith cfm
+ "UTF-16LE" -> return $ UTF16.utf16leFailingWith cfm
+ "UTF-16BE" -> return $ UTF16.utf16beFailingWith cfm
+ "UTF-32" -> return $ UTF32.utf32FailingWith cfm
+ "UTF-32LE" -> return $ UTF32.utf32leFailingWith cfm
+ "UTF-32BE" -> return $ UTF32.utf32beFailingWith cfm
+#if defined(mingw32_HOST_OS)
+ 'C':'P':n | [(cp,"")] <- reads n -> return $ CodePage.codePageEncodingFailingWith cp cfm
+ _ -> unknown_encoding
#else
-mkTextEncoding e = case (mb_coding_failure_mode, e) of
- (Just cfm, "UTF-8" -> return $ UTF8.utf8FailingWith cfm
- (Just cfm, "UTF-16" -> return $ UTF16.utf16FailingWith cfm
- (Just cfm, "UTF-16LE" -> return $ UTF16.utf16leFailingWith cfm
- (Just cfm, "UTF-16BE" -> return $ UTF16.utf16beFailingWith cfm
- (Just cfm, "UTF-32" -> return $ UTF32.utf32FailingWith cfm
- (Just cfm, "UTF-32LE" -> return $ UTF32.utf32leFailingWith cfm
- (Just cfm, "UTF-32BE" -> return $ UTF32.utf32beFailingWith cfm
- (Just cfm, 'C':'P':n)
- | [(cp,"")] <- reads n = return $ CodePage.codePageEncodingFailingWith cp cfm
- _ -> ioException (IOError Nothing NoSuchThing "mkTextEncoding"
- ("unknown encoding:" ++ e) Nothing Nothing)
+ _ -> Iconv.mkTextEncoding e
+#endif
where
- -- The only problem with actually documenting //IGNORE as a supported suffix
- -- is that it's not necessarily supported with non-GNU iconv
+ -- The only problem with actually documenting //IGNORE and //TRANSLIT as
+ -- supported suffixes is that they are not necessarily supported with non-GNU iconv
(enc, suffix) = span (/= '/') e
mb_coding_failure_mode = case suffix of
- "" -> Just ErrorOnCodingFailure
- "//IGNORE" -> Just IgnoreCodingFailure
- "//TRANSLIT" -> Just TransliterateCodingFailure
- _ -> Nothing
-#endif
+ "" -> Just ErrorOnCodingFailure
+ "//IGNORE" -> Just IgnoreCodingFailure
+ "//TRANSLIT" -> Just TransliterateCodingFailure
+ "//SURROGATE" -> Just SurrogateEscapeFailure
+ _ -> Nothing
+
+ unknown_encoding = ioException (IOError Nothing NoSuchThing "mkTextEncoding"
+ ("unknown encoding:" ++ e) Nothing Nothing)
latin1_encode :: CharBuffer -> Buffer Word8 -> IO (CharBuffer, Buffer Word8)
latin1_encode = Latin1.latin1_encode -- unchecked, used for binary
diff --git a/GHC/IO/Encoding/Failure.hs b/GHC/IO/Encoding/Failure.hs
index eed07ce..0e28cb7 100644
--- a/GHC/IO/Encoding/Failure.hs
+++ b/GHC/IO/Encoding/Failure.hs
@@ -39,7 +39,7 @@ codingFailureModeSuffix :: CodingFailureMode -> String
codingFailureModeSuffix ErrorOnCodingFailure = ""
codingFailureModeSuffix IgnoreCodingFailure = "//IGNORE"
codingFailureModeSuffix TransliterateCodingFailure = "//TRANSLIT"
-codingFailureModeSuffix SurrogateEscapeFailure = "//SURROGATE" -- FIXME: parse
+codingFailureModeSuffix SurrogateEscapeFailure = "//SURROGATE"
-- | In transliterate mode, we use this character when decoding unknown bytes.
--
diff --git a/GHC/IO/Encoding/Iconv.hs b/GHC/IO/Encoding/Iconv.hs
index 124f459..48b7b10 100644
--- a/GHC/IO/Encoding/Iconv.hs
+++ b/GHC/IO/Encoding/Iconv.hs
@@ -27,7 +27,7 @@ module GHC.IO.Encoding.Iconv (
utf8,
utf16, utf16le, utf16be,
utf32, utf32le, utf32be,
- localeEncoding, localeEncoding_ignore
+ localeEncoding, localeEncodingFailingWith
#endif
) where
@@ -111,9 +111,9 @@ localeEncodingName = unsafePerformIO $ do
localeEncoding :: TextEncoding
localeEncoding = unsafePerformIO $ mkTextEncoding localeEncodingName
-{-# NOINLINE localeEncoding_ignore #-}
-localeEncoding_ignore :: TextEncoding
-localeEncoding_ignore = unsafePerformIO $ mkTextEncodingFailingWith IgnoreCodingFailure localeEncodingName
+{-# NOINLINE localeEncodingFailingWith #-}
+localeEncodingFailingWith :: CodingFailureMode -> TextEncoding
+localeEncodingFailingWith cfm = unsafePerformIO $ mkTextEncodingFailingWith cfm localeEncodingName
-- We hope iconv_t is a storable type. It should be, since it has at least the
-- value -1, which is a possible return value from iconv_open.
More information about the Cvs-libraries
mailing list