diff --git a/magic.py b/magic.py index 07f7096a..af4fe3a1 100644 --- a/magic.py +++ b/magic.py @@ -18,12 +18,18 @@ import sys import glob +import locale import ctypes import ctypes.util import threading from ctypes import c_char_p, c_int, c_size_t, c_void_p, byref, POINTER +try: + import chardet +except ImportError: + chardet = None + class MagicException(Exception): def __init__(self, message): @@ -214,8 +220,20 @@ def errorcheck_negative_one(result, func, args): def maybe_decode(s): if str == bytes: return s - else: - return s.decode('utf-8') + try: + return s.decode('utf-8') # try 'utf-8' + except UnicodeDecodeError: + pass + try: + return s.decode(locale.getpreferredencoding()) # try system default encoding + except UnicodeDecodeError: + pass + if chardet is not None: + # try chardet, if available + encoding = chardet.detect(s)['encoding'] + if encoding is not None: + return s.decode(encoding) + raise # give up def coerce_filename(filename): diff --git a/test/test.py b/test/test.py index 0f3aac2e..0a4254f4 100755 --- a/test/test.py +++ b/test/test.py @@ -59,6 +59,7 @@ def test_mime_types(self): 'test.pdf': 'application/pdf', 'test.gz': ('application/gzip', 'application/x-gzip'), 'text.txt': 'text/plain', + 'msword.doc': 'application/msword', b'\xce\xbb'.decode('utf-8'): 'text/plain', b'\xce\xbb': 'text/plain', }) diff --git a/test/testdata/msword.doc b/test/testdata/msword.doc new file mode 100644 index 00000000..7bed3c29 Binary files /dev/null and b/test/testdata/msword.doc differ