From b93d4efb8cb5a7ef01f29274f5c0e17a2e2edb1d Mon Sep 17 00:00:00 2001 From: "J. F. Zhang" Date: Thu, 16 May 2019 17:33:35 +0800 Subject: [PATCH 1/5] Handle "UnicodeDecodeError" on some msoffice files --- magic.py | 23 +++++++++++++++++++++-- 1 file changed, 21 insertions(+), 2 deletions(-) diff --git a/magic.py b/magic.py index 07f7096a..306fed7d 100644 --- a/magic.py +++ b/magic.py @@ -18,12 +18,18 @@ import sys import glob +import locale import ctypes import ctypes.util import threading from ctypes import c_char_p, c_int, c_size_t, c_void_p, byref, POINTER +try: + import chardet +except ImportError: + chardet = None + class MagicException(Exception): def __init__(self, message): @@ -214,8 +220,21 @@ def errorcheck_negative_one(result, func, args): def maybe_decode(s): if str == bytes: return s - else: - return s.decode('utf-8') + try: + return s.decode('utf-8') # try 'utf-8' + except UnicodeDecodeError: + pass + try: + return s.decode(locale.getpreferredencoding()) # try system default encoding + except UnicodeDecodeError: + pass + if chardet is not None: + # try chardet, if available + encoding = chardet.detect(s)['encoding'] + if encoding is not None: + print(encoding) + return s.decode(encoding) + raise # give up def coerce_filename(filename): From da4a8ac197a95b3c7461c13f10cbf7bb3f0e0fd2 Mon Sep 17 00:00:00 2001 From: "J. F. Zhang" Date: Thu, 16 May 2019 17:49:51 +0800 Subject: [PATCH 2/5] Remove print log --- magic.py | 1 - 1 file changed, 1 deletion(-) diff --git a/magic.py b/magic.py index 306fed7d..af4fe3a1 100644 --- a/magic.py +++ b/magic.py @@ -232,7 +232,6 @@ def maybe_decode(s): # try chardet, if available encoding = chardet.detect(s)['encoding'] if encoding is not None: - print(encoding) return s.decode(encoding) raise # give up From 084532c286e59b8907a0feb9d6e4bd1cd11cf0ec Mon Sep 17 00:00:00 2001 From: Zhang Jianfei Date: Thu, 16 May 2019 18:18:05 +0800 Subject: [PATCH 3/5] Add test case --- test/testdata/msword.doc | Bin 0 -> 10752 bytes 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 test/testdata/msword.doc diff --git a/test/testdata/msword.doc b/test/testdata/msword.doc new file mode 100644 index 0000000000000000000000000000000000000000..7bed3c295da794493235ee1a1b3bc0a9b9638fe7 GIT binary patch literal 10752 zcmeI22~btn8OOhS-{O0MY$Ae7xZ(>+RC2+%OMD1X76lY{Kt-O2ilBmG61AAR6`hn! zVro0KO;X#K#!0D(#xBOuu_%95*W*7#8 zI|De|0#jyP_m9XE9vWG8ZA$aFrso96n8I%y=bu zYEmLg5ms!*^T9tyo;KsoML5$HLwCY%Fi%ottUQHx8dk7Q=D(Jqk&-0I&F>haUG8%a zU}HB8%m9U;3@iZUU@=$eCix%9;aIe+f6k58VEqxZS;->%|p zm31*it*o9l4`<6jSDPCz7-8+(pD$GZY>%sIYwU^B4B z`BGq4p9%CF3-lZb^c)EE>FrU!ehxRNnY+KlHawT8H5^Rs)llc? zU0>&jgZ4l_+91-kA>S6=utQDGQzfZ>DBICW;_aO!0#&msp|CxFGhuV)c@gG=^W&{) zmEAY)4LF{D<9XjGL$=wHww(R~<6G%&rtoE0tE87h`HHh-%i^UyEe9mV42-fPL)s+tCiPY^%ElEx~@7yJ6lmrDa8+RS{;ZdPsuz zHI&lBpN08Ukw#fTXO6^r6^T%I`(xdh@vXZ?qz`Uji#>|XqhNW7_c;|&^01QN;?;>I zCvZ`5HJ3P2G~^ejjAZ6b!u-0e5sAX=UF;E?hs#w!^6RNN)o`bWYpCX1K~6NPHI5nn zELjQR7dJMl9^I>|PMgE-P;K@N&xnTLS~CJ1MpNfc!r@|(dDPI*RIoWGTc#pKW+ZA6eEV9YqvQoBaf&QM`T1tqRY-s3 zx%cJ{klJT?r5({!1G^lPH%$Bi9dJMjJoL2oHzjMKG_i}~+88a~?aF6Y%gnymwrkHX zUjFW~4UFjToQ`nyH+{Tt$EGf96tpE6bU6JnMV8pa%Qvdbka+1O2}qeJ!)1hwk|J3p zJLFwCr2a|mqD5&jT2C!r8yu(Ts&3OAx?2y@JL#c%xZXwYsw;+SbTHgTun}T(GrAkG zMvCDz3XLLTrg6%=;%7PcOZrNIJd3(VYdtiV-cb+ML-a5`QjgMIsNV>q!1%;C`?YIT z=csecmu;e~iO}zP%e*Mzq;b_6xf_(1$FPJ(BNStdA>VNu`#(Bjg1N+T@@Xkbtzudb zt%yRKvD}uPhH%cN{4&o1OU9NuB(5Y(;yXu*9^{d#iHY)~q9IZ^Emc^eughXt6tY@g zOR1Oo#2?G2eO{Mw;V0x$?=un;bXiUhctkz1WT(1&rp@uY7>6Ub&mouHyUy(l`XFdW ze*e&{q-EiIh93`49s8ZmuPpyOD!j`tyZ$5Rg_vQZRIMm$dhb6kkMg`VVwWd>=A49x z+22q2XyE>S+h-*wx|9E=$ENfgbfswK;6=XNA?JHt9r@UR51)K~_~6us(y%+tL52m>l70V#st9S0}Di>1})-jjK)*N zU~*$etXaa`=!J^f_zVKW#MQ_FHzQ#5L&ere%g6q`v9FoP-Dpki@HUGo9nEyK7_J43 z{}^;cQ@FVwY-_RhUtGPcwj zUU=_FZt*F#DW7ZTcqfivt%doXTKCh7S8~h3-h9C&{rbFpg1z(#UT{k0`+C5XTLHh* zoq(UyUceqY47gIB1Z?*|0ejws#5jwI1|1CDADTB`JYUfs(xJKTkAvn&OoC>A6+-hj zRxYoQ+O5zmzX65DLbJ@k1&U$*sQ&jXyXAuA4-bqJ=w{t=uplorm*jFRDn9tC;k$Ky ztkYoN1+w>#%HDSlDRkGqmvZQ0&=Kyf%z9CCp`+%M#QH{o@z@yfjt)(v3uByKfw6+zw%Z(!W^x_IL4a+ zi)P%+!~TO5j2Kh}e7 zNETfN;W+@exQTElnmP$Hz@Qd&V%m0jFnhqV1KJVohPF$)<$Oh(<~+r3l=jB?fb$gR zDB3b@fcD9`iFOnTIH&U)?h3dZak=8(19S&mu6lw;0BuMEy+9o34Lsmc5D)qQ+HV5j zD$@^eju-$E!Q(&&Ngx>v1pE#L0nR-`z)--yO&Jb70*nMH;7P#m%L`J$Xpjcd!Bb!i z7z;8$CddNWU>q0^azHM~0~5eRkPjw-$zTfL5rp6VbT9)H0Dgx>U?!LaW`kld2b2JQ zpL0Pem&`Zq} zNYqA&g2h;Qt4P!O?}%=eO)ZT8V$I~~s6`3p93BW+kG4uTQ5CCv>b9n{{%%Fbs1;)b z_$$}lNC?WeAtAq=ww6B(ar5{wLDEq=e=)mT>3A3Ir#)EZO@$Rz!TOh(zufWgeP=c5 zfmK04W)9HS_Af--rsgAEzB%eV^wah-G5V#j1^=F-8unjl+PhWX*1B&~`#EQ|wfu6# zJ Date: Thu, 16 May 2019 18:20:47 +0800 Subject: [PATCH 4/5] Add test case code --- test/test.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/test/test.py b/test/test.py index 0f3aac2e..0e1b8fcc 100755 --- a/test/test.py +++ b/test/test.py @@ -59,6 +59,7 @@ def test_mime_types(self): 'test.pdf': 'application/pdf', 'test.gz': ('application/gzip', 'application/x-gzip'), 'text.txt': 'text/plain', + 'msword.doc': 'application/msword', b'\xce\xbb'.decode('utf-8'): 'text/plain', b'\xce\xbb': 'text/plain', }) @@ -80,6 +81,14 @@ def test_descriptions(self): 'gzip compressed data, was "test", last modified' ': Sun Jun 29 01:32:52 2008, from Unix, original size 15'), 'text.txt': 'ASCII text', + 'msword.doc': 'Composite Document File V2 Document, ' + 'Little Endian, Os: Windows, Version 6.1, ' + 'Code page: 1200, Title: ש×, Author: fers, ' + 'Template: Normal, Last Saved By: fers, Revision Number: 6, ' + 'Create Time/Date: Fri Jun 22 10:28:00 2007, ' + 'Last Saved Time/Date: Thu May 16 14:22:23 2019, ' + 'Number of Pages: 1, Number of Words: 10, Number of Characters: 44, ' + 'Name of Creating Application: Microsoft O, Security: 0, Locale ID: 2052', }, buf_equals_file=False) finally: del os.environ['TZ'] From 55f45edc1957dc7d9242dd88960dc0967be0eee8 Mon Sep 17 00:00:00 2001 From: Zhang Jianfei Date: Thu, 16 May 2019 18:33:02 +0800 Subject: [PATCH 5/5] Delete test_descriptions of "msword.doc" because of the encoding SytaxError --- test/test.py | 8 -------- 1 file changed, 8 deletions(-) diff --git a/test/test.py b/test/test.py index 0e1b8fcc..0a4254f4 100755 --- a/test/test.py +++ b/test/test.py @@ -81,14 +81,6 @@ def test_descriptions(self): 'gzip compressed data, was "test", last modified' ': Sun Jun 29 01:32:52 2008, from Unix, original size 15'), 'text.txt': 'ASCII text', - 'msword.doc': 'Composite Document File V2 Document, ' - 'Little Endian, Os: Windows, Version 6.1, ' - 'Code page: 1200, Title: ש×, Author: fers, ' - 'Template: Normal, Last Saved By: fers, Revision Number: 6, ' - 'Create Time/Date: Fri Jun 22 10:28:00 2007, ' - 'Last Saved Time/Date: Thu May 16 14:22:23 2019, ' - 'Number of Pages: 1, Number of Words: 10, Number of Characters: 44, ' - 'Name of Creating Application: Microsoft O, Security: 0, Locale ID: 2052', }, buf_equals_file=False) finally: del os.environ['TZ']