From 1b7e07edfb2fcaaf4edcbe056e78707e2cc4f476 Mon Sep 17 00:00:00 2001 From: Adam Hupp Date: Fri, 16 Apr 2021 12:01:27 -0700 Subject: [PATCH 01/78] Update README.md --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 378594ba..352346c4 100644 --- a/README.md +++ b/README.md @@ -119,9 +119,9 @@ To run against a specific python version: LC_ALL=en_US.UTF-8 python3 test/test.py ``` -## libmagic and python-magic +## libmagic python API compatability -See [COMPAT.md](COMPAT.md) for a guide to libmagic / python-magic compatability. +The python bindings shipped with libmagic use a module name that conflicts with this package. To work around this, python-magic includes a compatability layer for the libmagic API. See [COMPAT.md](COMPAT.md) for a guide to libmagic / python-magic compatability. ## Versioning From 3a9029d2c8238b3a4bdc3648e5e641a6d1c51d62 Mon Sep 17 00:00:00 2001 From: The Gitter Badger Date: Sun, 25 Apr 2021 23:25:51 +0000 Subject: [PATCH 02/78] Add Gitter badge --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 352346c4..4c2a5072 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # python-magic [![PyPI version](https://badge.fury.io/py/python-magic.svg)](https://badge.fury.io/py/python-magic) -[![Build Status](https://travis-ci.org/ahupp/python-magic.svg?branch=master)](https://travis-ci.org/ahupp/python-magic) +[![Build Status](https://travis-ci.org/ahupp/python-magic.svg?branch=master)](https://travis-ci.org/ahupp/python-magic) [![Join the chat at https://gitter.im/ahupp/python-magic](https://badges.gitter.im/ahupp/python-magic.svg)](https://gitter.im/ahupp/python-magic?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge) python-magic is a Python interface to the libmagic file type identification library. libmagic identifies file types by checking From 3174a7854553fe1ffb5cc22534d8169eac3a378c Mon Sep 17 00:00:00 2001 From: EtiennePelletier Date: Mon, 10 May 2021 17:52:11 -0400 Subject: [PATCH 03/78] Fix typo (compatability --> compatibility) --- CHANGELOG | 2 +- COMPAT.md | 4 ++-- README.md | 4 ++-- magic/__init__.py | 2 +- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/CHANGELOG b/CHANGELOG index fbabad62..b5b6c83b 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -5,7 +5,7 @@ Changes to 0.4.21, 0.4.22 Changes to 0.4.20 -- merge in a compatability layer for the upstream libmagic python binding. +- merge in a compatibility layer for the upstream libmagic python binding. Since both this package and that one are called 'magic', this compat layer removes a very common source of runtime errors. Use of that libmagic API will produce a deprecation warning. diff --git a/COMPAT.md b/COMPAT.md index 21b35a17..921abafa 100644 --- a/COMPAT.md +++ b/COMPAT.md @@ -1,10 +1,10 @@ There are two python modules named 'magic' that do the same thing, but with incompatible APIs. One of these ships with libmagic, and (this one) is distributed through pypi. Both have been around for many years and have -substantial user bases. This incompatability is a major source of pain for +substantial user bases. This incompatibility is a major source of pain for users, and bug reports for me. -To mitigate this pain, python-magic has added a compatability layer to export +To mitigate this pain, python-magic has added a compatibility layer to export the libmagic python API parallel to the existing one. The mapping between the libmagic and python-magic functions is: diff --git a/README.md b/README.md index 4c2a5072..9eb70e8a 100644 --- a/README.md +++ b/README.md @@ -119,9 +119,9 @@ To run against a specific python version: LC_ALL=en_US.UTF-8 python3 test/test.py ``` -## libmagic python API compatability +## libmagic python API compatibility -The python bindings shipped with libmagic use a module name that conflicts with this package. To work around this, python-magic includes a compatability layer for the libmagic API. See [COMPAT.md](COMPAT.md) for a guide to libmagic / python-magic compatability. +The python bindings shipped with libmagic use a module name that conflicts with this package. To work around this, python-magic includes a compatibility layer for the libmagic API. See [COMPAT.md](COMPAT.md) for a guide to libmagic / python-magic compatibility. ## Versioning diff --git a/magic/__init__.py b/magic/__init__.py index 7a75b751..f10a38a9 100644 --- a/magic/__init__.py +++ b/magic/__init__.py @@ -420,7 +420,7 @@ def _add_compat(to_module): def deprecation_wrapper(fn): def _(*args, **kwargs): warnings.warn( - "Using compatability mode with libmagic's python binding. " + "Using compatibility mode with libmagic's python binding. " "See https://github.com/ahupp/python-magic/blob/master/COMPAT.md for details.", PendingDeprecationWarning) From 0ab68b2d4f6ad0efce45e56df8e94778fe7f758c Mon Sep 17 00:00:00 2001 From: Jochen Sprickerhof Date: Sun, 16 May 2021 10:36:42 +0200 Subject: [PATCH 04/78] Revert "Fix bug in Magic when destructor called too early" This reverts commit 16972c2c6fda3573860f5f8b0f2c03b757e71d3c. --- magic/__init__.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/magic/__init__.py b/magic/__init__.py index f10a38a9..17456c15 100644 --- a/magic/__init__.py +++ b/magic/__init__.py @@ -53,8 +53,6 @@ def __init__(self, mime=False, magic_file=None, mime_encoding=False, raw - Do not try to decode "non-printable" chars. extension - Print a slash-separated list of valid extensions for the file type found. """ - - self.cookie = None self.flags = MAGIC_NONE if mime: self.flags |= MAGIC_MIME_TYPE From 12d0ea062d92a15baee69c4be0e4ae00fcdee416 Mon Sep 17 00:00:00 2001 From: Jochen Sprickerhof Date: Sun, 16 May 2021 10:37:25 +0200 Subject: [PATCH 05/78] Check for cookie attribute before trying to delete it __del__ is called regardless of whether __init__ passed, so the cookie attribute may not exits. This can be seen with: python3 -c "import magic; magic.Magic(foo=None)" Also discussed in https://github.com/ahupp/python-magic/pull/222#issuecomment-675354824 --- magic/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/magic/__init__.py b/magic/__init__.py index 17456c15..363e88ff 100644 --- a/magic/__init__.py +++ b/magic/__init__.py @@ -150,7 +150,7 @@ def __del__(self): # incorrect fix for a threading problem, however I'm leaving # it in because it's harmless and I'm slightly afraid to # remove it. - if self.cookie and magic_close: + if hasattr(self, 'cookie') and self.cookie and magic_close: magic_close(self.cookie) self.cookie = None From 29450ec911b08dc2b128fa00cd95cf6ba27bccdc Mon Sep 17 00:00:00 2001 From: Adam Hupp Date: Fri, 21 May 2021 10:10:20 -0700 Subject: [PATCH 06/78] add py.typed sentinal per https://www.python.org/dev/peps/pep-0561/#packaging-type-information --- magic/py.typed | 0 setup.py | 1 + 2 files changed, 1 insertion(+) create mode 100644 magic/py.typed diff --git a/magic/py.typed b/magic/py.typed new file mode 100644 index 00000000..e69de29b diff --git a/setup.py b/setup.py index 3b5489af..9da84668 100644 --- a/setup.py +++ b/setup.py @@ -22,6 +22,7 @@ def read(file_name): long_description=read('README.md'), long_description_content_type='text/markdown', packages=['magic'], + package_data=['py.typed'], keywords="mime magic file", license="MIT", python_requires='>=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*', From 44d01df064f406895d02ee1ff144840597488687 Mon Sep 17 00:00:00 2001 From: Adam Hupp Date: Fri, 21 May 2021 10:26:02 -0700 Subject: [PATCH 07/78] actually try building package --- setup.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 9da84668..4ffc68f2 100644 --- a/setup.py +++ b/setup.py @@ -22,7 +22,9 @@ def read(file_name): long_description=read('README.md'), long_description_content_type='text/markdown', packages=['magic'], - package_data=['py.typed'], + package_data={ + 'magic': ['py.typed'], + }, keywords="mime magic file", license="MIT", python_requires='>=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*', From 816e4d704d455eaf5c1e06e1af1b75fdb8ef2f5a Mon Sep 17 00:00:00 2001 From: Adam Hupp Date: Tue, 1 Jun 2021 10:27:15 -0700 Subject: [PATCH 08/78] Cleanup library loading Simplify library loading so it's clearer what should happen for each platform. There is a very small chance this is a regression if you were somehow relying on windows DLL names on a non-windows platform. Also resolves https://github.com/ahupp/python-magic/pull/245 --- magic/loader.py | 75 ++++++++++++++++++++++++++----------------------- 1 file changed, 40 insertions(+), 35 deletions(-) diff --git a/magic/loader.py b/magic/loader.py index 6b2bfcb3..d6e179b7 100644 --- a/magic/loader.py +++ b/magic/loader.py @@ -1,40 +1,45 @@ +from ctypes.util import find_library import ctypes import sys import glob +import os.path + +def _lib_candidates(): + + yield find_library('magic') + + if sys.platform == 'darwin': + + paths = [ + '/opt/local/lib', + '/usr/local/lib', + '/opt/homebrew/lib', + ] + glob.glob('/usr/local/Cellar/libmagic/*/lib') + + for i in paths: + yield os.path.join(i, 'libmagic.dylib') + + elif sys.platform in ('win32', 'cygwin'): + + prefixes = ['libmagic', 'magic1', 'cygmagic-1', 'libmagic-1', 'msys-magic-1'] + + for i in prefixes: + # find_library searches in %PATH% but not the current directory, + # so look for both + yield './%s.dll' % (i,) + yield find_library(i) def load_lib(): - libmagic = None - # Let's try to find magic or magic1 - dll = ctypes.util.find_library('magic') \ - or ctypes.util.find_library('magic1') \ - or ctypes.util.find_library('cygmagic-1') \ - or ctypes.util.find_library('libmagic-1') \ - or ctypes.util.find_library('msys-magic-1') # for MSYS2 - - # necessary because find_library returns None if it doesn't find the library - if dll: - libmagic = ctypes.CDLL(dll) - - if not libmagic or not libmagic._name: - windows_dlls = ['magic1.dll', 'cygmagic-1.dll', 'libmagic-1.dll', 'msys-magic-1.dll'] - platform_to_lib = {'darwin': ['/opt/local/lib/libmagic.dylib', - '/usr/local/lib/libmagic.dylib'] + - # Assumes there will only be one version installed - glob.glob('/usr/local/Cellar/libmagic/*/lib/libmagic.dylib'), # flake8:noqa - 'win32': windows_dlls, - 'cygwin': windows_dlls, - 'linux': ['libmagic.so.1'], - # fallback for some Linuxes (e.g. Alpine) where library search does not work # flake8:noqa - } - platform = 'linux' if sys.platform.startswith('linux') else sys.platform - for dll in platform_to_lib.get(platform, []): - try: - libmagic = ctypes.CDLL(dll) - break - except OSError: - pass - - if not libmagic or not libmagic._name: - # It is better to raise an ImportError since we are importing magic module - raise ImportError('failed to find libmagic. Check your installation') - return libmagic \ No newline at end of file + + for lib in _lib_candidates(): + # find_library returns None when lib not found + if lib is None: + continue + try: + return ctypes.CDLL(lib) + except OSError: + pass + else: + # It is better to raise an ImportError since we are importing magic module + raise ImportError('failed to find libmagic. Check your installation') + From ae483dc845ec6edd35861f5c46e70cd4d229e4f2 Mon Sep 17 00:00:00 2001 From: Adam Hupp Date: Tue, 1 Jun 2021 11:52:42 -0700 Subject: [PATCH 09/78] Update tests and docker config for newer versions --- test/Dockerfile_archlinux | 2 +- test/Dockerfile_centos8 | 6 +++++- test/libmagic_test.py | 4 ++-- test/test.py | 3 ++- 4 files changed, 10 insertions(+), 5 deletions(-) diff --git a/test/Dockerfile_archlinux b/test/Dockerfile_archlinux index 677add4d..fc96c447 100755 --- a/test/Dockerfile_archlinux +++ b/test/Dockerfile_archlinux @@ -1,4 +1,4 @@ -FROM archlinux:20200505 +FROM archlinux:latest RUN yes | pacman -Syyu --overwrite '*' RUN yes | pacman -S python python2 file which COPY . /python-magic diff --git a/test/Dockerfile_centos8 b/test/Dockerfile_centos8 index 505221b8..968f6b65 100644 --- a/test/Dockerfile_centos8 +++ b/test/Dockerfile_centos8 @@ -1,5 +1,9 @@ FROM centos:8 RUN yum -y update -RUN yum -y install file-libs python3 python2 which +RUN yum -y install file-libs python3 python2 which glibc-locale-source +RUN yum reinstall glibc-common -y && \ + localedef -i en_US -f UTF-8 en_US.UTF-8 && \ + echo "LANG=en_US.UTF-8" > /etc/locale.conf +ENV LANG en_US.UTF-8 COPY . /python-magic CMD cd /python-magic/test && python3 ./run.py diff --git a/test/libmagic_test.py b/test/libmagic_test.py index 64b7ec4f..88f1254e 100644 --- a/test/libmagic_test.py +++ b/test/libmagic_test.py @@ -11,12 +11,12 @@ class MagicTestCase(unittest.TestCase): filename = 'testdata/test.pdf' expected_mime_type = 'application/pdf' expected_encoding = 'us-ascii' - expected_name = 'PDF document, version 1.2' + expected_name = ('PDF document, version 1.2', 'PDF document, version 1.2, 2 pages') def assert_result(self, result): self.assertEqual(result.mime_type, self.expected_mime_type) self.assertEqual(result.encoding, self.expected_encoding) - self.assertEqual(result.name, self.expected_name) + self.assertIn(result.name, self.expected_name) def test_detect_from_filename(self): result = magic.detect_from_filename(self.filename) diff --git a/test/test.py b/test/test.py index 949c77eb..0cd12fd2 100755 --- a/test/test.py +++ b/test/test.py @@ -107,7 +107,8 @@ def test_descriptions(self): try: self.assert_values(m, { 'magic._pyc_': 'python 2.4 byte-compiled', - 'test.pdf': 'PDF document, version 1.2', + 'test.pdf': ('PDF document, version 1.2', + 'PDF document, version 1.2, 2 pages'), 'test.gz': ('gzip compressed data, was "test", from Unix, last ' 'modified: Sun Jun 29 01:32:52 2008', From 5e1f1500a37377216605c81fc9a965a0036aaef6 Mon Sep 17 00:00:00 2001 From: Adam Hupp Date: Tue, 1 Jun 2021 11:58:37 -0700 Subject: [PATCH 10/78] add changelog for 0.4.23 --- CHANGELOG | 7 +++++++ setup.py | 2 +- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/CHANGELOG b/CHANGELOG index b5b6c83b..34686c45 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -1,3 +1,10 @@ +Changes to 0.4.23 + + - Include a `py.typed` sentinal to enable type checking + - Improve fix for attribute error during destruction + - Cleanup library loading logic + - Add new homebrew library dir for OSX + Changes to 0.4.21, 0.4.22 - Unify dll loader between the standard and compat library, fixing load diff --git a/setup.py b/setup.py index 4ffc68f2..bea5c7a4 100644 --- a/setup.py +++ b/setup.py @@ -18,7 +18,7 @@ def read(file_name): author='Adam Hupp', author_email='adam@hupp.org', url="http://github.com/ahupp/python-magic", - version='0.4.22', + version='0.4.23', long_description=read('README.md'), long_description_content_type='text/markdown', packages=['magic'], From 4ed6b038ba8acd59ed58c888fbd18355b8955eba Mon Sep 17 00:00:00 2001 From: Adam Hupp Date: Thu, 3 Jun 2021 06:43:08 -0700 Subject: [PATCH 11/78] fix alpine library loading --- magic/loader.py | 5 +++++ test/{Dockerfile_alpine => docker/alpine} | 2 +- test/{Dockerfile_archlinux => docker/archlinux} | 0 test/{Dockerfile_bionic => docker/bionic} | 0 test/{Dockerfile_centos7 => docker/centos7} | 0 test/{Dockerfile_centos8 => docker/centos8} | 0 test/{Dockerfile_focal => docker/focal} | 0 test/{Dockerfile_xenial => docker/xenial} | 0 test_docker.sh | 17 ++++------------- 9 files changed, 10 insertions(+), 14 deletions(-) rename test/{Dockerfile_alpine => docker/alpine} (79%) rename test/{Dockerfile_archlinux => docker/archlinux} (100%) rename test/{Dockerfile_bionic => docker/bionic} (100%) rename test/{Dockerfile_centos7 => docker/centos7} (100%) rename test/{Dockerfile_centos8 => docker/centos8} (100%) rename test/{Dockerfile_focal => docker/focal} (100%) rename test/{Dockerfile_xenial => docker/xenial} (100%) diff --git a/magic/loader.py b/magic/loader.py index d6e179b7..931f1619 100644 --- a/magic/loader.py +++ b/magic/loader.py @@ -29,6 +29,11 @@ def _lib_candidates(): yield './%s.dll' % (i,) yield find_library(i) + elif sys.platform == 'linux': + # This is necessary because alpine is bad + yield 'libmagic.so.1' + + def load_lib(): for lib in _lib_candidates(): diff --git a/test/Dockerfile_alpine b/test/docker/alpine similarity index 79% rename from test/Dockerfile_alpine rename to test/docker/alpine index f8ec79e8..eb511d79 100755 --- a/test/Dockerfile_alpine +++ b/test/docker/alpine @@ -1,4 +1,4 @@ -FROM alpine:latest +FROM python:3.8-alpine3.12 RUN apk add python3 python2 libmagic COPY . /python-magic CMD cd /python-magic/test && python3 ./run.py diff --git a/test/Dockerfile_archlinux b/test/docker/archlinux similarity index 100% rename from test/Dockerfile_archlinux rename to test/docker/archlinux diff --git a/test/Dockerfile_bionic b/test/docker/bionic similarity index 100% rename from test/Dockerfile_bionic rename to test/docker/bionic diff --git a/test/Dockerfile_centos7 b/test/docker/centos7 similarity index 100% rename from test/Dockerfile_centos7 rename to test/docker/centos7 diff --git a/test/Dockerfile_centos8 b/test/docker/centos8 similarity index 100% rename from test/Dockerfile_centos8 rename to test/docker/centos8 diff --git a/test/Dockerfile_focal b/test/docker/focal similarity index 100% rename from test/Dockerfile_focal rename to test/docker/focal diff --git a/test/Dockerfile_xenial b/test/docker/xenial similarity index 100% rename from test/Dockerfile_xenial rename to test/docker/xenial diff --git a/test_docker.sh b/test_docker.sh index 57d53285..59d6b7c3 100755 --- a/test_docker.sh +++ b/test_docker.sh @@ -5,17 +5,8 @@ set -e -function TestInContainer { - local name="$1" - local TAG="python_magic/${name}:latest" - docker build -t $TAG -f "test/Dockerfile_${name}" . - docker run "python_magic/${name}:latest" -} +NAME=`basename $1` +TAG="python_magic/${NAME}:latest" +docker build -t $TAG -f $1 . +docker run $TAG -TestInContainer "xenial" -TestInContainer "bionic" -TestInContainer "focal" -TestInContainer "centos7" -TestInContainer "centos8" -TestInContainer "archlinux" -TestInContainer "alpine" From 5fa9055bdc4159ebab0699ae0429de29ae171267 Mon Sep 17 00:00:00 2001 From: Adam Hupp Date: Thu, 3 Jun 2021 06:44:28 -0700 Subject: [PATCH 12/78] bump to 0.4.24 --- CHANGELOG | 3 +++ setup.py | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/CHANGELOG b/CHANGELOG index 34686c45..c578572d 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -1,3 +1,6 @@ +Changes to 0.4.24: + - Fix regression in library loading on some Alpine docker images. + Changes to 0.4.23 - Include a `py.typed` sentinal to enable type checking diff --git a/setup.py b/setup.py index bea5c7a4..295b8db7 100644 --- a/setup.py +++ b/setup.py @@ -18,7 +18,7 @@ def read(file_name): author='Adam Hupp', author_email='adam@hupp.org', url="http://github.com/ahupp/python-magic", - version='0.4.23', + version='0.4.24', long_description=read('README.md'), long_description_content_type='text/markdown', packages=['magic'], From 73bcc7482bf3e6d1f6a74a91bf9689b289a88910 Mon Sep 17 00:00:00 2001 From: Adam Hupp Date: Thu, 2 Sep 2021 15:19:16 -0700 Subject: [PATCH 13/78] Fix compat mode handling with empty mime string I can't repro this, but PR #250 suggests that some versions of libmagic will return a mimetype that doesn't include a charset, leading to an exception. Fall back to an empty charset in this case. --- .gitignore | 1 + magic/compat.py | 7 ++++++- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 0346a859..111565e3 100644 --- a/.gitignore +++ b/.gitignore @@ -10,3 +10,4 @@ pip-selfcheck.json pyvenv.cfg *.pyc *~ +dist/ diff --git a/magic/compat.py b/magic/compat.py index e2d71ee4..07fad45a 100644 --- a/magic/compat.py +++ b/magic/compat.py @@ -245,7 +245,12 @@ def open(flags): def _create_filemagic(mime_detected, type_detected): - mime_type, mime_encoding = mime_detected.split('; ') + splat = mime_detected.split('; ') + mime_type = splat[0] + if len(splat) == 2: + mime_encoding = splat[1] + else: + mime_encoding = '' return FileMagic(name=type_detected, mime_type=mime_type, encoding=mime_encoding.replace('charset=', '')) From 7f7542fcbc192fef6e4939f4eb748e941a720b2c Mon Sep 17 00:00:00 2001 From: Adam Hupp Date: Mon, 4 Oct 2021 14:46:37 -0700 Subject: [PATCH 14/78] Support os.PathLike types See https://github.com/ahupp/python-magic/pull/251 --- CHANGELOG | 3 +++ magic/__init__.py | 17 +++++++++++++++++ magic/__init__.pyi | 5 +++-- test/test.py | 8 ++++++++ test_docker.sh | 14 ++++++++++---- 5 files changed, 41 insertions(+), 6 deletions(-) diff --git a/CHANGELOG b/CHANGELOG index c578572d..26e01f48 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -1,3 +1,6 @@ +Changes to 0.4.25: + - Support os.PathLike values in Magic.from_file and magic.from_file + Changes to 0.4.24: - Fix regression in library loading on some Alpine docker images. diff --git a/magic/__init__.py b/magic/__init__.py index 363e88ff..bab7c7b1 100644 --- a/magic/__init__.py +++ b/magic/__init__.py @@ -100,6 +100,7 @@ def from_buffer(self, buf): # if we're on python3, convert buf to bytes # otherwise this string is passed as wchar* # which is not what libmagic expects + # NEXTBREAK: only take bytes if type(buf) == str and str != bytes: buf = buf.encode('utf-8', errors='replace') return maybe_decode(magic_buffer(self.cookie, buf)) @@ -229,6 +230,7 @@ def errorcheck_negative_one(result, func, args): # return str on python3. Don't want to unconditionally # decode because that results in unicode on python2 def maybe_decode(s): + # NEXTBREAK: remove if str == bytes: return s else: @@ -237,13 +239,28 @@ def maybe_decode(s): return s.decode('utf-8', 'backslashreplace') +try: + from os import PathLike + def unpath(filename): + if isinstance(filename, PathLike): + return filename.__fspath__() + else: + return filename +except ImportError: + def unpath(filename): + return filename + def coerce_filename(filename): if filename is None: return None + + filename = unpath(filename) + # ctypes will implicitly convert unicode strings to bytes with # .encode('ascii'). If you use the filesystem encoding # then you'll get inconsistent behavior (crashes) depending on the user's # LANG environment variable + # NEXTBREAK: remove is_unicode = (sys.version_info[0] <= 2 and isinstance(filename, unicode)) or \ (sys.version_info[0] >= 3 and diff --git a/magic/__init__.pyi b/magic/__init__.pyi index 8d5f38f5..b6b5489c 100644 --- a/magic/__init__.pyi +++ b/magic/__init__.pyi @@ -1,6 +1,7 @@ import ctypes.util import threading from typing import Any, Text, Optional, Union +from os import PathLike class MagicException(Exception): message: Any = ... @@ -12,13 +13,13 @@ class Magic: lock: threading.Lock = ... def __init__(self, mime: bool = ..., magic_file: Optional[Any] = ..., mime_encoding: bool = ..., keep_going: bool = ..., uncompress: bool = ..., raw: bool = ...) -> None: ... def from_buffer(self, buf: Union[bytes, str]) -> Text: ... - def from_file(self, filename: Union[bytes, str]) -> Text: ... + def from_file(self, filename: Union[bytes, str, PathLike]) -> Text: ... def from_descriptor(self, fd: int, mime: bool = ...) -> Text: ... def setparam(self, param: Any, val: Any): ... def getparam(self, param: Any): ... def __del__(self) -> None: ... -def from_file(filename: Union[bytes, str], mime: bool = ...) -> Text: ... +def from_file(filename: Union[bytes, str, PathLike], mime: bool = ...) -> Text: ... def from_buffer(buffer: Union[bytes, str], mime: bool = ...) -> Text: ... def from_descriptor(fd: int, mime: bool = ...) -> Text: ... diff --git a/test/test.py b/test/test.py index 0cd12fd2..0c4621c5 100755 --- a/test/test.py +++ b/test/test.py @@ -219,6 +219,14 @@ def test_name_count(self): with open(os.path.join(self.TESTDATA_DIR, 'name_use.jpg'), 'rb') as f: m.from_buffer(f.read()) + def test_pathlike(self): + if sys.version_info < (3, 6): + return + from pathlib import Path + path = Path(self.TESTDATA_DIR, "test.pdf") + m = magic.Magic(mime=True) + self.assertEqual('application/pdf', m.from_file(path)) + if __name__ == '__main__': unittest.main() diff --git a/test_docker.sh b/test_docker.sh index 59d6b7c3..ad2bc5d5 100755 --- a/test_docker.sh +++ b/test_docker.sh @@ -5,8 +5,14 @@ set -e -NAME=`basename $1` -TAG="python_magic/${NAME}:latest" -docker build -t $TAG -f $1 . -docker run $TAG +DEFAULT_TARGETS="xenial bionic focal centos7 centos8 archlinux alpine" +TARGETS=${1:-${DEFAULT_TARGETS}} + +HERE=`dirname $0` + +for i in $TARGETS; do + TAG="python_magic/${i}:latest" + docker build -t $TAG -f ${HERE}/test/docker/$i . + docker run $TAG +done From 0ae7e7ceac0e80e03adc75c858bb378c0427331a Mon Sep 17 00:00:00 2001 From: Martin Liska Date: Fri, 22 Oct 2021 09:47:32 +0200 Subject: [PATCH 15/78] Support file 5.41. In https://github.com/file/file/commit/7d9b0f0d853957ad88dae0f440fecd58d2740ca7, the MIME was changed for Python bytecode. --- test/test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/test.py b/test/test.py index 0c4621c5..e443b844 100755 --- a/test/test.py +++ b/test/test.py @@ -90,7 +90,7 @@ def test_mime_types(self): try: m = magic.Magic(mime=True) self.assert_values(m, { - 'magic._pyc_': ('application/octet-stream', 'text/x-bytecode.python'), + 'magic._pyc_': ('application/octet-stream', 'text/x-bytecode.python', 'application/x-bytecode.python'), 'test.pdf': 'application/pdf', 'test.gz': ('application/gzip', 'application/x-gzip'), 'test.snappy.parquet': 'application/octet-stream', From 261eed864ba8e7b5b105cecbf2da2f56cbe31543 Mon Sep 17 00:00:00 2001 From: Vivien Maisonneuve Date: Thu, 30 Dec 2021 10:46:02 +0100 Subject: [PATCH 16/78] Include type stubs in package --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 295b8db7..36819b62 100644 --- a/setup.py +++ b/setup.py @@ -23,7 +23,7 @@ def read(file_name): long_description_content_type='text/markdown', packages=['magic'], package_data={ - 'magic': ['py.typed'], + 'magic': ['py.typed', '__init__.pyi'], }, keywords="mime magic file", license="MIT", From 0fb1922da4a7b27bd19b75a03dca2f51bff4362f Mon Sep 17 00:00:00 2001 From: Adam Hupp Date: Tue, 1 Feb 2022 16:17:18 -0800 Subject: [PATCH 17/78] include typing stubs in package, bump to 0.4.25 --- CHANGELOG | 3 +++ setup.py | 4 ++-- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/CHANGELOG b/CHANGELOG index 26e01f48..89e63143 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -1,5 +1,8 @@ Changes to 0.4.25: - Support os.PathLike values in Magic.from_file and magic.from_file + - Handle some versions of libmagic that return mime string without charset + - Fix tests for file 5.41 + - Include typing stub in package Changes to 0.4.24: - Fix regression in library loading on some Alpine docker images. diff --git a/setup.py b/setup.py index 36819b62..8d27ce2c 100644 --- a/setup.py +++ b/setup.py @@ -18,12 +18,12 @@ def read(file_name): author='Adam Hupp', author_email='adam@hupp.org', url="http://github.com/ahupp/python-magic", - version='0.4.24', + version='0.4.25', long_description=read('README.md'), long_description_content_type='text/markdown', packages=['magic'], package_data={ - 'magic': ['py.typed', '__init__.pyi'], + 'magic': ['py.typed', '*.pyi', '**/*.pyi'], }, keywords="mime magic file", license="MIT", From 6b34bde052be74334dad71963d92a1c49eecd168 Mon Sep 17 00:00:00 2001 From: Adam Hupp Date: Mon, 25 Apr 2022 06:47:40 -0700 Subject: [PATCH 18/78] improve test script name clarity; expand test docs; cleanup docker run script --- test/README | 12 ++++-------- test/run_all_docker_test.sh | 13 +++++++++++++ test/{run.py => run_all_versions.py} | 0 test_docker.sh | 18 ------------------ 4 files changed, 17 insertions(+), 26 deletions(-) create mode 100755 test/run_all_docker_test.sh rename test/{run.py => run_all_versions.py} (100%) delete mode 100755 test_docker.sh diff --git a/test/README b/test/README index 12d4e4fc..c34cb6ac 100644 --- a/test/README +++ b/test/README @@ -1,10 +1,6 @@ -To run the tests across a selection of Ubuntu LTS versions: +There are a few ways to run the python-magic tests -docker build -t "python_magic/xenial:latest" -f test/Dockerfile_xenial . -docker build -t "python_magic/bionic:latest" -f test/Dockerfile_bionic . -docker build -t "python_magic/focal:latest" -f test/Dockerfile_focal . - -docker run python_magic/xenial:latest -docker run python_magic/bionic:latest -docker run python_magic/focal:latest + 1. `pytest` will run the test suite against your default version of python + 2. `./test/run_all_versions.py` will run the tests against all installed versions of python. + 3. `./test/run_all_docker_test.sh` will run against a variety of different Linux distributions, using docker. diff --git a/test/run_all_docker_test.sh b/test/run_all_docker_test.sh new file mode 100755 index 00000000..68a28081 --- /dev/null +++ b/test/run_all_docker_test.sh @@ -0,0 +1,13 @@ +#!/bin/sh + +set -e +set -x + +ROOT=$(dirname $0)/.. +cd $ROOT + +for f in test/docker/*; do + H=$(docker build -q -f ${f} .) + docker run --rm $H +done + diff --git a/test/run.py b/test/run_all_versions.py similarity index 100% rename from test/run.py rename to test/run_all_versions.py diff --git a/test_docker.sh b/test_docker.sh deleted file mode 100755 index ad2bc5d5..00000000 --- a/test_docker.sh +++ /dev/null @@ -1,18 +0,0 @@ -#!/bin/bash - -# Test with various versions of ubuntu. This more or less re-creates the -# Travis CI test environment - -set -e - -DEFAULT_TARGETS="xenial bionic focal centos7 centos8 archlinux alpine" - -TARGETS=${1:-${DEFAULT_TARGETS}} - -HERE=`dirname $0` - -for i in $TARGETS; do - TAG="python_magic/${i}:latest" - docker build -t $TAG -f ${HERE}/test/docker/$i . - docker run $TAG -done From 7e760728889b3d3928310a453ebe3383adb37984 Mon Sep 17 00:00:00 2001 From: Adam Hupp Date: Mon, 25 Apr 2022 06:52:45 -0700 Subject: [PATCH 19/78] correctly find path to testdata when running from root --- test/libmagic_test.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/test/libmagic_test.py b/test/libmagic_test.py index 88f1254e..5719a58e 100644 --- a/test/libmagic_test.py +++ b/test/libmagic_test.py @@ -3,12 +3,16 @@ import unittest import os import magic +import os.path # magic_descriptor is broken (?) in centos 7, so don't run those tests SKIP_FROM_DESCRIPTOR = bool(os.environ.get('SKIP_FROM_DESCRIPTOR')) +TESTDATA_DIR = os.path.abspath(os.path.join(os.path.dirname(__file__), 'testdata')) + + class MagicTestCase(unittest.TestCase): - filename = 'testdata/test.pdf' + filename = os.path.join(TESTDATA_DIR, 'test.pdf') expected_mime_type = 'application/pdf' expected_encoding = 'us-ascii' expected_name = ('PDF document, version 1.2', 'PDF document, version 1.2, 2 pages') From de97f00b128b04de65af2c480c411f4e6d0527b3 Mon Sep 17 00:00:00 2001 From: Adam Hupp Date: Mon, 25 Apr 2022 06:53:12 -0700 Subject: [PATCH 20/78] less surprising use of abspath --- test/test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/test.py b/test/test.py index e443b844..624a443c 100755 --- a/test/test.py +++ b/test/test.py @@ -19,7 +19,7 @@ SKIP_FROM_DESCRIPTOR = bool(os.environ.get('SKIP_FROM_DESCRIPTOR')) class MagicTest(unittest.TestCase): - TESTDATA_DIR = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'testdata') + TESTDATA_DIR = os.path.abspath(os.path.join(os.path.dirname(__file__), 'testdata')) def test_version(self): try: From 585373b5d952e579a9f7bd8f6418ca0bfd6a5e35 Mon Sep 17 00:00:00 2001 From: Adam Hupp Date: Mon, 25 Apr 2022 06:55:06 -0700 Subject: [PATCH 21/78] rename test to match standard naming rules, so pytest finds it: https://docs.pytest.org/en/6.2.x/goodpractices.html#test-discovery --- test/{test.py => python_magic_test.py} | 0 test/run_all_versions.py | 2 +- 2 files changed, 1 insertion(+), 1 deletion(-) rename test/{test.py => python_magic_test.py} (100%) diff --git a/test/test.py b/test/python_magic_test.py similarity index 100% rename from test/test.py rename to test/python_magic_test.py diff --git a/test/run_all_versions.py b/test/run_all_versions.py index cf62eee7..d9e7ec58 100644 --- a/test/run_all_versions.py +++ b/test/run_all_versions.py @@ -24,7 +24,7 @@ def run_test(versions): continue found = True print("Testing %s" % i) - subprocess.run([i, os.path.join(this_dir, "test.py")], env=new_env, check=True) + subprocess.run([i, os.path.join(this_dir, "python_magic_test.py")], env=new_env, check=True) subprocess.run([i, os.path.join(this_dir, "libmagic_test.py")], env=new_env, check=True) if not found: From 51e34d2b8b84f1a582b4dbbfa3693957ddc3f48a Mon Sep 17 00:00:00 2001 From: Adam Hupp Date: Mon, 25 Apr 2022 15:21:35 -0700 Subject: [PATCH 22/78] use tox for all the multi-version testing --- .dockerignore | 1 + .gitignore | 2 +- __init__.py | 0 test/docker/alpine | 7 +++++-- test/docker/bionic | 11 ++++++----- test/docker/centos7 | 8 ++++++-- test/docker/centos8 | 8 +++++--- test/docker/focal | 11 ++++++----- test/docker/xenial | 11 ++++++----- test/run_all_versions.py | 35 ----------------------------------- tox.ini | 7 ++++--- 11 files changed, 40 insertions(+), 61 deletions(-) create mode 120000 .dockerignore delete mode 100644 __init__.py delete mode 100644 test/run_all_versions.py diff --git a/.dockerignore b/.dockerignore new file mode 120000 index 00000000..3e4e48b0 --- /dev/null +++ b/.dockerignore @@ -0,0 +1 @@ +.gitignore \ No newline at end of file diff --git a/.gitignore b/.gitignore index 111565e3..40c8c4eb 100644 --- a/.gitignore +++ b/.gitignore @@ -4,7 +4,7 @@ bin/ deb_dist htmlcov/ lib/ -__pycache__/ +**/__pycache__ python_magic.egg-info pip-selfcheck.json pyvenv.cfg diff --git a/__init__.py b/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/test/docker/alpine b/test/docker/alpine index eb511d79..c36b7201 100755 --- a/test/docker/alpine +++ b/test/docker/alpine @@ -1,4 +1,7 @@ FROM python:3.8-alpine3.12 RUN apk add python3 python2 libmagic -COPY . /python-magic -CMD cd /python-magic/test && python3 ./run.py +WORKDIR /python-magic +COPY . . +RUN python3 -m pip install tox +CMD python3 -m tox + diff --git a/test/docker/bionic b/test/docker/bionic index e335b8ee..2fe482c6 100755 --- a/test/docker/bionic +++ b/test/docker/bionic @@ -1,8 +1,9 @@ FROM ubuntu:bionic +WORKDIR /python-magic +COPY . . RUN apt-get update -RUN apt-get -y install python -RUN apt-get -y install python3 -RUN apt-get -y install locales +RUN apt-get -y install python python3 locales python3-pip libmagic1 RUN locale-gen en_US.UTF-8 -COPY . /python-magic -CMD cd /python-magic/test && python3 ./run.py +RUN python3 -m pip install tox +CMD python3 -m tox + diff --git a/test/docker/centos7 b/test/docker/centos7 index f2ac6e40..10f170a0 100644 --- a/test/docker/centos7 +++ b/test/docker/centos7 @@ -1,5 +1,9 @@ FROM centos:7 RUN yum -y update RUN yum -y install file-devel python3 python2 which -COPY . /python-magic -CMD cd /python-magic/test && SKIP_FROM_DESCRIPTOR=1 python3 ./run.py +WORKDIR /python-magic +COPY . . +RUN python3 -m pip install tox +ENV SKIP_FROM_DESCRIPTOR=1 +CMD python3 -m tox + diff --git a/test/docker/centos8 b/test/docker/centos8 index 968f6b65..3b3da0af 100644 --- a/test/docker/centos8 +++ b/test/docker/centos8 @@ -4,6 +4,8 @@ RUN yum -y install file-libs python3 python2 which glibc-locale-source RUN yum reinstall glibc-common -y && \ localedef -i en_US -f UTF-8 en_US.UTF-8 && \ echo "LANG=en_US.UTF-8" > /etc/locale.conf -ENV LANG en_US.UTF-8 -COPY . /python-magic -CMD cd /python-magic/test && python3 ./run.py +WORKDIR /python-magic +COPY . . +RUN python3 -m pip install tox +CMD python3 -m tox + diff --git a/test/docker/focal b/test/docker/focal index 74e4d78a..b35040b5 100755 --- a/test/docker/focal +++ b/test/docker/focal @@ -1,8 +1,9 @@ FROM ubuntu:focal +WORKDIR /python-magic +COPY . . RUN apt-get update -RUN apt-get -y install python2 -RUN apt-get -y install python3 -RUN apt-get -y install locales +RUN apt-get -y install python python3 locales python3-pip libmagic1 RUN locale-gen en_US.UTF-8 -COPY . /python-magic -CMD cd /python-magic/test && python3 ./run.py +RUN python3 -m pip install tox +CMD python3 -m tox + diff --git a/test/docker/xenial b/test/docker/xenial index bc0440be..e3c0433b 100755 --- a/test/docker/xenial +++ b/test/docker/xenial @@ -1,8 +1,9 @@ FROM ubuntu:xenial +WORKDIR /python-magic +COPY . . RUN apt-get update -RUN apt-get -y install python -RUN apt-get -y install python3 -RUN apt-get -y install locales +RUN apt-get -y install python python3 locales python3-pip libmagic1 RUN locale-gen en_US.UTF-8 -COPY . /python-magic -CMD cd /python-magic/test && python3 ./run.py +RUN python3 -m pip install tox +CMD python3 -m tox + diff --git a/test/run_all_versions.py b/test/run_all_versions.py deleted file mode 100644 index d9e7ec58..00000000 --- a/test/run_all_versions.py +++ /dev/null @@ -1,35 +0,0 @@ -import subprocess -import os.path -import sys - -this_dir = os.path.dirname(sys.argv[0]) - -new_env = dict(os.environ) -new_env.update({ - 'LC_ALL': 'en_US.UTF-8', - 'PYTHONPATH': os.path.join(this_dir, ".."), -}) - - -def has_py(version): - ret = subprocess.run("which %s" % version, shell=True, stdout=subprocess.DEVNULL) - return ret.returncode == 0 - - -def run_test(versions): - found = False - for i in versions: - if not has_py(i): - # if this version doesn't exist in path, skip - continue - found = True - print("Testing %s" % i) - subprocess.run([i, os.path.join(this_dir, "python_magic_test.py")], env=new_env, check=True) - subprocess.run([i, os.path.join(this_dir, "libmagic_test.py")], env=new_env, check=True) - - if not found: - sys.exit("No versions found: " + str(versions)) - -run_test(["python2", "python2.7"]) -run_test(["python3.5", "python3.6", "python3.7", "python3.8", "python3.9"]) - diff --git a/tox.ini b/tox.ini index 65595983..1aa47a9d 100644 --- a/tox.ini +++ b/tox.ini @@ -12,15 +12,15 @@ envlist = [testenv] commands = - coverage run --source=magic ./test/test.py + coverage run -m pytest setenv = COVERAGE_FILE=.coverage.{envname} LC_ALL=en_US.UTF-8 deps = .[test] - zope.testrunner coverage + pytest [testenv:coverage-clean] deps = coverage @@ -44,4 +44,5 @@ commands = deps = mypy skip_install = true commands = - mypy magic.pyi + mypy -p magic + From 65a971f17942d4b6610efc40d5c698b3241baa39 Mon Sep 17 00:00:00 2001 From: Adam Hupp Date: Sun, 22 May 2022 07:34:43 -0700 Subject: [PATCH 23/78] update to 0.4.26 --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 8d27ce2c..a98fbef7 100644 --- a/setup.py +++ b/setup.py @@ -18,7 +18,7 @@ def read(file_name): author='Adam Hupp', author_email='adam@hupp.org', url="http://github.com/ahupp/python-magic", - version='0.4.25', + version='0.4.26', long_description=read('README.md'), long_description_content_type='text/markdown', packages=['magic'], From 73e98a5d0de8da9f660ea501e335aec8cdd27333 Mon Sep 17 00:00:00 2001 From: Adam Hupp Date: Sun, 22 May 2022 09:33:32 -0700 Subject: [PATCH 24/78] update CHANGELOG --- CHANGELOG | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGELOG b/CHANGELOG index 89e63143..3ae6ceb0 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -1,3 +1,7 @@ +Changes to 0.4.26: + - Use tox for all multi-version testing + - Fix use of pytest, use it via tox + Changes to 0.4.25: - Support os.PathLike values in Magic.from_file and magic.from_file - Handle some versions of libmagic that return mime string without charset From b443195104d89363b93a547584c1a12fce3b57ec Mon Sep 17 00:00:00 2001 From: Adam Hupp Date: Tue, 7 Jun 2022 12:50:57 -0700 Subject: [PATCH 25/78] bump version to remove accidental pyproject.toml in sdist --- CHANGELOG | 3 +++ setup.py | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/CHANGELOG b/CHANGELOG index 3ae6ceb0..d8212467 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -1,3 +1,6 @@ +Changes to 0.4.27: + - remove spurious pyproject.toml that breaks source builds + Changes to 0.4.26: - Use tox for all multi-version testing - Fix use of pytest, use it via tox diff --git a/setup.py b/setup.py index a98fbef7..06386c30 100644 --- a/setup.py +++ b/setup.py @@ -18,7 +18,7 @@ def read(file_name): author='Adam Hupp', author_email='adam@hupp.org', url="http://github.com/ahupp/python-magic", - version='0.4.26', + version='0.4.27', long_description=read('README.md'), long_description_content_type='text/markdown', packages=['magic'], From f3ab085ce3610e5d4e8f695370cecc5d7d034cbe Mon Sep 17 00:00:00 2001 From: valpogus <32718480+valpogus@users.noreply.github.com> Date: Thu, 16 Jun 2022 13:26:45 +0200 Subject: [PATCH 26/78] Add "magic-1.dll" to the list of DLL files to search for on Windows This is the name of the DLL file generated when using vcpkg to build libmagic: https://github.com/microsoft/vcpkg/tree/master/ports/libmagic --- magic/loader.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/magic/loader.py b/magic/loader.py index 931f1619..228a35cb 100644 --- a/magic/loader.py +++ b/magic/loader.py @@ -21,7 +21,7 @@ def _lib_candidates(): elif sys.platform in ('win32', 'cygwin'): - prefixes = ['libmagic', 'magic1', 'cygmagic-1', 'libmagic-1', 'msys-magic-1'] + prefixes = ['libmagic', 'magic1', 'magic-1', 'cygmagic-1', 'libmagic-1', 'msys-magic-1'] for i in prefixes: # find_library searches in %PATH% but not the current directory, From cc0c5874d4006b0121e6cc59dff1f12536d5bba7 Mon Sep 17 00:00:00 2001 From: ekko Date: Sat, 22 Oct 2022 00:45:54 +0530 Subject: [PATCH 27/78] corrected the command for the test for python3 --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 9eb70e8a..7b855273 100644 --- a/README.md +++ b/README.md @@ -116,7 +116,7 @@ To run tests locally across all available python versions: To run against a specific python version: ``` -LC_ALL=en_US.UTF-8 python3 test/test.py +LC_ALL=en_US.UTF-8 python3 test/python_magic_test.py ``` ## libmagic python API compatibility From cd3929fa7cbc2e383629d0893fc08bcb68a7614c Mon Sep 17 00:00:00 2001 From: Manuele <57706020+manvento@users.noreply.github.com> Date: Mon, 24 Oct 2022 17:42:37 +0200 Subject: [PATCH 28/78] added path for local homebrew installation (#267) * added path for local homebrew installation * used brew --prefix to detect local path --- magic/loader.py | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/magic/loader.py b/magic/loader.py index 228a35cb..6d413655 100644 --- a/magic/loader.py +++ b/magic/loader.py @@ -3,6 +3,7 @@ import sys import glob import os.path +import subprocess def _lib_candidates(): @@ -13,8 +14,16 @@ def _lib_candidates(): paths = [ '/opt/local/lib', '/usr/local/lib', - '/opt/homebrew/lib', - ] + glob.glob('/usr/local/Cellar/libmagic/*/lib') + '/opt/homebrew/lib' + ] + + try: + local_brew_path = subprocess.check_output(['brew', '--prefix']).decode('UTF-8') + paths.append(f'{local_brew_path.strip()}/lib') + except: + pass + + paths += glob.glob('/usr/local/Cellar/libmagic/*/lib') for i in paths: yield os.path.join(i, 'libmagic.dylib') From ea2150d2a739dff8f182c32252c601b350f01f2c Mon Sep 17 00:00:00 2001 From: Adam Hupp Date: Mon, 24 Oct 2022 08:11:21 -0700 Subject: [PATCH 29/78] add python 3.10 to tox environment list --- tox.ini | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tox.ini b/tox.ini index 1aa47a9d..f0e65c81 100644 --- a/tox.ini +++ b/tox.ini @@ -7,16 +7,17 @@ envlist = py37, py38, py39, + py310, coverage-report, mypy [testenv] commands = - coverage run -m pytest + coverage run -m pytest setenv = COVERAGE_FILE=.coverage.{envname} - LC_ALL=en_US.UTF-8 + LC_ALL=en_US.UTF-8 deps = .[test] coverage From 05fde96b7c7feac611c1f2e7eb6bcf4b9ff933b9 Mon Sep 17 00:00:00 2001 From: Adam Hupp Date: Mon, 24 Oct 2022 08:19:55 -0700 Subject: [PATCH 30/78] further update README for test running changes --- README.md | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index 7b855273..fb1bc0eb 100644 --- a/README.md +++ b/README.md @@ -101,22 +101,23 @@ triage it. ## Running the tests -To run the tests across a variety of linux distributions (depends on Docker): +We use the `tox` test runner which can be installed with `python -m pip install tox`. + +To run tests locally across all available python versions: ``` -./test_docker.sh +python -m tox ``` -To run tests locally across all available python versions: +Or to run just against a single version: ``` -./test/run.py +python -m tox py ``` - -To run against a specific python version: +To run the tests across a variety of linux distributions (depends on Docker): ``` -LC_ALL=en_US.UTF-8 python3 test/python_magic_test.py +./test/run_all_docker_test.sh ``` ## libmagic python API compatibility From 3794a39a9dff31a12dc2e6140b736be431c17750 Mon Sep 17 00:00:00 2001 From: Adam Hupp Date: Mon, 24 Oct 2022 08:20:54 -0700 Subject: [PATCH 31/78] docker test cleanup --- test/docker/alpine | 2 -- test/docker/archlinux | 7 ++++--- test/docker/bionic | 7 +++---- test/docker/centos7 | 5 ++--- test/docker/centos8 | 3 +-- test/docker/focal | 7 ++++--- test/docker/xenial | 7 +++---- test/run_all_docker_test.sh | 4 ++-- 8 files changed, 19 insertions(+), 23 deletions(-) diff --git a/test/docker/alpine b/test/docker/alpine index c36b7201..60b0698d 100755 --- a/test/docker/alpine +++ b/test/docker/alpine @@ -3,5 +3,3 @@ RUN apk add python3 python2 libmagic WORKDIR /python-magic COPY . . RUN python3 -m pip install tox -CMD python3 -m tox - diff --git a/test/docker/archlinux b/test/docker/archlinux index fc96c447..6592ffc8 100755 --- a/test/docker/archlinux +++ b/test/docker/archlinux @@ -1,5 +1,6 @@ FROM archlinux:latest RUN yes | pacman -Syyu --overwrite '*' -RUN yes | pacman -S python python2 file which -COPY . /python-magic -CMD cd /python-magic/test && python3 ./run.py +RUN yes | pacman -S python python-pip file which +WORKDIR /python-magic +COPY . . +RUN python3 -m pip install tox diff --git a/test/docker/bionic b/test/docker/bionic index 2fe482c6..a37b2534 100755 --- a/test/docker/bionic +++ b/test/docker/bionic @@ -1,9 +1,8 @@ FROM ubuntu:bionic -WORKDIR /python-magic -COPY . . RUN apt-get update RUN apt-get -y install python python3 locales python3-pip libmagic1 RUN locale-gen en_US.UTF-8 -RUN python3 -m pip install tox -CMD python3 -m tox +WORKDIR /python-magic +COPY . . +RUN python3 -m pip install tox diff --git a/test/docker/centos7 b/test/docker/centos7 index 10f170a0..9caa9898 100644 --- a/test/docker/centos7 +++ b/test/docker/centos7 @@ -1,9 +1,8 @@ FROM centos:7 RUN yum -y update RUN yum -y install file-devel python3 python2 which +ENV SKIP_FROM_DESCRIPTOR=1 + WORKDIR /python-magic COPY . . RUN python3 -m pip install tox -ENV SKIP_FROM_DESCRIPTOR=1 -CMD python3 -m tox - diff --git a/test/docker/centos8 b/test/docker/centos8 index 3b3da0af..7f2dbd06 100644 --- a/test/docker/centos8 +++ b/test/docker/centos8 @@ -4,8 +4,7 @@ RUN yum -y install file-libs python3 python2 which glibc-locale-source RUN yum reinstall glibc-common -y && \ localedef -i en_US -f UTF-8 en_US.UTF-8 && \ echo "LANG=en_US.UTF-8" > /etc/locale.conf + WORKDIR /python-magic COPY . . RUN python3 -m pip install tox -CMD python3 -m tox - diff --git a/test/docker/focal b/test/docker/focal index b35040b5..f24d2317 100755 --- a/test/docker/focal +++ b/test/docker/focal @@ -1,9 +1,10 @@ FROM ubuntu:focal -WORKDIR /python-magic -COPY . . RUN apt-get update RUN apt-get -y install python python3 locales python3-pip libmagic1 RUN locale-gen en_US.UTF-8 + +WORKDIR /python-magic +COPY . . RUN python3 -m pip install tox -CMD python3 -m tox + diff --git a/test/docker/xenial b/test/docker/xenial index e3c0433b..fe7829be 100755 --- a/test/docker/xenial +++ b/test/docker/xenial @@ -1,9 +1,8 @@ FROM ubuntu:xenial -WORKDIR /python-magic -COPY . . RUN apt-get update RUN apt-get -y install python python3 locales python3-pip libmagic1 RUN locale-gen en_US.UTF-8 -RUN python3 -m pip install tox -CMD python3 -m tox +WORKDIR /python-magic +COPY . . +RUN python3 -m pip install tox diff --git a/test/run_all_docker_test.sh b/test/run_all_docker_test.sh index 68a28081..dce930b7 100755 --- a/test/run_all_docker_test.sh +++ b/test/run_all_docker_test.sh @@ -6,8 +6,8 @@ set -x ROOT=$(dirname $0)/.. cd $ROOT -for f in test/docker/*; do +for f in test/docker/*; do H=$(docker build -q -f ${f} .) - docker run --rm $H + docker run --rm $H python3 -m tox done From 4ffcd59113fa26d7c2e9d5897b1eef919fd4b457 Mon Sep 17 00:00:00 2001 From: Adam Hupp Date: Mon, 9 Jan 2023 12:55:15 -0800 Subject: [PATCH 32/78] update test for upstream added gzip extensions --- test/python_magic_test.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/test/python_magic_test.py b/test/python_magic_test.py index 624a443c..d51587ce 100755 --- a/test/python_magic_test.py +++ b/test/python_magic_test.py @@ -134,7 +134,7 @@ def test_extension(self): self.assert_values(m, { # some versions return '' for the extensions of a gz file, # including w/ the command line. Who knows... - 'test.gz': ('gz/tgz/tpz/zabw/svgz', '', '???'), + 'test.gz': ('gz/tgz/tpz/zabw/svgz/adz/kmy/xcfgz', 'gz/tgz/tpz/zabw/svgz', '', '???'), 'name_use.jpg': 'jpeg/jpg/jpe/jfif', }) except NotImplementedError: @@ -227,6 +227,5 @@ def test_pathlike(self): m = magic.Magic(mime=True) self.assertEqual('application/pdf', m.from_file(path)) - if __name__ == '__main__': unittest.main() From b80592597061bb679e99768319a57b4de3ca45cc Mon Sep 17 00:00:00 2001 From: Adam Hupp Date: Tue, 10 Jan 2023 15:03:02 -0800 Subject: [PATCH 33/78] don't run coverage by default in tox config --- tox.ini | 2 -- 1 file changed, 2 deletions(-) diff --git a/tox.ini b/tox.ini index f0e65c81..272d8c58 100644 --- a/tox.ini +++ b/tox.ini @@ -1,6 +1,5 @@ [tox] envlist = - coverage-clean, py27, py35, py36, @@ -8,7 +7,6 @@ envlist = py38, py39, py310, - coverage-report, mypy [testenv] From c7a2e7bc7c387af23b3c896bf05003cf2bd8646b Mon Sep 17 00:00:00 2001 From: Adam Hupp Date: Tue, 10 Jan 2023 15:03:22 -0800 Subject: [PATCH 34/78] remove python3-ism from loader --- magic/loader.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/magic/loader.py b/magic/loader.py index 6d413655..82b02aad 100644 --- a/magic/loader.py +++ b/magic/loader.py @@ -19,7 +19,8 @@ def _lib_candidates(): try: local_brew_path = subprocess.check_output(['brew', '--prefix']).decode('UTF-8') - paths.append(f'{local_brew_path.strip()}/lib') + local_brew_path = local_brew_path.strip() + paths.append(local_brew_path + '/lib') except: pass From c7642f02b26f15a5e1bb692d103cadeeeaba9bc1 Mon Sep 17 00:00:00 2001 From: Stevie Gayet <87695919+stegayet@users.noreply.github.com> Date: Mon, 23 Jan 2023 16:06:24 +0100 Subject: [PATCH 35/78] chore(tests): add Python 3.10 and 3.11 in CI runs/tox/trove classifier (#284) --- .travis.yml | 2 ++ setup.py | 2 ++ tox.ini | 1 + 3 files changed, 5 insertions(+) diff --git a/.travis.yml b/.travis.yml index 8c306d9a..bf4dbaf2 100644 --- a/.travis.yml +++ b/.travis.yml @@ -9,6 +9,8 @@ python: - "3.7" - "3.8" - "3.9" + - "3.10" + - "3.11" install: - pip install coverage coveralls codecov diff --git a/setup.py b/setup.py index 06386c30..b7cfdc6b 100644 --- a/setup.py +++ b/setup.py @@ -39,6 +39,8 @@ def read(file_name): 'Programming Language :: Python :: 3.7', 'Programming Language :: Python :: 3.8', 'Programming Language :: Python :: 3.9', + 'Programming Language :: Python :: 3.10', + 'Programming Language :: Python :: 3.11', 'Programming Language :: Python :: Implementation :: CPython', ], ) diff --git a/tox.ini b/tox.ini index 272d8c58..ae9f02a6 100644 --- a/tox.ini +++ b/tox.ini @@ -7,6 +7,7 @@ envlist = py38, py39, py310, + py311, mypy [testenv] From e0052c5f9ef3e78be1bf27b4832ba6c73715d055 Mon Sep 17 00:00:00 2001 From: Adam Hupp Date: Thu, 23 Mar 2023 13:54:10 -0700 Subject: [PATCH 36/78] bump to 0.4.28 --- CHANGELOG | 5 +++++ setup.py | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/CHANGELOG b/CHANGELOG index d8212467..b4f5f202 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -1,3 +1,8 @@ +Changes to 0.4.28: + - support "magic-1.dll" on Windows, which is produced by vcpkg + - add python 3.10 to tox config + - update test for upstream gzip extensions + Changes to 0.4.27: - remove spurious pyproject.toml that breaks source builds diff --git a/setup.py b/setup.py index b7cfdc6b..ac47d8c4 100644 --- a/setup.py +++ b/setup.py @@ -18,7 +18,7 @@ def read(file_name): author='Adam Hupp', author_email='adam@hupp.org', url="http://github.com/ahupp/python-magic", - version='0.4.27', + version='0.4.28', long_description=read('README.md'), long_description_content_type='text/markdown', packages=['magic'], From 7cde7850911d3c5e0da2fe6b2b066789ee4ae021 Mon Sep 17 00:00:00 2001 From: Adam Hupp Date: Thu, 30 Mar 2023 11:22:14 -0700 Subject: [PATCH 37/78] Revert "remove python3-ism from loader" This reverts commit c7a2e7bc7c387af23b3c896bf05003cf2bd8646b. --- magic/loader.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/magic/loader.py b/magic/loader.py index 82b02aad..6d413655 100644 --- a/magic/loader.py +++ b/magic/loader.py @@ -19,8 +19,7 @@ def _lib_candidates(): try: local_brew_path = subprocess.check_output(['brew', '--prefix']).decode('UTF-8') - local_brew_path = local_brew_path.strip() - paths.append(local_brew_path + '/lib') + paths.append(f'{local_brew_path.strip()}/lib') except: pass From 3ab96087d22872fc973a3b3a2db551db9bbcb5fa Mon Sep 17 00:00:00 2001 From: Adam Hupp Date: Thu, 30 Mar 2023 11:22:46 -0700 Subject: [PATCH 38/78] Revert "added path for local homebrew installation (#267)" This reverts commit cd3929fa7cbc2e383629d0893fc08bcb68a7614c. --- magic/loader.py | 13 ++----------- 1 file changed, 2 insertions(+), 11 deletions(-) diff --git a/magic/loader.py b/magic/loader.py index 6d413655..228a35cb 100644 --- a/magic/loader.py +++ b/magic/loader.py @@ -3,7 +3,6 @@ import sys import glob import os.path -import subprocess def _lib_candidates(): @@ -14,16 +13,8 @@ def _lib_candidates(): paths = [ '/opt/local/lib', '/usr/local/lib', - '/opt/homebrew/lib' - ] - - try: - local_brew_path = subprocess.check_output(['brew', '--prefix']).decode('UTF-8') - paths.append(f'{local_brew_path.strip()}/lib') - except: - pass - - paths += glob.glob('/usr/local/Cellar/libmagic/*/lib') + '/opt/homebrew/lib', + ] + glob.glob('/usr/local/Cellar/libmagic/*/lib') for i in paths: yield os.path.join(i, 'libmagic.dylib') From 545a2a561522efc2869066792062694b59b1b39c Mon Sep 17 00:00:00 2001 From: Dominique Leuenberger Date: Wed, 2 Aug 2023 11:29:47 +0200 Subject: [PATCH 39/78] Fix test suite with file 5.45 [ 12s] test/python_magic_test.py:53: in assert_values [ 12s] self.assertIn(value, expected_value) [ 12s] E AssertionError: 'PDF document, version 1.2, 2 page(s)' not found in ('PDF document, version 1.2', 'PDF document, version 1.2, 2 pages') --- test/libmagic_test.py | 2 +- test/python_magic_test.py | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/test/libmagic_test.py b/test/libmagic_test.py index 5719a58e..7b4665b5 100644 --- a/test/libmagic_test.py +++ b/test/libmagic_test.py @@ -15,7 +15,7 @@ class MagicTestCase(unittest.TestCase): filename = os.path.join(TESTDATA_DIR, 'test.pdf') expected_mime_type = 'application/pdf' expected_encoding = 'us-ascii' - expected_name = ('PDF document, version 1.2', 'PDF document, version 1.2, 2 pages') + expected_name = ('PDF document, version 1.2', 'PDF document, version 1.2, 2 pages', 'PDF document, version 1.2, 2 page(s)') def assert_result(self, result): self.assertEqual(result.mime_type, self.expected_mime_type) diff --git a/test/python_magic_test.py b/test/python_magic_test.py index d51587ce..410a1495 100755 --- a/test/python_magic_test.py +++ b/test/python_magic_test.py @@ -108,7 +108,8 @@ def test_descriptions(self): self.assert_values(m, { 'magic._pyc_': 'python 2.4 byte-compiled', 'test.pdf': ('PDF document, version 1.2', - 'PDF document, version 1.2, 2 pages'), + 'PDF document, version 1.2, 2 pages', + 'PDF document, version 1.2, 2 page(s)'), 'test.gz': ('gzip compressed data, was "test", from Unix, last ' 'modified: Sun Jun 29 01:32:52 2008', From 0cc3cf885135d50e18feabd9ed2496b6355d72df Mon Sep 17 00:00:00 2001 From: Stevie Date: Sun, 6 Aug 2023 00:13:22 +0200 Subject: [PATCH 40/78] chore(python): add Python 3.12 in test matrix --- .travis.yml | 1 + setup.py | 1 + tox.ini | 1 + 3 files changed, 3 insertions(+) diff --git a/.travis.yml b/.travis.yml index bf4dbaf2..c83c031b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -11,6 +11,7 @@ python: - "3.9" - "3.10" - "3.11" + - "3.12" install: - pip install coverage coveralls codecov diff --git a/setup.py b/setup.py index ac47d8c4..d98b7318 100644 --- a/setup.py +++ b/setup.py @@ -41,6 +41,7 @@ def read(file_name): 'Programming Language :: Python :: 3.9', 'Programming Language :: Python :: 3.10', 'Programming Language :: Python :: 3.11', + 'Programming Language :: Python :: 3.12', 'Programming Language :: Python :: Implementation :: CPython', ], ) diff --git a/tox.ini b/tox.ini index ae9f02a6..b6ed98c7 100644 --- a/tox.ini +++ b/tox.ini @@ -8,6 +8,7 @@ envlist = py39, py310, py311, + py312, mypy [testenv] From 722995466fda1451570c555e688819b75f1b8e65 Mon Sep 17 00:00:00 2001 From: Marten Ringwelski Date: Wed, 23 Aug 2023 22:17:30 +0200 Subject: [PATCH 41/78] fix: Don't raise FileNotFoundException on symlinks The builtin `open` will always follow symlinks. Using `os.stat` is the easiest solution imo. An alternative would be using `os.access` but that does not raise a FileNotFoundException so I chose `os.stat`. --- magic/__init__.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/magic/__init__.py b/magic/__init__.py index bab7c7b1..1c1acc2e 100644 --- a/magic/__init__.py +++ b/magic/__init__.py @@ -17,6 +17,7 @@ """ import sys +import os import glob import ctypes import ctypes.util @@ -25,9 +26,6 @@ from ctypes import c_char_p, c_int, c_size_t, c_void_p, byref, POINTER -# avoid shadowing the real open with the version from compat.py -_real_open = open - class MagicException(Exception): def __init__(self, message): @@ -109,8 +107,7 @@ def from_buffer(self, buf): def from_file(self, filename): # raise FileNotFoundException or IOError if the file does not exist - with _real_open(filename): - pass + os.stat(filename, follow_symlinks=self.flags & MAGIC_SYMLINK) with self.lock: try: From 2a01b18ae0fe27e51977a54f0589910ddcc05804 Mon Sep 17 00:00:00 2001 From: Adam Hupp Date: Fri, 25 Aug 2023 11:02:53 -0700 Subject: [PATCH 42/78] add MAGIC_SYMLINK support, and tests for same --- CHANGELOG | 78 +++++++------ magic/__init__.py | 6 +- test/README | 6 +- test/python_magic_test.py | 224 ++++++++++++++++++++++++-------------- 4 files changed, 193 insertions(+), 121 deletions(-) diff --git a/CHANGELOG b/CHANGELOG index b4f5f202..b6766f94 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -1,41 +1,51 @@ +Changes to 0.4.29: + +- support MAGIC_SYMLINK (via follow_symlink flag on Magic constructor) +- correctly throw FileNotFoundException depending on flag + Changes to 0.4.28: - - support "magic-1.dll" on Windows, which is produced by vcpkg - - add python 3.10 to tox config - - update test for upstream gzip extensions + +- support "magic-1.dll" on Windows, which is produced by vcpkg +- add python 3.10 to tox config +- update test for upstream gzip extensions Changes to 0.4.27: - - remove spurious pyproject.toml that breaks source builds + +- remove spurious pyproject.toml that breaks source builds Changes to 0.4.26: - - Use tox for all multi-version testing - - Fix use of pytest, use it via tox + +- Use tox for all multi-version testing +- Fix use of pytest, use it via tox Changes to 0.4.25: - - Support os.PathLike values in Magic.from_file and magic.from_file - - Handle some versions of libmagic that return mime string without charset - - Fix tests for file 5.41 - - Include typing stub in package + +- Support os.PathLike values in Magic.from_file and magic.from_file +- Handle some versions of libmagic that return mime string without charset +- Fix tests for file 5.41 +- Include typing stub in package Changes to 0.4.24: - - Fix regression in library loading on some Alpine docker images. + +- Fix regression in library loading on some Alpine docker images. Changes to 0.4.23 - - Include a `py.typed` sentinal to enable type checking - - Improve fix for attribute error during destruction - - Cleanup library loading logic - - Add new homebrew library dir for OSX +- Include a `py.typed` sentinal to enable type checking +- Improve fix for attribute error during destruction +- Cleanup library loading logic +- Add new homebrew library dir for OSX Changes to 0.4.21, 0.4.22 - - Unify dll loader between the standard and compat library, fixing load - failures on some previously supported platforms. +- Unify dll loader between the standard and compat library, fixing load + failures on some previously supported platforms. Changes to 0.4.20 - merge in a compatibility layer for the upstream libmagic python binding. Since both this package and that one are called 'magic', this compat layer - removes a very common source of runtime errors. Use of that libmagic API will + removes a very common source of runtime errors. Use of that libmagic API will produce a deprecation warning. - support python 3.9 in tests and pypi metadata @@ -44,9 +54,9 @@ Changes to 0.4.20 rather than a filename. - sometimes the returned description includes snippets of the file, e.g a title - for MS Word docs. Since this is in an unknown encoding, we would throw a - unicode decode error trying to decode. Now, it decodes with - 'backslashreplace' to handle this more gracefully. The undecodable characters + for MS Word docs. Since this is in an unknown encoding, we would throw a + unicode decode error trying to decode. Now, it decodes with + 'backslashreplace' to handle this more gracefully. The undecodable characters are replaced with hex escapes. - add support for MAGIC_EXTENSION, to return possible file extensions. @@ -55,18 +65,18 @@ Changes to 0.4.20 Changes in 0.4.18 -- Make bindings for magic_[set|get]param optional, and throw NotImplementedError -if they are used but not supported. Only call setparam() in the constructor if -it's supported. This prevents breakage on CentOS7 which uses an old version of -libmagic. +- Make bindings for magic\_[set|get]param optional, and throw NotImplementedError + if they are used but not supported. Only call setparam() in the constructor if + it's supported. This prevents breakage on CentOS7 which uses an old version of + libmagic. - Add tests for CentOS 7 & 8 Changes in 0.4.16 and 0.4.17 - add MAGIC_MIME_TYPE constant, use that in preference to MAGIC_MIME internally. -This sets up for a breaking change in a future major version bump where -MAGIC_MIME will change to mathch magic.h. + This sets up for a breaking change in a future major version bump where + MAGIC_MIME will change to mathch magic.h. - add magic.version() function to return library version - add setparam/getparam to control internal behavior - increase internal limits with setparam to prevent spurious error on some jpeg files @@ -76,12 +86,12 @@ MAGIC_MIME will change to mathch magic.h. - include tests in source distribution - many test improvements: --- tox runner support --- remove deprecated test_suite field from setup.py --- docker tests that cover all LTS ubuntu versions --- add test for snapp file identification + -- tox runner support + -- remove deprecated test_suite field from setup.py + -- docker tests that cover all LTS ubuntu versions + -- add test for snapp file identification - doc improvements --- document dependency install process for debian --- various typos --- document test running process + -- document dependency install process for debian + -- various typos + -- document test running process diff --git a/magic/__init__.py b/magic/__init__.py index 1c1acc2e..d05ebf98 100644 --- a/magic/__init__.py +++ b/magic/__init__.py @@ -39,7 +39,8 @@ class Magic: """ def __init__(self, mime=False, magic_file=None, mime_encoding=False, - keep_going=False, uncompress=False, raw=False, extension=False): + keep_going=False, uncompress=False, raw=False, extension=False, + follow_symlinks=False): """ Create a new libmagic wrapper. @@ -65,6 +66,9 @@ def __init__(self, mime=False, magic_file=None, mime_encoding=False, if extension: self.flags |= MAGIC_EXTENSION + if follow_symlinks: + self.flags |= MAGIC_SYMLINK + self.cookie = magic_open(self.flags) self.lock = threading.Lock() diff --git a/test/README b/test/README index c34cb6ac..215ee43a 100644 --- a/test/README +++ b/test/README @@ -1,6 +1,4 @@ There are a few ways to run the python-magic tests - 1. `pytest` will run the test suite against your default version of python - 2. `./test/run_all_versions.py` will run the tests against all installed versions of python. - 3. `./test/run_all_docker_test.sh` will run against a variety of different Linux distributions, using docker. - +1. `tox` will run the tests against all installed versions of python +2. `./test/run_all_docker_test.sh` will run against a variety of different Linux distributions, using docker. diff --git a/test/python_magic_test.py b/test/python_magic_test.py index 410a1495..41ed4af7 100755 --- a/test/python_magic_test.py +++ b/test/python_magic_test.py @@ -1,9 +1,10 @@ +import tempfile import os # for output which reports a local time -os.environ['TZ'] = 'GMT' +os.environ["TZ"] = "GMT" -if os.environ.get('LC_ALL', '') != 'en_US.UTF-8': +if os.environ.get("LC_ALL", "") != "en_US.UTF-8": # this ensure we're in a utf-8 default filesystem encoding which is # necessary for some tests raise Exception("must run `export LC_ALL=en_US.UTF-8` before running test suite") @@ -16,10 +17,11 @@ import sys # magic_descriptor is broken (?) in centos 7, so don't run those tests -SKIP_FROM_DESCRIPTOR = bool(os.environ.get('SKIP_FROM_DESCRIPTOR')) +SKIP_FROM_DESCRIPTOR = bool(os.environ.get("SKIP_FROM_DESCRIPTOR")) + class MagicTest(unittest.TestCase): - TESTDATA_DIR = os.path.abspath(os.path.join(os.path.dirname(__file__), 'testdata')) + TESTDATA_DIR = os.path.abspath(os.path.join(os.path.dirname(__file__), "testdata")) def test_version(self): try: @@ -28,20 +30,19 @@ def test_version(self): pass def test_fs_encoding(self): - self.assertEqual('utf-8', sys.getfilesystemencoding().lower()) + self.assertEqual("utf-8", sys.getfilesystemencoding().lower()) def assert_values(self, m, expected_values, buf_equals_file=True): for filename, expected_value in expected_values.items(): try: filename = os.path.join(self.TESTDATA_DIR, filename) except TypeError: - filename = os.path.join( - self.TESTDATA_DIR.encode('utf-8'), filename) + filename = os.path.join(self.TESTDATA_DIR.encode("utf-8"), filename) if type(expected_value) is not tuple: expected_value = (expected_value,) - with open(filename, 'rb') as f: + with open(filename, "rb") as f: buf_value = m.from_buffer(f.read()) file_value = m.from_file(filename) @@ -55,10 +56,10 @@ def assert_values(self, m, expected_values, buf_equals_file=True): def test_from_file_str_and_bytes(self): filename = os.path.join(self.TESTDATA_DIR, "test.pdf") - self.assertEqual('application/pdf', - magic.from_file(filename, mime=True)) - self.assertEqual('application/pdf', - magic.from_file(filename.encode('utf-8'), mime=True)) + self.assertEqual("application/pdf", magic.from_file(filename, mime=True)) + self.assertEqual( + "application/pdf", magic.from_file(filename.encode("utf-8"), mime=True) + ) def test_from_descriptor_str_and_bytes(self): if SKIP_FROM_DESCRIPTOR: @@ -66,10 +67,12 @@ def test_from_descriptor_str_and_bytes(self): filename = os.path.join(self.TESTDATA_DIR, "test.pdf") with open(filename) as f: - self.assertEqual('application/pdf', - magic.from_descriptor(f.fileno(), mime=True)) - self.assertEqual('application/pdf', - magic.from_descriptor(f.fileno(), mime=True)) + self.assertEqual( + "application/pdf", magic.from_descriptor(f.fileno(), mime=True) + ) + self.assertEqual( + "application/pdf", magic.from_descriptor(f.fileno(), mime=True) + ) def test_from_buffer_str_and_bytes(self): if SKIP_FROM_DESCRIPTOR: @@ -78,125 +81,151 @@ def test_from_buffer_str_and_bytes(self): self.assertTrue( m.from_buffer('#!/usr/bin/env python\nprint("foo")') - in ("text/x-python", "text/x-script.python")) + in ("text/x-python", "text/x-script.python") + ) self.assertTrue( m.from_buffer(b'#!/usr/bin/env python\nprint("foo")') - in ("text/x-python", "text/x-script.python")) + in ("text/x-python", "text/x-script.python") + ) def test_mime_types(self): - dest = os.path.join(MagicTest.TESTDATA_DIR, - b'\xce\xbb'.decode('utf-8')) - shutil.copyfile(os.path.join(MagicTest.TESTDATA_DIR, 'lambda'), dest) + dest = os.path.join(MagicTest.TESTDATA_DIR, b"\xce\xbb".decode("utf-8")) + shutil.copyfile(os.path.join(MagicTest.TESTDATA_DIR, "lambda"), dest) try: m = magic.Magic(mime=True) - self.assert_values(m, { - 'magic._pyc_': ('application/octet-stream', 'text/x-bytecode.python', 'application/x-bytecode.python'), - 'test.pdf': 'application/pdf', - 'test.gz': ('application/gzip', 'application/x-gzip'), - 'test.snappy.parquet': 'application/octet-stream', - 'text.txt': 'text/plain', - b'\xce\xbb'.decode('utf-8'): 'text/plain', - b'\xce\xbb': 'text/plain', - }) + self.assert_values( + m, + { + "magic._pyc_": ( + "application/octet-stream", + "text/x-bytecode.python", + "application/x-bytecode.python", + ), + "test.pdf": "application/pdf", + "test.gz": ("application/gzip", "application/x-gzip"), + "test.snappy.parquet": "application/octet-stream", + "text.txt": "text/plain", + b"\xce\xbb".decode("utf-8"): "text/plain", + b"\xce\xbb": "text/plain", + }, + ) finally: os.unlink(dest) def test_descriptions(self): m = magic.Magic() - os.environ['TZ'] = 'UTC' # To get last modified date of test.gz in UTC + os.environ["TZ"] = "UTC" # To get last modified date of test.gz in UTC try: - self.assert_values(m, { - 'magic._pyc_': 'python 2.4 byte-compiled', - 'test.pdf': ('PDF document, version 1.2', - 'PDF document, version 1.2, 2 pages', - 'PDF document, version 1.2, 2 page(s)'), - 'test.gz': - ('gzip compressed data, was "test", from Unix, last ' - 'modified: Sun Jun 29 01:32:52 2008', - 'gzip compressed data, was "test", last modified' - ': Sun Jun 29 01:32:52 2008, from Unix', - 'gzip compressed data, was "test", last modified' - ': Sun Jun 29 01:32:52 2008, from Unix, original size 15', - 'gzip compressed data, was "test", ' - 'last modified: Sun Jun 29 01:32:52 2008, ' - 'from Unix, original size modulo 2^32 15', - 'gzip compressed data, was "test", last modified' - ': Sun Jun 29 01:32:52 2008, from Unix, truncated' - ), - 'text.txt': 'ASCII text', - 'test.snappy.parquet': ('Apache Parquet', 'Par archive data'), - }, buf_equals_file=False) + self.assert_values( + m, + { + "magic._pyc_": "python 2.4 byte-compiled", + "test.pdf": ( + "PDF document, version 1.2", + "PDF document, version 1.2, 2 pages", + "PDF document, version 1.2, 2 page(s)", + ), + "test.gz": ( + 'gzip compressed data, was "test", from Unix, last ' + "modified: Sun Jun 29 01:32:52 2008", + 'gzip compressed data, was "test", last modified' + ": Sun Jun 29 01:32:52 2008, from Unix", + 'gzip compressed data, was "test", last modified' + ": Sun Jun 29 01:32:52 2008, from Unix, original size 15", + 'gzip compressed data, was "test", ' + "last modified: Sun Jun 29 01:32:52 2008, " + "from Unix, original size modulo 2^32 15", + 'gzip compressed data, was "test", last modified' + ": Sun Jun 29 01:32:52 2008, from Unix, truncated", + ), + "text.txt": "ASCII text", + "test.snappy.parquet": ("Apache Parquet", "Par archive data"), + }, + buf_equals_file=False, + ) finally: - del os.environ['TZ'] + del os.environ["TZ"] def test_extension(self): try: m = magic.Magic(extension=True) - self.assert_values(m, { - # some versions return '' for the extensions of a gz file, - # including w/ the command line. Who knows... - 'test.gz': ('gz/tgz/tpz/zabw/svgz/adz/kmy/xcfgz', 'gz/tgz/tpz/zabw/svgz', '', '???'), - 'name_use.jpg': 'jpeg/jpg/jpe/jfif', - }) + self.assert_values( + m, + { + # some versions return '' for the extensions of a gz file, + # including w/ the command line. Who knows... + "test.gz": ( + "gz/tgz/tpz/zabw/svgz/adz/kmy/xcfgz", + "gz/tgz/tpz/zabw/svgz", + "", + "???", + ), + "name_use.jpg": "jpeg/jpg/jpe/jfif", + }, + ) except NotImplementedError: - self.skipTest('MAGIC_EXTENSION not supported in this version') + self.skipTest("MAGIC_EXTENSION not supported in this version") def test_unicode_result_nonraw(self): m = magic.Magic(raw=False) - src = os.path.join(MagicTest.TESTDATA_DIR, 'pgpunicode') + src = os.path.join(MagicTest.TESTDATA_DIR, "pgpunicode") result = m.from_file(src) # NOTE: This check is added as otherwise some magic files don't identify the test case as a PGP key. - if 'PGP' in result: + if "PGP" in result: assert r"PGP\011Secret Sub-key -" == result else: raise unittest.SkipTest("Magic file doesn't return expected type.") def test_unicode_result_raw(self): m = magic.Magic(raw=True) - src = os.path.join(MagicTest.TESTDATA_DIR, 'pgpunicode') + src = os.path.join(MagicTest.TESTDATA_DIR, "pgpunicode") result = m.from_file(src) - if 'PGP' in result: - assert b'PGP\tSecret Sub-key -' == result.encode('utf-8') + if "PGP" in result: + assert b"PGP\tSecret Sub-key -" == result.encode("utf-8") else: raise unittest.SkipTest("Magic file doesn't return expected type.") def test_mime_encodings(self): m = magic.Magic(mime_encoding=True) - self.assert_values(m, { - 'text-iso8859-1.txt': 'iso-8859-1', - 'text.txt': 'us-ascii', - }) + self.assert_values( + m, + { + "text-iso8859-1.txt": "iso-8859-1", + "text.txt": "us-ascii", + }, + ) def test_errors(self): m = magic.Magic() - self.assertRaises(IOError, m.from_file, 'nonexistent') - self.assertRaises(magic.MagicException, magic.Magic, - magic_file='nonexistent') - os.environ['MAGIC'] = 'nonexistent' + self.assertRaises(IOError, m.from_file, "nonexistent") + self.assertRaises(magic.MagicException, magic.Magic, magic_file="nonexistent") + os.environ["MAGIC"] = "nonexistent" try: self.assertRaises(magic.MagicException, magic.Magic) finally: - del os.environ['MAGIC'] + del os.environ["MAGIC"] def test_keep_going(self): - filename = os.path.join(self.TESTDATA_DIR, 'keep-going.jpg') + filename = os.path.join(self.TESTDATA_DIR, "keep-going.jpg") m = magic.Magic(mime=True) - self.assertEqual(m.from_file(filename), 'image/jpeg') + self.assertEqual(m.from_file(filename), "image/jpeg") try: # this will throw if you have an "old" version of the library # I'm otherwise not sure how to query if keep_going is supported magic.version() m = magic.Magic(mime=True, keep_going=True) - self.assertEqual(m.from_file(filename), - 'image/jpeg\\012- application/octet-stream') + self.assertEqual( + m.from_file(filename), "image/jpeg\\012- application/octet-stream" + ) except NotImplementedError: pass def test_rethrow(self): old = magic.magic_buffer try: + def t(x, y): raise magic.MagicException("passthrough") @@ -217,16 +246,47 @@ def test_getparam(self): def test_name_count(self): m = magic.Magic() - with open(os.path.join(self.TESTDATA_DIR, 'name_use.jpg'), 'rb') as f: + with open(os.path.join(self.TESTDATA_DIR, "name_use.jpg"), "rb") as f: m.from_buffer(f.read()) def test_pathlike(self): if sys.version_info < (3, 6): return from pathlib import Path - path = Path(self.TESTDATA_DIR, "test.pdf") + + path = Path(self.TESTDATA_DIR, "test.pdf") m = magic.Magic(mime=True) - self.assertEqual('application/pdf', m.from_file(path)) + self.assertEqual("application/pdf", m.from_file(path)) + + def test_symlink(self): + # TODO: 3.0 + if not hasattr(tempfile, "TemporaryDirectory"): + return + + with tempfile.TemporaryDirectory() as tmp: + tmp_link = os.path.join(tmp, "test_link") + tmp_broken = os.path.join(tmp, "nonexistent") + + os.symlink( + os.path.join(self.TESTDATA_DIR, "test.pdf"), + tmp_link, + ) + + os.symlink("/nonexistent", tmp_broken) + + m = magic.Magic() + m_follow = magic.Magic(follow_symlinks=True) + self.assertTrue(m.from_file(tmp_link).startswith("symbolic link to ")) + self.assertTrue(m_follow.from_file(tmp_link).startswith("PDF document")) + + self.assertTrue( + m.from_file(tmp_broken).startswith( + "broken symbolic link to /nonexistent" + ) + ) + + self.assertRaises(IOError, m_follow.from_file, tmp_broken) + -if __name__ == '__main__': +if __name__ == "__main__": unittest.main() From 64ed0bdf17226e5c88e723dcea2cd607696b1ad6 Mon Sep 17 00:00:00 2001 From: Robert Scott Date: Sat, 7 Oct 2023 16:35:09 +0100 Subject: [PATCH 43/78] typing stubs: add Magic.__init__ extension & follow_symlinks args --- magic/__init__.pyi | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/magic/__init__.pyi b/magic/__init__.pyi index b6b5489c..542fb420 100644 --- a/magic/__init__.pyi +++ b/magic/__init__.pyi @@ -11,7 +11,7 @@ class Magic: flags: int = ... cookie: Any = ... lock: threading.Lock = ... - def __init__(self, mime: bool = ..., magic_file: Optional[Any] = ..., mime_encoding: bool = ..., keep_going: bool = ..., uncompress: bool = ..., raw: bool = ...) -> None: ... + def __init__(self, mime: bool = ..., magic_file: Optional[Any] = ..., mime_encoding: bool = ..., keep_going: bool = ..., uncompress: bool = ..., raw: bool = ..., extension: bool = ..., follow_symlinks: bool = ...) -> None: ... def from_buffer(self, buf: Union[bytes, str]) -> Text: ... def from_file(self, filename: Union[bytes, str, PathLike]) -> Text: ... def from_descriptor(self, fd: int, mime: bool = ...) -> Text: ... From fd279e08c7aa6c651413f5b2c56605b72989b74e Mon Sep 17 00:00:00 2001 From: Robert Scott Date: Sat, 7 Oct 2023 18:18:08 +0100 Subject: [PATCH 44/78] Magic.__init__: add kwargs to enable/disable different types of magic detection --- magic/__init__.py | 40 +++++++++++++++++++++++++++++++++++----- magic/__init__.pyi | 8 +++++++- 2 files changed, 42 insertions(+), 6 deletions(-) diff --git a/magic/__init__.py b/magic/__init__.py index d05ebf98..d56caafc 100644 --- a/magic/__init__.py +++ b/magic/__init__.py @@ -40,7 +40,10 @@ class Magic: def __init__(self, mime=False, magic_file=None, mime_encoding=False, keep_going=False, uncompress=False, raw=False, extension=False, - follow_symlinks=False): + follow_symlinks=False, check_tar=True, check_soft=True, + check_apptype=True, check_elf=True, check_text=True, + check_cdf=True, check_csv=True, check_encoding=True, + check_json=True, check_simh=True): """ Create a new libmagic wrapper. @@ -69,6 +72,27 @@ def __init__(self, mime=False, magic_file=None, mime_encoding=False, if follow_symlinks: self.flags |= MAGIC_SYMLINK + if not check_tar: + self.flags |= MAGIC_NO_CHECK_TAR + if not check_soft: + self.flags |= MAGIC_NO_CHECK_SOFT + if not check_apptype: + self.flags |= MAGIC_NO_CHECK_APPTYPE + if not check_elf: + self.flags |= MAGIC_NO_CHECK_ELF + if not check_text: + self.flags |= MAGIC_NO_CHECK_TEXT + if not check_cdf: + self.flags |= MAGIC_NO_CHECK_CDF + if not check_csv: + self.flags |= MAGIC_NO_CHECK_CSV + if not check_encoding: + self.flags |= MAGIC_NO_CHECK_ENCODING + if not check_json: + self.flags |= MAGIC_NO_CHECK_JSON + if not check_simh: + self.flags |= MAGIC_NO_CHECK_SIMH + self.cookie = magic_open(self.flags) self.lock = threading.Lock() @@ -411,10 +435,16 @@ def version(): MAGIC_NO_CHECK_SOFT = 0x004000 # Don't check magic entries MAGIC_NO_CHECK_APPTYPE = 0x008000 # Don't check application type MAGIC_NO_CHECK_ELF = 0x010000 # Don't check for elf details -MAGIC_NO_CHECK_ASCII = 0x020000 # Don't check for ascii files -MAGIC_NO_CHECK_TROFF = 0x040000 # Don't check ascii/troff -MAGIC_NO_CHECK_FORTRAN = 0x080000 # Don't check ascii/fortran -MAGIC_NO_CHECK_TOKENS = 0x100000 # Don't check ascii/tokens +MAGIC_NO_CHECK_TEXT = 0x020000 # Don't check for ascii files +MAGIC_NO_CHECK_ASCII = 0x020000 # Deprecated alias for MAGIC_NO_CHECK_TEXT +MAGIC_NO_CHECK_TROFF = 0x040000 # Don't check ascii/troff (deprecated) +MAGIC_NO_CHECK_FORTRAN = 0x080000 # Don't check ascii/fortran (deprecated) +MAGIC_NO_CHECK_TOKENS = 0x100000 # Don't check ascii/tokens (deprecated) +MAGIC_NO_CHECK_CDF = 0x0040000 # Don't check for CDF files +MAGIC_NO_CHECK_CSV = 0x0080000 # Don't check for CSV files +MAGIC_NO_CHECK_ENCODING = 0x0200000 # Don't check text encodings +MAGIC_NO_CHECK_JSON = 0x0400000 # Don't check for JSON files +MAGIC_NO_CHECK_SIMH = 0x0800000 # Don't check for SIMH tape files MAGIC_PARAM_INDIR_MAX = 0 # Recursion limit for indirect magic MAGIC_PARAM_NAME_MAX = 1 # Use count limit for name/use magic diff --git a/magic/__init__.pyi b/magic/__init__.pyi index 542fb420..0e375881 100644 --- a/magic/__init__.pyi +++ b/magic/__init__.pyi @@ -11,7 +11,7 @@ class Magic: flags: int = ... cookie: Any = ... lock: threading.Lock = ... - def __init__(self, mime: bool = ..., magic_file: Optional[Any] = ..., mime_encoding: bool = ..., keep_going: bool = ..., uncompress: bool = ..., raw: bool = ..., extension: bool = ..., follow_symlinks: bool = ...) -> None: ... + def __init__(self, mime: bool = ..., magic_file: Optional[Any] = ..., mime_encoding: bool = ..., keep_going: bool = ..., uncompress: bool = ..., raw: bool = ..., extension: bool = ..., follow_symlinks: bool = ..., check_tar: bool = ..., check_soft: bool = ..., check_apptype: bool = ..., check_elf: bool = ..., check_text: bool = ..., check_encoding: bool = ..., check_json: bool = ..., check_simh: bool = ...) -> None: ... def from_buffer(self, buf: Union[bytes, str]) -> Text: ... def from_file(self, filename: Union[bytes, str, PathLike]) -> Text: ... def from_descriptor(self, fd: int, mime: bool = ...) -> Text: ... @@ -74,10 +74,16 @@ MAGIC_NO_CHECK_TAR: int MAGIC_NO_CHECK_SOFT: int MAGIC_NO_CHECK_APPTYPE: int MAGIC_NO_CHECK_ELF: int +MAGIC_NO_CHECK_TEXT: int MAGIC_NO_CHECK_ASCII: int MAGIC_NO_CHECK_TROFF: int MAGIC_NO_CHECK_FORTRAN: int +MAGIC_NO_CHECK_CDF: int +MAGIC_NO_CHECK_CSV: int MAGIC_NO_CHECK_TOKENS: int +MAGIC_NO_CHECK_ENCODING: int +MAGIC_NO_CHECK_JSON: int +MAGIC_NO_CHECK_SIMH: int MAGIC_PARAM_INDIR_MAX: int MAGIC_PARAM_NAME_MAX: int MAGIC_PARAM_ELF_PHNUM_MAX: int From 54d86fd61b5d655e6976115a786c77d646e50341 Mon Sep 17 00:00:00 2001 From: Robert Scott Date: Sat, 7 Oct 2023 18:42:23 +0100 Subject: [PATCH 45/78] python-magic tests: add test files for elf and json, use to test flags elf-NetBSD-x86_64-echo is from https://github.com/JonathanSalwan/binary-samples under an MIT license incidentally this exposes that the ELF builtin magic detector only works on files, falling back to the soft magic for buffers --- test/python_magic_test.py | 88 +++++++++++++++++++++++++++ test/testdata/elf-NetBSD-x86_64-echo | Bin 0 -> 9552 bytes test/testdata/test.json | 7 +++ 3 files changed, 95 insertions(+) create mode 100644 test/testdata/elf-NetBSD-x86_64-echo create mode 100644 test/testdata/test.json diff --git a/test/python_magic_test.py b/test/python_magic_test.py index 41ed4af7..7ead8dd6 100755 --- a/test/python_magic_test.py +++ b/test/python_magic_test.py @@ -96,6 +96,10 @@ def test_mime_types(self): self.assert_values( m, { + "elf-NetBSD-x86_64-echo": ( + "application/x-pie-executable", + "application/x-sharedlib", + ), "magic._pyc_": ( "application/octet-stream", "text/x-bytecode.python", @@ -107,7 +111,9 @@ def test_mime_types(self): "text.txt": "text/plain", b"\xce\xbb".decode("utf-8"): "text/plain", b"\xce\xbb": "text/plain", + "test.json": "application/json", }, + buf_equals_file=False, ) finally: os.unlink(dest) @@ -119,6 +125,88 @@ def test_descriptions(self): self.assert_values( m, { + "elf-NetBSD-x86_64-echo": ( + "ELF 64-bit LSB shared object, x86-64, version 1 (SYSV)", + "ELF 64-bit LSB pie executable, x86-64, version 1 (SYSV), dynamically linked, interpreter /libexec/ld.elf_so, for NetBSD 8.0, not stripped", + ), + "magic._pyc_": "python 2.4 byte-compiled", + "test.pdf": ( + "PDF document, version 1.2", + "PDF document, version 1.2, 2 pages", + "PDF document, version 1.2, 2 page(s)", + ), + "test.gz": ( + 'gzip compressed data, was "test", from Unix, last ' + "modified: Sun Jun 29 01:32:52 2008", + 'gzip compressed data, was "test", last modified' + ": Sun Jun 29 01:32:52 2008, from Unix", + 'gzip compressed data, was "test", last modified' + ": Sun Jun 29 01:32:52 2008, from Unix, original size 15", + 'gzip compressed data, was "test", ' + "last modified: Sun Jun 29 01:32:52 2008, " + "from Unix, original size modulo 2^32 15", + 'gzip compressed data, was "test", last modified' + ": Sun Jun 29 01:32:52 2008, from Unix, truncated", + ), + "text.txt": "ASCII text", + "test.snappy.parquet": ("Apache Parquet", "Par archive data"), + "test.json": "JSON text data", + }, + buf_equals_file=False, + ) + finally: + del os.environ["TZ"] + + def test_descriptions_no_soft(self): + m = magic.Magic(check_soft=False) + self.assert_values( + m, + { + "elf-NetBSD-x86_64-echo": ( + "data", + "ELF 64-bit LSB pie executable, x86-64, version 1 (SYSV), dynamically linked, interpreter /libexec/ld.elf_so, for NetBSD 8.0, not stripped", + ), + "magic._pyc_": "data", + "test.pdf": "ASCII text", + "test.gz": "data", + "text.txt": "ASCII text", + "test.snappy.parquet": "data", + "test.json": "JSON text data", + }, + buf_equals_file=False, + ) + + def test_descriptions_no_elf(self): + m = magic.Magic(check_elf=False) + self.assert_values( + m, + { + "elf-NetBSD-x86_64-echo": "ELF 64-bit LSB shared object, x86-64, version 1 (SYSV)", + }, + buf_equals_file=True, + ) + + def test_descriptions_no_json(self): + m = magic.Magic(check_elf=False) + self.assert_values( + m, + { + "test.json": "data", + }, + buf_equals_file=True, + ) + + def test_descriptions_no_json(self): + m = magic.Magic(check_json=False) + os.environ["TZ"] = "UTC" # To get last modified date of test.gz in UTC + try: + self.assert_values( + m, + { + "elf-NetBSD-x86_64-echo": ( + "ELF 64-bit LSB shared object, x86-64, version 1 (SYSV)", + "ELF 64-bit LSB pie executable, x86-64, version 1 (SYSV), dynamically linked, interpreter /libexec/ld.elf_so, for NetBSD 8.0, not stripped", + ), "magic._pyc_": "python 2.4 byte-compiled", "test.pdf": ( "PDF document, version 1.2", diff --git a/test/testdata/elf-NetBSD-x86_64-echo b/test/testdata/elf-NetBSD-x86_64-echo new file mode 100644 index 0000000000000000000000000000000000000000..74affab97af982e9c9aee4ee650247ece95813ba GIT binary patch literal 9552 zcmd^Fdw5jU5kH#^VMRvy zy>sSwX6DQ}_w0F0_(^4TRf@sDh>UD5Ln&;EAtj>1JV6wIm9Ro)g7-vr9!mpx7{_Au zB{2rIF$xRU#D!aCgz8jONr~y{`Dg(XrwEZ_dqa_YR;-3NV-g3@4r9$p=OS>47l^MH zaXdru2s;$fu1?t12|MB)=opAN^$+`o&lurP&!?ji5OF>hAlDsIU0CiNiM6Nl>=?|5 zlidPgH&p*V6mb$4BFpu88{KVgXRgm>ar>I=N`SH9&<6TNof@}Vwy**Url)fe!w0`z zontY)8Owwxii@Jtx-u+GnJz|AlM2e9)RS`Ou8er_BBzhww~9pK5v@OF-8u|8cq z_X0i%9z8t3skC2 z*lHDX`&W2_0Y3=NHU~!I@OiI;2JFcrP*yg!HZg}93m&@zv0*>lE{Ey>vqlA5cLWjg zyZ8#h*oPSsj7aQvg`Czu{e0db#;Zj)A4u*KesGPzwSwq?2B-A{>mnLlEbZK6K!ek7 z94n4#a9nSQvP3-i^DqzyT+-mUo)G0|aP3dZs=;y1Au811+NGjIgNr4do5&hGlbisr z)8N{9yjX+ddP%fIgX4OuCpq-FDdK@mNDgf?ZDlN4Fv=vc=#(+=&N#mWN+^5Ktp7k1 zXvzqbaftf+xSWDA?qdC6E*~EbG7eS$Gh9A`GJeYbja>c=Wn7~Bd%65E%4sNfaQP#Y zaY*}Hxcm;vxE%H`;qqH3pMi26moEglsW&<$Pw=7yE{8rDz&^@dAIM!h<*twV!ICvB zSofZK!RCqF)!FaE zxV?&Fw*?2t7LNS!WHh=32R9OeQfO7GK~3K*fh_l&y$ci%?}~f|6_KyuCHEB902JAS zzzleGUvHAT8zAER$O$m%dNXn#R~xng!hpb!A?q^rJ7L^5<0cDEe*hl(+YvY~224i4 z13*Xdi#&S6)`i=LJqbkvbZGA|2|6@V?k-*kNMs%NKVe)dIPNyeonh4!?UO@WO^0F7 zi+?7Rdy02roR@*}m0LTCmkhdX5N><8=4iR6AQcek*e}tb5WRF}3VZY^yntry?8>ub4tZp8gB9It{>7+PC2GK))B!)Udd z;M=HUHsgGS2JV9TqYr1Sei+0`+ZS^8#h*aes(T)~8|%CCQCSAP<>)6I$wfcVTRAj` zsRz0{9|WyX#L&eHo#D1pxx15F_S6~Vt~;@~ok=l2tuIz~excUNJ+B}f`OQD0(Qi5* zg<^@C9h(6M0h-ep2I$5Ys=GejB0(v#7r_1?d>-gV7|L?b)#lPyaK`l402+D{V=%Oh zOV;v;!?(#j&tY?0aA}J4qOa@03Irm)9vJw*^YvhGr;(byiUM0sY*njisIHc38<*KD z{c5mXY6-x8;|@w*zcjD59#qN$!C-5P`duo#$%=N^?YvIO=k;F;_AA^$#T)QrTg?vD z>4`6)7yH>11uP8XjdQYr;|E)EA{t!+ybAhJ0(>fjeJgMzS_2K&Er8)L8^(_uZt8_$ z`T^A8a}sQ`ao9@J&1+Ih)3ZXxigbzbbMp@YK6X4BeFiv7H&>-+{V3zim8K4M$+>eb z%+H>LFvj45=PsB6x#*`X-Q1H>mY%iNSeh=aNi9v!2@R`Aw?fVT1;H%hM4i>kgyT>l z5vRTRNDSuZe=O2DmG*QxU#AMa5?COR=&WxQg|ryV21Gqgpl=0EdvBJI@lK*8oL8L0 zzA46@04TGX<{_e39Tyg0B#Kjo^0)u0I!^F0Rwp&nqu4l5%4AQ_0_$C*@gfh1UGM zS<|EfOM%78vSW2ck~64UEeofwkZhKG)UxGTXXRRLk~OcWV0MwUKw1`X2bW5fZ7ouE zqSeLDKy!1zZ*g*C+YGdxnQMiAvm{$yk#$y4{?OK5(WVWpthv@gZdF)hwSAXWBisr6 z!jkvNnhOEvTP0gzQT{9lIC->grzb#R&K6Igy-HJQ7!tJu)T>1OkDYWG9se{uFf6KUz7s8Kx zBGk5hRKNcI+N~vHwr^Q^`NqR(Yd4iTCQo~+F*u^2`}OHL+ipIX8=m-B@9Na@vNGwl z`_0)od7b6=w`8TJbiV%geJ|g&V*WL&zTJQ0m-|-!d_+g{A67Ljaa{Y}dF8L0Cr@ha zd{$DPTf{!i>$|aZ$D(FA<#W#;H_rdtqHFgkM{jBV{B~unam}Gm%HQ+6aO|A3Hn_%K z=FRqg{OFOJonLQy^vap}yDJwT|G@H>2M4Zu?5b(1t-<#8^w!7kyS?hRmDe79f7#LZ zU)ivG(#AJ7*%$2Ed(Yg*_rG@hmEOQ3vn^Ts4o-RMr`OxZefw%dP0PHUliw`f+@bui zcjCisUlp}ao&3p>vgfULUcUZI@BSU>RiEy=qQXD-h7JGNtyF#U;}gn>`ThgjPxjv1 zwrR_%b*j2_-B#P^3VhtV*JrXBzCLh!#=cK)`Rv>m?pDVy-Znq$qD>Dxz3r##ZwSw- zJ8RM5-yd4FX18xoj(_t#8{dAU_3~%eUi#3hZ)D6m^HvZ4Rk+-ZtxI8sZ2anH(3c?y zCyxPUjD9_g?JpVed>oli;QDam68i|@$c?!B(Z-^wFak(}mc$5!K^kZf(Ce_c3=XVqJN3lXY1nS5L8A&-Qe(HQcQzK+LDUYFb zWyFkU()lo$_e<0t%;P2E=V--v*~rd~m$d~W8?P1PWg|Zc$HjQT$Z+;1Qa*^IVDO(8 z2P%H%1+0+ciMl9ZJ~LTfv;uH#-U4-z7QiE6K}X9#PL>yK1zghUqSysbGXK4Rn^Q8G zRrehq06bZ~bPoJPljTb<03635teekI57F++L*Q==f#(e2{|`J)ct$QXa0m{%0m z9>6E*bWx1JBVj>Dvq4TaZYPJp)A;zDnqM6ScxK8tM)^F7=X~BTU85jQMLcCBdtWH> zI~m|Ozmnx+XAa>9Z}yn)JtZG&uehS7bpEB~khq*z(_pWZ?Ntj(=U3Y0iUrI*ue!FZ zwAx-XlOTV|=ZFzm|0(MOkRYxOmH5lU_D)Ebd3$yqGs@vjstBs1w z;&s7sEfH#Qv@wgv0qJXttKARoVq6U}OVI6eASg;LJ}~7eXA2Z9O9LQ6HX7bMlWqwH zc#IaeN2Ja@E^tB;vpCg2Pyz2Tw#*5Y;2g6I44t7!c$nvn*_= zx!mrA47z2hzZH`8a6-Tg#i{f6zeR8*hN(KZ{nNfKIHu*agt}Uq3<7SkWJc!&;&iVh z8@ex(JX2>+=L+I3<$z&KB0Uh#b5OnmPB;l~ zu>#o<$Mcfb-XqRU9$}B?P%V)iVIM#NF={_7>|YZ0v`&$}-hMAsVEg)hlKn(76dZMt zi05j`yh6PGB--yHMS<%?c4!EDmpJ@6v`&-g4kW?QWQVXHOFt|Gf%Lzh84N-nK48ZZ{y>?-LuB1n5L~==;%k#XmeIYd Date: Sat, 11 May 2024 08:19:57 +0200 Subject: [PATCH 46/78] Travis CI: Test on Python 3.13 beta The Python 3.13 release notes mention `python-magic` as one of the alternatives for `imghdr` which was removed from the Standard Library so let's ensure that its tests pass on Python 3.13 beta. https://www.python.org/downloads/release/python-3130b1/ May raise `ModuleNotFoundError: No module named 'imghdr'` because Python 3.13 removes it from the Standard Library. * https://docs.python.org/3/library/imghdr.html > imghdr: use the projects [filetype](https://pypi.org/project/filetype/), [puremagic](https://pypi.org/project/puremagic/), or [python-magic](https://pypi.org/project/python-magic/) instead. (Contributed by Victor Stinner in [gh-104773](https://github.com/python/cpython/issues/104773).) https://docs.python.org/3.13/whatsnew/3.13.html#pep-594-dead-batteries-and-other-module-removals --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index c83c031b..08c3d0e6 100644 --- a/.travis.yml +++ b/.travis.yml @@ -12,6 +12,7 @@ python: - "3.10" - "3.11" - "3.12" + - "3.13" install: - pip install coverage coveralls codecov From 7ee418053ef4d40dcfc1abd61b39bc8bcf0399a8 Mon Sep 17 00:00:00 2001 From: Christian Clauss Date: Thu, 16 May 2024 06:35:14 +0200 Subject: [PATCH 47/78] Delete .travis.yml As discussed at https://github.com/ahupp/python-magic/pull/317#issuecomment-2111634995 this file is useless without a paid subscription to Travis CI. GitHub Actions is free to open source projects but Travis CI is not. * #318 --- .travis.yml | 26 -------------------------- 1 file changed, 26 deletions(-) delete mode 100644 .travis.yml diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index 08c3d0e6..00000000 --- a/.travis.yml +++ /dev/null @@ -1,26 +0,0 @@ -language: python -dist: xenial -cache: pip - -python: - - "2.7" - - "3.5" - - "3.6" - - "3.7" - - "3.8" - - "3.9" - - "3.10" - - "3.11" - - "3.12" - - "3.13" - -install: - - pip install coverage coveralls codecov - - pip install . - -script: - - LC_ALL=en_US.UTF-8 coverage run -m unittest test - -after_success: - - coveralls - - codecov From e578995cdb6a2612a6b8fff65bf631105b3ee55d Mon Sep 17 00:00:00 2001 From: Christian Clauss Date: Sat, 11 May 2024 08:31:58 +0200 Subject: [PATCH 48/78] GitHub Action to replace Travis CI --- .github/workflows/ci.yml | 31 +++++++++++++++++++++++++++++++ test/python_magic_test.py | 18 ++++++++++++------ 2 files changed, 43 insertions(+), 6 deletions(-) create mode 100644 .github/workflows/ci.yml diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 00000000..def9b2ec --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,31 @@ +name: ci +on: [push, pull_request] +jobs: + ci: + strategy: + fail-fast: false + matrix: + os: ['ubuntu-latest'] + python-version: ['3.8', '3.9', '3.10', '3.11', '3.12', '3.13'] + include: + - os: macos-latest + python-version: '3.13' + # - os: windows-latest # TODO: Fix the Windows test that runs in an infinite loop. + # python-version: '3.13' + runs-on: ${{ matrix.os }} + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + allow-prereleases: true + - run: pip install --upgrade pip + - run: pip install --upgrade pytest + - run: pip install --editable . + - if: runner.os == 'macOS' + run: brew install libmagic + - if: runner.os == 'Windows' + run: pip install python-magic-bin + - run: LC_ALL=en_US.UTF-8 pytest + shell: bash + timeout-minutes: 15 # Limit Windows infinite loop. diff --git a/test/python_magic_test.py b/test/python_magic_test.py index 7ead8dd6..633fcaba 100755 --- a/test/python_magic_test.py +++ b/test/python_magic_test.py @@ -1,5 +1,11 @@ -import tempfile import os +import os.path +import shutil +import sys +import tempfile +import unittest + +import pytest # for output which reports a local time os.environ["TZ"] = "GMT" @@ -9,12 +15,8 @@ # necessary for some tests raise Exception("must run `export LC_ALL=en_US.UTF-8` before running test suite") -import shutil -import os.path -import unittest - import magic -import sys + # magic_descriptor is broken (?) in centos 7, so don't run those tests SKIP_FROM_DESCRIPTOR = bool(os.environ.get("SKIP_FROM_DESCRIPTOR")) @@ -118,6 +120,8 @@ def test_mime_types(self): finally: os.unlink(dest) + # TODO: Fix this failing test on Ubuntu + @pytest.mark.skipif(sys.platform == "linux", reason="'JSON data' not found") def test_descriptions(self): m = magic.Magic() os.environ["TZ"] = "UTC" # To get last modified date of test.gz in UTC @@ -157,6 +161,8 @@ def test_descriptions(self): finally: del os.environ["TZ"] + # TODO: Fix this failing test on Ubuntu + @pytest.mark.skipif(sys.platform == "linux", reason="'JSON data' not found") def test_descriptions_no_soft(self): m = magic.Magic(check_soft=False) self.assert_values( From ab1b2a4707776397116aad929e443c4a4e657b3c Mon Sep 17 00:00:00 2001 From: Christian Clauss Date: Thu, 16 May 2024 17:37:31 +0200 Subject: [PATCH 49/78] Update ci.yml --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index def9b2ec..9c4e4c9a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -10,7 +10,7 @@ jobs: include: - os: macos-latest python-version: '3.13' - # - os: windows-latest # TODO: Fix the Windows test that runs in an infinite loop. + # - os: windows-latest # TODO: Fix the Windows test that runs in an infinite loop # python-version: '3.13' runs-on: ${{ matrix.os }} steps: From aa49677fbec9a4fc96b54e42c2517a2118d66db8 Mon Sep 17 00:00:00 2001 From: Adam Hupp Date: Thu, 16 May 2024 09:06:59 -0700 Subject: [PATCH 50/78] Update README.md Remove travis build badge --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index fb1bc0eb..01b7a8f5 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # python-magic [![PyPI version](https://badge.fury.io/py/python-magic.svg)](https://badge.fury.io/py/python-magic) -[![Build Status](https://travis-ci.org/ahupp/python-magic.svg?branch=master)](https://travis-ci.org/ahupp/python-magic) [![Join the chat at https://gitter.im/ahupp/python-magic](https://badges.gitter.im/ahupp/python-magic.svg)](https://gitter.im/ahupp/python-magic?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge) +[![Join the chat at https://gitter.im/ahupp/python-magic](https://badges.gitter.im/ahupp/python-magic.svg)](https://gitter.im/ahupp/python-magic?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge) python-magic is a Python interface to the libmagic file type identification library. libmagic identifies file types by checking From eae08a39f401e4673f2a72a44bbd9f9424e56a47 Mon Sep 17 00:00:00 2001 From: Christian Clauss Date: Thu, 16 May 2024 18:16:22 +0200 Subject: [PATCH 51/78] README.md: Add a badge for GitHub Actions https://docs.github.com/en/actions/monitoring-and-troubleshooting-workflows/adding-a-workflow-status-badge [![Tests](https://github.com/ahupp/python-magic/actions/workflows/ci.yml/badge.svg)](https://github.com/ahupp/python-magic/actions/workflows/ci.yml) --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 01b7a8f5..40c3737c 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,6 @@ # python-magic [![PyPI version](https://badge.fury.io/py/python-magic.svg)](https://badge.fury.io/py/python-magic) +[![Tests](https://github.com/ahupp/python-magic/actions/workflows/ci.yml/badge.svg)](https://github.com/ahupp/python-magic/actions/workflows/ci.yml) [![Join the chat at https://gitter.im/ahupp/python-magic](https://badges.gitter.im/ahupp/python-magic.svg)](https://gitter.im/ahupp/python-magic?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge) python-magic is a Python interface to the libmagic file type From fc7ebc0d6f9e34d422dbf49aec2382e4a5bde379 Mon Sep 17 00:00:00 2001 From: Christian Clauss Date: Thu, 16 May 2024 18:17:01 +0200 Subject: [PATCH 52/78] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 40c3737c..02374d7a 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # python-magic [![PyPI version](https://badge.fury.io/py/python-magic.svg)](https://badge.fury.io/py/python-magic) -[![Tests](https://github.com/ahupp/python-magic/actions/workflows/ci.yml/badge.svg)](https://github.com/ahupp/python-magic/actions/workflows/ci.yml) +[![ci](https://github.com/ahupp/python-magic/actions/workflows/ci.yml/badge.svg)](https://github.com/ahupp/python-magic/actions/workflows/ci.yml) [![Join the chat at https://gitter.im/ahupp/python-magic](https://badges.gitter.im/ahupp/python-magic.svg)](https://gitter.im/ahupp/python-magic?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge) python-magic is a Python interface to the libmagic file type From 1217005672363871bad24cffa4d87bb2347dac16 Mon Sep 17 00:00:00 2001 From: ddelange <14880945+ddelange@users.noreply.github.com> Date: Wed, 22 May 2024 14:10:24 +0200 Subject: [PATCH 53/78] Fix typos discovered by codespell --- CHANGELOG | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGELOG b/CHANGELOG index b6766f94..a8370c68 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -31,7 +31,7 @@ Changes to 0.4.24: Changes to 0.4.23 -- Include a `py.typed` sentinal to enable type checking +- Include a `py.typed` sentinel to enable type checking - Improve fix for attribute error during destruction - Cleanup library loading logic - Add new homebrew library dir for OSX @@ -76,7 +76,7 @@ Changes in 0.4.16 and 0.4.17 - add MAGIC_MIME_TYPE constant, use that in preference to MAGIC_MIME internally. This sets up for a breaking change in a future major version bump where - MAGIC_MIME will change to mathch magic.h. + MAGIC_MIME will change to match magic.h. - add magic.version() function to return library version - add setparam/getparam to control internal behavior - increase internal limits with setparam to prevent spurious error on some jpeg files From cf210655e2b4a176f998909611ba936b064afc26 Mon Sep 17 00:00:00 2001 From: Christian Clauss Date: Wed, 22 May 2024 16:15:04 +0200 Subject: [PATCH 54/78] Clean up loader.py --- magic/loader.py | 78 ++++++++++++++++++++++++++++++------------------- 1 file changed, 48 insertions(+), 30 deletions(-) diff --git a/magic/loader.py b/magic/loader.py index 228a35cb..145fcfa9 100644 --- a/magic/loader.py +++ b/magic/loader.py @@ -4,47 +4,65 @@ import glob import os.path -def _lib_candidates(): - yield find_library('magic') +def _lib_candidates_linux(): + """Yield possible libmagic library names on Linux. + + This is necessary because alpine is bad + """ + yield "libmagic.so.1" - if sys.platform == 'darwin': +def _lib_candidates_macos(): + """Yield possible libmagic library names on macOS.""" paths = [ - '/opt/local/lib', - '/usr/local/lib', - '/opt/homebrew/lib', - ] + glob.glob('/usr/local/Cellar/libmagic/*/lib') + "/opt/homebrew/lib", + "/opt/local/lib", + "/usr/local/lib", + ] + glob.glob("/usr/local/Cellar/libmagic/*/lib") + for path in paths: + yield os.path.join(path, "libmagic.dylib") - for i in paths: - yield os.path.join(i, 'libmagic.dylib') - elif sys.platform in ('win32', 'cygwin'): +def _lib_candidates_windows(): + """Yield possible libmagic library names on Windows.""" + prefixes = ( + "libmagic", + "magic1", + "magic-1", + "cygmagic-1", + "libmagic-1", + "msys-magic-1", + ) + for prefix in prefixes: + # find_library searches in %PATH% but not the current directory, + # so look for both + yield "./%s.dll" % (prefix,) + yield find_library(prefix) - prefixes = ['libmagic', 'magic1', 'magic-1', 'cygmagic-1', 'libmagic-1', 'msys-magic-1'] - for i in prefixes: - # find_library searches in %PATH% but not the current directory, - # so look for both - yield './%s.dll' % (i,) - yield find_library(i) +def _lib_candidates(): + yield find_library("magic") - elif sys.platform == 'linux': - # This is necessary because alpine is bad - yield 'libmagic.so.1' + func = { + "cygwin": _lib_candidates_windows, + "darwin": _lib_candidates_macos, + "linux": _lib_candidates_linux, + "win32": _lib_candidates_windows, + }[sys.platform] + # When we drop legacy Python, we can just `yield from func()` + for path in func(): + yield path def load_lib(): + for lib in _lib_candidates(): + # find_library returns None when lib not found + if lib: + try: + return ctypes.CDLL(lib) + except OSError: + pass - for lib in _lib_candidates(): - # find_library returns None when lib not found - if lib is None: - continue - try: - return ctypes.CDLL(lib) - except OSError: - pass - else: # It is better to raise an ImportError since we are importing magic module - raise ImportError('failed to find libmagic. Check your installation') - + raise ImportError("failed to find libmagic. Check your installation") From 0a2fda39ba235360b6a8ec6f1d531b08ab306b25 Mon Sep 17 00:00:00 2001 From: Adam Hupp Date: Sun, 26 May 2024 01:01:34 -0700 Subject: [PATCH 55/78] handle unknown platforms gracefully in loader.py --- magic/loader.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/magic/loader.py b/magic/loader.py index 145fcfa9..33bf4d43 100644 --- a/magic/loader.py +++ b/magic/loader.py @@ -49,7 +49,9 @@ def _lib_candidates(): "darwin": _lib_candidates_macos, "linux": _lib_candidates_linux, "win32": _lib_candidates_windows, - }[sys.platform] + }.get(sys.platform) + if func is None: + raise ImportError("python-magic: Unsupported platform: " + sys.platform) # When we drop legacy Python, we can just `yield from func()` for path in func(): yield path @@ -65,4 +67,4 @@ def load_lib(): pass # It is better to raise an ImportError since we are importing magic module - raise ImportError("failed to find libmagic. Check your installation") + raise ImportError("python-magic: failed to find libmagic. Check your installation") From 4b776d72729b4a1cf87f6939089cacdeff171972 Mon Sep 17 00:00:00 2001 From: Adam Hupp Date: Sun, 26 May 2024 01:11:45 -0700 Subject: [PATCH 56/78] rename no_json test to avoid duplicate function definitions --- test/python_magic_test.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test/python_magic_test.py b/test/python_magic_test.py index 633fcaba..7ad15c8b 100755 --- a/test/python_magic_test.py +++ b/test/python_magic_test.py @@ -202,7 +202,8 @@ def test_descriptions_no_json(self): buf_equals_file=True, ) - def test_descriptions_no_json(self): + def test_descriptions_no_json_unchanged(self): + # verify non-json results are unchanged m = magic.Magic(check_json=False) os.environ["TZ"] = "UTC" # To get last modified date of test.gz in UTC try: From 339eac0450f2fc87b48a9c62b21a874a9464bb05 Mon Sep 17 00:00:00 2001 From: Adam Hupp Date: Sun, 26 May 2024 01:15:00 -0700 Subject: [PATCH 57/78] smartos support, adapted from #132 --- README.md | 4 ++++ magic/loader.py | 1 + 2 files changed, 5 insertions(+) diff --git a/README.md b/README.md index 02374d7a..5fce8c1f 100644 --- a/README.md +++ b/README.md @@ -75,6 +75,10 @@ pip install python-magic-bin - When using Homebrew: `brew install libmagic` - When using macports: `port install file` +### SmartOS: +- Install libmagic for source https://github.com/threatstack/libmagic/ +- Depending on your ./configure --prefix settings set your LD_LIBRARY_PATH to /lib + ### Troubleshooting - 'MagicException: could not find any magic files!': some diff --git a/magic/loader.py b/magic/loader.py index 33bf4d43..3179c05b 100644 --- a/magic/loader.py +++ b/magic/loader.py @@ -49,6 +49,7 @@ def _lib_candidates(): "darwin": _lib_candidates_macos, "linux": _lib_candidates_linux, "win32": _lib_candidates_windows, + "sunos5": _lib_candidates_linux, }.get(sys.platform) if func is None: raise ImportError("python-magic: Unsupported platform: " + sys.platform) From a9e627644efcee42b70cf7634f0ec6d2bab318c8 Mon Sep 17 00:00:00 2001 From: Adam Hupp Date: Sun, 26 May 2024 01:21:47 -0700 Subject: [PATCH 58/78] log warning on ctypes load error, adapted from #279 --- magic/loader.py | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/magic/loader.py b/magic/loader.py index 3179c05b..e6edc7bf 100644 --- a/magic/loader.py +++ b/magic/loader.py @@ -3,7 +3,9 @@ import sys import glob import os.path +import logging +logger = logging.getLogger(__name__) def _lib_candidates_linux(): """Yield possible libmagic library names on Linux. @@ -61,11 +63,15 @@ def _lib_candidates(): def load_lib(): for lib in _lib_candidates(): # find_library returns None when lib not found - if lib: - try: - return ctypes.CDLL(lib) - except OSError: - pass + if lib is None: + continue + if not os.path.exists(lib): + continue + + try: + return ctypes.CDLL(lib) + except OSError: + logger.warning("Failed to load: " + lib, exc_info=True) # It is better to raise an ImportError since we are importing magic module raise ImportError("python-magic: failed to find libmagic. Check your installation") From 067399b1ae104975fe83b5162dd3b83788cdfdd0 Mon Sep 17 00:00:00 2001 From: Adam Hupp Date: Sun, 9 Jun 2024 17:31:24 -0700 Subject: [PATCH 59/78] Update README.md --- README.md | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index 5fce8c1f..010cc8f2 100644 --- a/README.md +++ b/README.md @@ -62,19 +62,13 @@ that must be installed as well: sudo apt-get install libmagic1 ``` -### Windows - -You'll need DLLs for libmagic. @julian-r maintains a pypi package with the DLLs, you can fetch it with: - -``` -pip install python-magic-bin -``` - ### OSX - When using Homebrew: `brew install libmagic` - When using macports: `port install file` +If python-magic fails to load the library it may be in a non-standard location, in which case you can set the environment variable `DYLD_LIBRARY_PATH` to point to it. + ### SmartOS: - Install libmagic for source https://github.com/threatstack/libmagic/ - Depending on your ./configure --prefix settings set your LD_LIBRARY_PATH to /lib From 42980e5f16029f852b73c1ed14fcf8955b9a140d Mon Sep 17 00:00:00 2001 From: Adam Hupp Date: Sun, 26 May 2024 18:06:37 -0700 Subject: [PATCH 60/78] simplify tests into something more delarative --- test/python_magic_test.py | 383 +++++++++++++++----------------------- 1 file changed, 150 insertions(+), 233 deletions(-) diff --git a/test/python_magic_test.py b/test/python_magic_test.py index 7ad15c8b..b5577620 100755 --- a/test/python_magic_test.py +++ b/test/python_magic_test.py @@ -1,3 +1,5 @@ +from dataclasses import dataclass +from enum import Enum import os import os.path import shutil @@ -17,11 +19,140 @@ import magic +@dataclass +class TestFile: + file_name: str + mime_results: list[str] + text_results: list[str] + no_check_elf_results: list[str] | None + buf_equals_file: bool = True # magic_descriptor is broken (?) in centos 7, so don't run those tests SKIP_FROM_DESCRIPTOR = bool(os.environ.get("SKIP_FROM_DESCRIPTOR")) +COMMON_PLAIN = [ + {}, + {"check_soft": True}, + {"check_soft": False}, + {"check_json": True}, + {"check_json": False}, +] + +NO_SOFT = {"check_soft": False} + +COMMON_MIME = [{"mime": True, **k} for k in COMMON_PLAIN] + +CASES = { + "magic._pyc_": [ + (COMMON_MIME, [ + "application/octet-stream", + "text/x-bytecode.python", + "application/x-bytecode.python", + ]), + (COMMON_PLAIN, ["python 2.4 byte-compiled"]), + (NO_SOFT, ["data"]), + ], + "test.pdf": [ + (COMMON_MIME, ["application/pdf"]), + (COMMON_PLAIN, [ + "PDF document, version 1.2", + "PDF document, version 1.2, 2 pages", + "PDF document, version 1.2, 2 page(s)", + ]), + (NO_SOFT, ["ASCII text"]), + ], + "test.gz": [ + (COMMON_MIME, ["application/gzip", "application/x-gzip"]), + (COMMON_PLAIN, [ + 'gzip compressed data, was "test", from Unix, last modified: Sun Jun 29 01:32:52 2008', + 'gzip compressed data, was "test", last modified: Sun Jun 29 01:32:52 2008, from Unix', + 'gzip compressed data, was "test", last modified: Sun Jun 29 01:32:52 2008, from Unix, original size 15', + 'gzip compressed data, was "test", last modified: Sun Jun 29 01:32:52 2008, from Unix, original size modulo 2^32 15', + 'gzip compressed data, was "test", last modified: Sun Jun 29 01:32:52 2008, from Unix, truncated', + ]), + ({"extension": True}, [ + # some versions return '' for the extensions of a gz file, + # including w/ the command line. Who knows... + "gz/tgz/tpz/zabw/svgz/adz/kmy/xcfgz", + "gz/tgz/tpz/zabw/svgz", + "", + "???", + ]), + (NO_SOFT, ["data"]), + ], + "test.snappy.parquet": [ + (COMMON_MIME, ["application/octet-stream"]), + (COMMON_PLAIN, ["Apache Parquet", "Par archive data"]), + (NO_SOFT, ["data"]), + ], + "test.json": [ + # TODO: soft, no_json + (COMMON_MIME, ["application/json"]), + (COMMON_PLAIN, ["JSON text data"]), + ({"mime": True, "check_json": False}, [ + "data", + ]), + (NO_SOFT, ["JSON text data"]) + ], + "elf-NetBSD-x86_64-echo": [ + # TODO: soft, no elf + (COMMON_PLAIN, [ + "ELF 64-bit LSB shared object, x86-64, version 1 (SYSV)", + "ELF 64-bit LSB pie executable, x86-64, version 1 (SYSV), dynamically linked, interpreter /libexec/ld.elf_so, for NetBSD 8.0, not stripped", + ]), + (COMMON_MIME, [ + "application/x-pie-executable", + "application/x-sharedlib", + ]), + ({"check_elf": False}, [ + "ELF 64-bit LSB shared object, x86-64, version 1 (SYSV)", + ]), + # TODO: sometimes + # "ELF 64-bit LSB pie executable, x86-64, version 1 (SYSV), dynamically linked, interpreter /libexec/ld.elf_so, for NetBSD 8.0, not stripped", + + (NO_SOFT, ["data"]), + ], + "test.txt": [ + (COMMON_MIME, ["text/plain"]), + (COMMON_PLAIN, ["ASCII text"]), + ({"mime_encoding": True}, [ + "us-ascii", + ]), + (NO_SOFT, ["ASCII text"]), + ], + "text-iso8859-1.txt": [ + ({"mime_encoding": True}, [ + "iso-8859-1", + ]), + ], + b"\xce\xbb": [ + (COMMON_MIME, ["text/plain"]), + ], + "b\xce\xbb".decode("utf-8"): [ + (COMMON_MIME, ["text/plain"]), + ], + "name_use.jpg": [ + ({"extension": True}, [ + "jpeg/jpg/jpe/jfif" + ]), + ], + "keep-going.jpg": [ + (COMMON_MIME, [ + "image/jpeg" + ]), + ({"mime": True, "keep_going": True}, [ + "image/jpeg\\012- application/octet-stream", + ]) + ], + "test.py": [ + (COMMON_MIME, [ + "text/x-python", + "text/x-script.python", + ]) + ] +} + class MagicTest(unittest.TestCase): TESTDATA_DIR = os.path.abspath(os.path.join(os.path.dirname(__file__), "testdata")) @@ -34,26 +165,6 @@ def test_version(self): def test_fs_encoding(self): self.assertEqual("utf-8", sys.getfilesystemencoding().lower()) - def assert_values(self, m, expected_values, buf_equals_file=True): - for filename, expected_value in expected_values.items(): - try: - filename = os.path.join(self.TESTDATA_DIR, filename) - except TypeError: - filename = os.path.join(self.TESTDATA_DIR.encode("utf-8"), filename) - - if type(expected_value) is not tuple: - expected_value = (expected_value,) - - with open(filename, "rb") as f: - buf_value = m.from_buffer(f.read()) - - file_value = m.from_file(filename) - - if buf_equals_file: - self.assertEqual(buf_value, file_value) - - for value in (buf_value, file_value): - self.assertIn(value, expected_value) def test_from_file_str_and_bytes(self): filename = os.path.join(self.TESTDATA_DIR, "test.pdf") @@ -63,203 +174,34 @@ def test_from_file_str_and_bytes(self): "application/pdf", magic.from_file(filename.encode("utf-8"), mime=True) ) - def test_from_descriptor_str_and_bytes(self): - if SKIP_FROM_DESCRIPTOR: - self.skipTest("magic_descriptor is broken in this version of libmagic") - - filename = os.path.join(self.TESTDATA_DIR, "test.pdf") - with open(filename) as f: - self.assertEqual( - "application/pdf", magic.from_descriptor(f.fileno(), mime=True) - ) - self.assertEqual( - "application/pdf", magic.from_descriptor(f.fileno(), mime=True) - ) - - def test_from_buffer_str_and_bytes(self): - if SKIP_FROM_DESCRIPTOR: - self.skipTest("magic_descriptor is broken in this version of libmagic") - m = magic.Magic(mime=True) - - self.assertTrue( - m.from_buffer('#!/usr/bin/env python\nprint("foo")') - in ("text/x-python", "text/x-script.python") - ) - self.assertTrue( - m.from_buffer(b'#!/usr/bin/env python\nprint("foo")') - in ("text/x-python", "text/x-script.python") - ) - def test_mime_types(self): + def test_all_cases(self): + # TODO: + # * MAGIC_EXTENSION not supported + # * keep_going not supported + # * buffer checks dest = os.path.join(MagicTest.TESTDATA_DIR, b"\xce\xbb".decode("utf-8")) shutil.copyfile(os.path.join(MagicTest.TESTDATA_DIR, "lambda"), dest) + os.environ["TZ"] = "UTC" try: - m = magic.Magic(mime=True) - self.assert_values( - m, - { - "elf-NetBSD-x86_64-echo": ( - "application/x-pie-executable", - "application/x-sharedlib", - ), - "magic._pyc_": ( - "application/octet-stream", - "text/x-bytecode.python", - "application/x-bytecode.python", - ), - "test.pdf": "application/pdf", - "test.gz": ("application/gzip", "application/x-gzip"), - "test.snappy.parquet": "application/octet-stream", - "text.txt": "text/plain", - b"\xce\xbb".decode("utf-8"): "text/plain", - b"\xce\xbb": "text/plain", - "test.json": "application/json", - }, - buf_equals_file=False, - ) - finally: - os.unlink(dest) + for file_name, cases in CASES: + filename = os.path.join(self.TESTDATA_DIR, file_name) + for flags, outputs in cases: + m = magic.Magic(**flags) + with open(filename) as f: + self.assertIn(m.from_descriptor(f.fileno()), outputs) - # TODO: Fix this failing test on Ubuntu - @pytest.mark.skipif(sys.platform == "linux", reason="'JSON data' not found") - def test_descriptions(self): - m = magic.Magic() - os.environ["TZ"] = "UTC" # To get last modified date of test.gz in UTC - try: - self.assert_values( - m, - { - "elf-NetBSD-x86_64-echo": ( - "ELF 64-bit LSB shared object, x86-64, version 1 (SYSV)", - "ELF 64-bit LSB pie executable, x86-64, version 1 (SYSV), dynamically linked, interpreter /libexec/ld.elf_so, for NetBSD 8.0, not stripped", - ), - "magic._pyc_": "python 2.4 byte-compiled", - "test.pdf": ( - "PDF document, version 1.2", - "PDF document, version 1.2, 2 pages", - "PDF document, version 1.2, 2 page(s)", - ), - "test.gz": ( - 'gzip compressed data, was "test", from Unix, last ' - "modified: Sun Jun 29 01:32:52 2008", - 'gzip compressed data, was "test", last modified' - ": Sun Jun 29 01:32:52 2008, from Unix", - 'gzip compressed data, was "test", last modified' - ": Sun Jun 29 01:32:52 2008, from Unix, original size 15", - 'gzip compressed data, was "test", ' - "last modified: Sun Jun 29 01:32:52 2008, " - "from Unix, original size modulo 2^32 15", - 'gzip compressed data, was "test", last modified' - ": Sun Jun 29 01:32:52 2008, from Unix, truncated", - ), - "text.txt": "ASCII text", - "test.snappy.parquet": ("Apache Parquet", "Par archive data"), - "test.json": "JSON text data", - }, - buf_equals_file=False, - ) - finally: - del os.environ["TZ"] - - # TODO: Fix this failing test on Ubuntu - @pytest.mark.skipif(sys.platform == "linux", reason="'JSON data' not found") - def test_descriptions_no_soft(self): - m = magic.Magic(check_soft=False) - self.assert_values( - m, - { - "elf-NetBSD-x86_64-echo": ( - "data", - "ELF 64-bit LSB pie executable, x86-64, version 1 (SYSV), dynamically linked, interpreter /libexec/ld.elf_so, for NetBSD 8.0, not stripped", - ), - "magic._pyc_": "data", - "test.pdf": "ASCII text", - "test.gz": "data", - "text.txt": "ASCII text", - "test.snappy.parquet": "data", - "test.json": "JSON text data", - }, - buf_equals_file=False, - ) + self.assertIn(m.from_file(filename), outputs) - def test_descriptions_no_elf(self): - m = magic.Magic(check_elf=False) - self.assert_values( - m, - { - "elf-NetBSD-x86_64-echo": "ELF 64-bit LSB shared object, x86-64, version 1 (SYSV)", - }, - buf_equals_file=True, - ) - - def test_descriptions_no_json(self): - m = magic.Magic(check_elf=False) - self.assert_values( - m, - { - "test.json": "data", - }, - buf_equals_file=True, - ) + fname_bytes = filename.encode("utf-8") + self.assertIn(m.from_file(fname_bytes), outputs) - def test_descriptions_no_json_unchanged(self): - # verify non-json results are unchanged - m = magic.Magic(check_json=False) - os.environ["TZ"] = "UTC" # To get last modified date of test.gz in UTC - try: - self.assert_values( - m, - { - "elf-NetBSD-x86_64-echo": ( - "ELF 64-bit LSB shared object, x86-64, version 1 (SYSV)", - "ELF 64-bit LSB pie executable, x86-64, version 1 (SYSV), dynamically linked, interpreter /libexec/ld.elf_so, for NetBSD 8.0, not stripped", - ), - "magic._pyc_": "python 2.4 byte-compiled", - "test.pdf": ( - "PDF document, version 1.2", - "PDF document, version 1.2, 2 pages", - "PDF document, version 1.2, 2 page(s)", - ), - "test.gz": ( - 'gzip compressed data, was "test", from Unix, last ' - "modified: Sun Jun 29 01:32:52 2008", - 'gzip compressed data, was "test", last modified' - ": Sun Jun 29 01:32:52 2008, from Unix", - 'gzip compressed data, was "test", last modified' - ": Sun Jun 29 01:32:52 2008, from Unix, original size 15", - 'gzip compressed data, was "test", ' - "last modified: Sun Jun 29 01:32:52 2008, " - "from Unix, original size modulo 2^32 15", - 'gzip compressed data, was "test", last modified' - ": Sun Jun 29 01:32:52 2008, from Unix, truncated", - ), - "text.txt": "ASCII text", - "test.snappy.parquet": ("Apache Parquet", "Par archive data"), - }, - buf_equals_file=False, - ) + with open(file_name, "rb") as f: + buf_result = m.from_buffer(f.read(1024)) + self.assertIn(buf_result, outputs) finally: del os.environ["TZ"] - - def test_extension(self): - try: - m = magic.Magic(extension=True) - self.assert_values( - m, - { - # some versions return '' for the extensions of a gz file, - # including w/ the command line. Who knows... - "test.gz": ( - "gz/tgz/tpz/zabw/svgz/adz/kmy/xcfgz", - "gz/tgz/tpz/zabw/svgz", - "", - "???", - ), - "name_use.jpg": "jpeg/jpg/jpe/jfif", - }, - ) - except NotImplementedError: - self.skipTest("MAGIC_EXTENSION not supported in this version") + os.unlink(dest) def test_unicode_result_nonraw(self): m = magic.Magic(raw=False) @@ -280,15 +222,6 @@ def test_unicode_result_raw(self): else: raise unittest.SkipTest("Magic file doesn't return expected type.") - def test_mime_encodings(self): - m = magic.Magic(mime_encoding=True) - self.assert_values( - m, - { - "text-iso8859-1.txt": "iso-8859-1", - "text.txt": "us-ascii", - }, - ) def test_errors(self): m = magic.Magic() @@ -300,22 +233,6 @@ def test_errors(self): finally: del os.environ["MAGIC"] - def test_keep_going(self): - filename = os.path.join(self.TESTDATA_DIR, "keep-going.jpg") - - m = magic.Magic(mime=True) - self.assertEqual(m.from_file(filename), "image/jpeg") - - try: - # this will throw if you have an "old" version of the library - # I'm otherwise not sure how to query if keep_going is supported - magic.version() - m = magic.Magic(mime=True, keep_going=True) - self.assertEqual( - m.from_file(filename), "image/jpeg\\012- application/octet-stream" - ) - except NotImplementedError: - pass def test_rethrow(self): old = magic.magic_buffer From 36ecbf9866637badc22d0750fed4617dee4decf4 Mon Sep 17 00:00:00 2001 From: Adam Hupp Date: Mon, 5 Aug 2024 09:24:16 -0700 Subject: [PATCH 61/78] update magic/compat.py This pulls changes from https://github.com/file/file, commit 512840337ead1076519332d24fefcaa8fac36e06 --- .gitignore | 1 + magic/compat.py | 112 ++++++++++++++++++++++++++++++++++++++---------- 2 files changed, 90 insertions(+), 23 deletions(-) diff --git a/.gitignore b/.gitignore index 40c8c4eb..1f961bbb 100644 --- a/.gitignore +++ b/.gitignore @@ -11,3 +11,4 @@ pyvenv.cfg *.pyc *~ dist/ +.vscode/ diff --git a/magic/compat.py b/magic/compat.py index 07fad45a..6ab9400e 100644 --- a/magic/compat.py +++ b/magic/compat.py @@ -4,14 +4,12 @@ Python bindings for libmagic ''' -import ctypes - +import threading from collections import namedtuple from ctypes import * from ctypes.util import find_library - from . import loader _libraries = {} @@ -45,13 +43,19 @@ MAGIC_NO_CHECK_BUILTIN = NO_CHECK_BUILTIN = 4173824 +MAGIC_PARAM_INDIR_MAX = PARAM_INDIR_MAX = 0 +MAGIC_PARAM_NAME_MAX = PARAM_NAME_MAX = 1 +MAGIC_PARAM_ELF_PHNUM_MAX = PARAM_ELF_PHNUM_MAX = 2 +MAGIC_PARAM_ELF_SHNUM_MAX = PARAM_ELF_SHNUM_MAX = 3 +MAGIC_PARAM_ELF_NOTES_MAX = PARAM_ELF_NOTES_MAX = 4 +MAGIC_PARAM_REGEX_MAX = PARAM_REGEX_MAX = 5 +MAGIC_PARAM_BYTES_MAX = PARAM_BYTES_MAX = 6 + FileMagic = namedtuple('FileMagic', ('mime_type', 'encoding', 'name')) class magic_set(Structure): pass - - magic_set._fields_ = [] magic_t = POINTER(magic_set) @@ -103,6 +107,14 @@ class magic_set(Structure): _errno.restype = c_int _errno.argtypes = [magic_t] +_getparam = _libraries['magic'].magic_getparam +_getparam.restype = c_int +_getparam.argtypes = [magic_t, c_int, c_void_p] + +_setparam = _libraries['magic'].magic_setparam +_setparam.restype = c_int +_setparam.argtypes = [magic_t, c_int, c_void_p] + class Magic(object): def __init__(self, ms): @@ -228,29 +240,81 @@ def errno(self): """ return _errno(self._magic_t) + def getparam(self, param): + """ + Returns the param value if successful and -1 if the parameter + was unknown. + """ + v = c_int() + i = _getparam(self._magic_t, param, byref(v)) + if i == -1: + return -1 + return v.value + + def setparam(self, param, value): + """ + Returns 0 if successful and -1 if the parameter was unknown. + """ + v = c_int(value) + return _setparam(self._magic_t, param, byref(v)) + def open(flags): """ Returns a magic object on success and None on failure. Flags argument as for setflags. """ - return Magic(_open(flags)) + magic_t = _open(flags) + if magic_t is None: + return None + return Magic(magic_t) # Objects used by `detect_from_` functions -mime_magic = Magic(_open(MAGIC_MIME)) -mime_magic.load() -none_magic = Magic(_open(MAGIC_NONE)) -none_magic.load() +class error(Exception): + pass +class MagicDetect(object): + def __init__(self): + self.mime_magic = open(MAGIC_MIME) + if self.mime_magic is None: + raise error + if self.mime_magic.load() == -1: + self.mime_magic.close() + self.mime_magic = None + raise error + self.none_magic = open(MAGIC_NONE) + if self.none_magic is None: + self.mime_magic.close() + self.mime_magic = None + raise error + if self.none_magic.load() == -1: + self.none_magic.close() + self.none_magic = None + self.mime_magic.close() + self.mime_magic = None + raise error + + def __del__(self): + if self.mime_magic is not None: + self.mime_magic.close() + if self.none_magic is not None: + self.none_magic.close() + +threadlocal = threading.local() + +def _detect_make(): + v = getattr(threadlocal, "magic_instance", None) + if v is None: + v = MagicDetect() + setattr(threadlocal, "magic_instance", v) + return v def _create_filemagic(mime_detected, type_detected): - splat = mime_detected.split('; ') - mime_type = splat[0] - if len(splat) == 2: - mime_encoding = splat[1] - else: - mime_encoding = '' + try: + mime_type, mime_encoding = mime_detected.split('; ') + except ValueError: + raise ValueError(mime_detected) return FileMagic(name=type_detected, mime_type=mime_type, encoding=mime_encoding.replace('charset=', '')) @@ -261,9 +325,9 @@ def detect_from_filename(filename): Returns a `FileMagic` namedtuple. ''' - - return _create_filemagic(mime_magic.file(filename), - none_magic.file(filename)) + x = _detect_make() + return _create_filemagic(x.mime_magic.file(filename), + x.none_magic.file(filename)) def detect_from_fobj(fobj): @@ -273,8 +337,9 @@ def detect_from_fobj(fobj): ''' file_descriptor = fobj.fileno() - return _create_filemagic(mime_magic.descriptor(file_descriptor), - none_magic.descriptor(file_descriptor)) + x = _detect_make() + return _create_filemagic(x.mime_magic.descriptor(file_descriptor), + x.none_magic.descriptor(file_descriptor)) def detect_from_content(byte_content): @@ -283,5 +348,6 @@ def detect_from_content(byte_content): Returns a `FileMagic` namedtuple. ''' - return _create_filemagic(mime_magic.buffer(byte_content), - none_magic.buffer(byte_content)) + x = _detect_make() + return _create_filemagic(x.mime_magic.buffer(byte_content), + x.none_magic.buffer(byte_content)) From a3ed0862e67b50aaeafb7d14de9169b51efb3984 Mon Sep 17 00:00:00 2001 From: Adam Hupp Date: Tue, 18 Feb 2025 10:55:05 -0800 Subject: [PATCH 62/78] Unbreak various things * A merge to reduce error spam during loading broke .so loading in at least some (maybe all?) cases, where find_library doesn't return an absolute path. * Prematurely pushed some in-progress test changes that were super broken, all fixed now. --- magic/loader.py | 16 ++- test/python_magic_test.py | 238 +++++++++++++++++++++----------------- 2 files changed, 140 insertions(+), 114 deletions(-) diff --git a/magic/loader.py b/magic/loader.py index e6edc7bf..f8d59faf 100644 --- a/magic/loader.py +++ b/magic/loader.py @@ -7,6 +7,7 @@ logger = logging.getLogger(__name__) + def _lib_candidates_linux(): """Yield possible libmagic library names on Linux. @@ -51,7 +52,7 @@ def _lib_candidates(): "darwin": _lib_candidates_macos, "linux": _lib_candidates_linux, "win32": _lib_candidates_windows, - "sunos5": _lib_candidates_linux, + "sunos5": _lib_candidates_linux, }.get(sys.platform) if func is None: raise ImportError("python-magic: Unsupported platform: " + sys.platform) @@ -61,17 +62,20 @@ def _lib_candidates(): def load_lib(): + exc = [] for lib in _lib_candidates(): # find_library returns None when lib not found if lib is None: continue - if not os.path.exists(lib): - continue try: return ctypes.CDLL(lib) - except OSError: - logger.warning("Failed to load: " + lib, exc_info=True) + except OSError as e: + exc.append(e) + + msg = "\n".join([str(e) for e in exc]) # It is better to raise an ImportError since we are importing magic module - raise ImportError("python-magic: failed to find libmagic. Check your installation") + raise ImportError( + "python-magic: failed to find libmagic. Check your installation: \n" + msg + ) diff --git a/test/python_magic_test.py b/test/python_magic_test.py index b5577620..50760440 100755 --- a/test/python_magic_test.py +++ b/test/python_magic_test.py @@ -5,6 +5,7 @@ import shutil import sys import tempfile +from typing import List, Union import unittest import pytest @@ -19,140 +20,162 @@ import magic + @dataclass class TestFile: file_name: str - mime_results: list[str] - text_results: list[str] - no_check_elf_results: list[str] | None + mime_results: List[str] + text_results: List[str] + no_check_elf_results: Union[List[str], None] buf_equals_file: bool = True + # magic_descriptor is broken (?) in centos 7, so don't run those tests SKIP_FROM_DESCRIPTOR = bool(os.environ.get("SKIP_FROM_DESCRIPTOR")) -COMMON_PLAIN = [ - {}, - {"check_soft": True}, - {"check_soft": False}, - {"check_json": True}, - {"check_json": False}, -] - -NO_SOFT = {"check_soft": False} - -COMMON_MIME = [{"mime": True, **k} for k in COMMON_PLAIN] +COMMON_PLAIN = [{}] +NO_SOFT = [{"check_soft": False}] +COMMON_MIME = [{"mime": True}] CASES = { - "magic._pyc_": [ - (COMMON_MIME, [ - "application/octet-stream", - "text/x-bytecode.python", - "application/x-bytecode.python", - ]), + b"magic._pyc_": [ + ( + COMMON_MIME, + [ + "application/octet-stream", + "text/x-bytecode.python", + "application/x-bytecode.python", + ], + ), (COMMON_PLAIN, ["python 2.4 byte-compiled"]), (NO_SOFT, ["data"]), ], - "test.pdf": [ + b"test.pdf": [ (COMMON_MIME, ["application/pdf"]), - (COMMON_PLAIN, [ - "PDF document, version 1.2", - "PDF document, version 1.2, 2 pages", - "PDF document, version 1.2, 2 page(s)", - ]), + ( + COMMON_PLAIN, + [ + "PDF document, version 1.2", + "PDF document, version 1.2, 2 pages", + "PDF document, version 1.2, 2 page(s)", + ], + ), (NO_SOFT, ["ASCII text"]), ], - "test.gz": [ + b"test.gz": [ (COMMON_MIME, ["application/gzip", "application/x-gzip"]), - (COMMON_PLAIN, [ - 'gzip compressed data, was "test", from Unix, last modified: Sun Jun 29 01:32:52 2008', - 'gzip compressed data, was "test", last modified: Sun Jun 29 01:32:52 2008, from Unix', - 'gzip compressed data, was "test", last modified: Sun Jun 29 01:32:52 2008, from Unix, original size 15', - 'gzip compressed data, was "test", last modified: Sun Jun 29 01:32:52 2008, from Unix, original size modulo 2^32 15', - 'gzip compressed data, was "test", last modified: Sun Jun 29 01:32:52 2008, from Unix, truncated', - ]), - ({"extension": True}, [ - # some versions return '' for the extensions of a gz file, - # including w/ the command line. Who knows... - "gz/tgz/tpz/zabw/svgz/adz/kmy/xcfgz", - "gz/tgz/tpz/zabw/svgz", - "", - "???", - ]), + ( + COMMON_PLAIN, + [ + 'gzip compressed data, was "test", from Unix, last modified: Sun Jun 29 01:32:52 2008', + 'gzip compressed data, was "test", last modified: Sun Jun 29 01:32:52 2008, from Unix', + 'gzip compressed data, was "test", last modified: Sun Jun 29 01:32:52 2008, from Unix, original size 15', + 'gzip compressed data, was "test", last modified: Sun Jun 29 01:32:52 2008, from Unix, original size modulo 2^32 15', + 'gzip compressed data, was "test", last modified: Sun Jun 29 01:32:52 2008, from Unix, truncated', + ], + ), + ( + [{"extension": True}], + [ + # some versions return '' for the extensions of a gz file, + # including w/ the command line. Who knows... + "gz/tgz/tpz/zabw/svgz/adz/kmy/xcfgz", + "gz/tgz/tpz/zabw/svgz", + "", + "???", + ], + ), (NO_SOFT, ["data"]), ], - "test.snappy.parquet": [ + b"test.snappy.parquet": [ (COMMON_MIME, ["application/octet-stream"]), (COMMON_PLAIN, ["Apache Parquet", "Par archive data"]), (NO_SOFT, ["data"]), ], - "test.json": [ - # TODO: soft, no_json + b"test.json": [ (COMMON_MIME, ["application/json"]), (COMMON_PLAIN, ["JSON text data"]), - ({"mime": True, "check_json": False}, [ - "data", - ]), - (NO_SOFT, ["JSON text data"]) + ( + [{"mime": True, "check_json": False}], + [ + "text/plain", + ], + ), + (NO_SOFT, ["JSON text data"]), ], - "elf-NetBSD-x86_64-echo": [ + b"elf-NetBSD-x86_64-echo": [ # TODO: soft, no elf - (COMMON_PLAIN, [ - "ELF 64-bit LSB shared object, x86-64, version 1 (SYSV)", - "ELF 64-bit LSB pie executable, x86-64, version 1 (SYSV), dynamically linked, interpreter /libexec/ld.elf_so, for NetBSD 8.0, not stripped", - ]), - (COMMON_MIME, [ - "application/x-pie-executable", - "application/x-sharedlib", - ]), - ({"check_elf": False}, [ - "ELF 64-bit LSB shared object, x86-64, version 1 (SYSV)", - ]), + ( + COMMON_PLAIN, + [ + "ELF 64-bit LSB shared object, x86-64, version 1 (SYSV)", + "ELF 64-bit LSB pie executable, x86-64, version 1 (SYSV), dynamically linked, interpreter /libexec/ld.elf_so, for NetBSD 8.0, not stripped", + ], + ), + ( + COMMON_MIME, + [ + "application/x-pie-executable", + "application/x-sharedlib", + ], + ), + ( + [{"check_elf": False}], + [ + "ELF 64-bit LSB shared object, x86-64, version 1 (SYSV)", + ], + ), # TODO: sometimes # "ELF 64-bit LSB pie executable, x86-64, version 1 (SYSV), dynamically linked, interpreter /libexec/ld.elf_so, for NetBSD 8.0, not stripped", - (NO_SOFT, ["data"]), ], - "test.txt": [ + b"text.txt": [ (COMMON_MIME, ["text/plain"]), (COMMON_PLAIN, ["ASCII text"]), - ({"mime_encoding": True}, [ - "us-ascii", - ]), + ( + [{"mime_encoding": True}], + [ + "us-ascii", + ], + ), (NO_SOFT, ["ASCII text"]), ], - "text-iso8859-1.txt": [ - ({"mime_encoding": True}, [ - "iso-8859-1", - ]), + b"text-iso8859-1.txt": [ + ( + [{"mime_encoding": True}], + [ + "iso-8859-1", + ], + ), ], b"\xce\xbb": [ (COMMON_MIME, ["text/plain"]), ], - "b\xce\xbb".decode("utf-8"): [ - (COMMON_MIME, ["text/plain"]), + b"name_use.jpg": [ + ([{"extension": True}], ["jpeg/jpg/jpe/jfif"]), ], - "name_use.jpg": [ - ({"extension": True}, [ - "jpeg/jpg/jpe/jfif" - ]), + b"keep-going.jpg": [ + (COMMON_MIME, ["image/jpeg"]), + ( + [{"mime": True, "keep_going": True}], + [ + "image/jpeg\\012- application/octet-stream", + ], + ), ], - "keep-going.jpg": [ - (COMMON_MIME, [ - "image/jpeg" - ]), - ({"mime": True, "keep_going": True}, [ - "image/jpeg\\012- application/octet-stream", - ]) + b"../../magic/loader.py": [ + ( + COMMON_MIME, + [ + "text/x-python", + "text/x-script.python", + ], + ) ], - "test.py": [ - (COMMON_MIME, [ - "text/x-python", - "text/x-script.python", - ]) - ] } + class MagicTest(unittest.TestCase): TESTDATA_DIR = os.path.abspath(os.path.join(os.path.dirname(__file__), "testdata")) @@ -165,7 +188,6 @@ def test_version(self): def test_fs_encoding(self): self.assertEqual("utf-8", sys.getfilesystemencoding().lower()) - def test_from_file_str_and_bytes(self): filename = os.path.join(self.TESTDATA_DIR, "test.pdf") @@ -174,7 +196,6 @@ def test_from_file_str_and_bytes(self): "application/pdf", magic.from_file(filename.encode("utf-8"), mime=True) ) - def test_all_cases(self): # TODO: # * MAGIC_EXTENSION not supported @@ -184,21 +205,24 @@ def test_all_cases(self): shutil.copyfile(os.path.join(MagicTest.TESTDATA_DIR, "lambda"), dest) os.environ["TZ"] = "UTC" try: - for file_name, cases in CASES: - filename = os.path.join(self.TESTDATA_DIR, file_name) - for flags, outputs in cases: - m = magic.Magic(**flags) - with open(filename) as f: - self.assertIn(m.from_descriptor(f.fileno()), outputs) - - self.assertIn(m.from_file(filename), outputs) - - fname_bytes = filename.encode("utf-8") - self.assertIn(m.from_file(fname_bytes), outputs) - - with open(file_name, "rb") as f: - buf_result = m.from_buffer(f.read(1024)) - self.assertIn(buf_result, outputs) + for filename, cases in CASES.items(): + filename = os.path.join(self.TESTDATA_DIR.encode("utf-8"), filename) + print("test case ", filename, file=sys.stderr) + for flag_variants, outputs in cases: + for flags in flag_variants: + print("flags", flags, file=sys.stderr) + m = magic.Magic(**flags) + with open(filename) as f: + self.assertIn(m.from_descriptor(f.fileno()), outputs) + + self.assertIn(m.from_file(filename), outputs) + + fname_str = filename.decode("utf-8") + self.assertIn(m.from_file(fname_str), outputs) + + with open(filename, "rb") as f: + buf_result = m.from_buffer(f.read(1024)) + self.assertIn(buf_result, outputs) finally: del os.environ["TZ"] os.unlink(dest) @@ -222,7 +246,6 @@ def test_unicode_result_raw(self): else: raise unittest.SkipTest("Magic file doesn't return expected type.") - def test_errors(self): m = magic.Magic() self.assertRaises(IOError, m.from_file, "nonexistent") @@ -233,7 +256,6 @@ def test_errors(self): finally: del os.environ["MAGIC"] - def test_rethrow(self): old = magic.magic_buffer try: From 5a896446296e1cdcfa1ba55410d338b724cc8a90 Mon Sep 17 00:00:00 2001 From: Adam Hupp Date: Wed, 19 Feb 2025 12:30:05 -0800 Subject: [PATCH 63/78] add support for python 3.13 --- .github/workflows/ci.yml | 8 +++--- setup.py | 56 +++++++++++++++++++++------------------- tox.ini | 1 + 3 files changed, 34 insertions(+), 31 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9c4e4c9a..251eb0b7 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -5,11 +5,11 @@ jobs: strategy: fail-fast: false matrix: - os: ['ubuntu-latest'] - python-version: ['3.8', '3.9', '3.10', '3.11', '3.12', '3.13'] + os: ["ubuntu-latest"] + python-version: ["3.8", "3.9", "3.10", "3.11", "3.12", "3.13"] include: - os: macos-latest - python-version: '3.13' + python-version: "3.13" # - os: windows-latest # TODO: Fix the Windows test that runs in an infinite loop # python-version: '3.13' runs-on: ${{ matrix.os }} @@ -28,4 +28,4 @@ jobs: run: pip install python-magic-bin - run: LC_ALL=en_US.UTF-8 pytest shell: bash - timeout-minutes: 15 # Limit Windows infinite loop. + timeout-minutes: 15 # Limit Windows infinite loop. diff --git a/setup.py b/setup.py index d98b7318..54aff089 100644 --- a/setup.py +++ b/setup.py @@ -8,41 +8,43 @@ def read(file_name): """Read a text file and return the content as a string.""" - with io.open(os.path.join(os.path.dirname(__file__), file_name), - encoding='utf-8') as f: + with io.open( + os.path.join(os.path.dirname(__file__), file_name), encoding="utf-8" + ) as f: return f.read() + setuptools.setup( - name='python-magic', - description='File type identification using libmagic', - author='Adam Hupp', - author_email='adam@hupp.org', + name="python-magic", + description="File type identification using libmagic", + author="Adam Hupp", + author_email="adam@hupp.org", url="http://github.com/ahupp/python-magic", - version='0.4.28', - long_description=read('README.md'), - long_description_content_type='text/markdown', - packages=['magic'], + version="0.4.28", + long_description=read("README.md"), + long_description_content_type="text/markdown", + packages=["magic"], package_data={ - 'magic': ['py.typed', '*.pyi', '**/*.pyi'], + "magic": ["py.typed", "*.pyi", "**/*.pyi"], }, keywords="mime magic file", license="MIT", - python_requires='>=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*', + python_requires=">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*", classifiers=[ - 'Intended Audience :: Developers', - 'License :: OSI Approved :: MIT License', - 'Programming Language :: Python', - 'Programming Language :: Python :: 2.7', - 'Programming Language :: Python :: 3', - 'Programming Language :: Python :: 3.5', - 'Programming Language :: Python :: 3.6', - 'Programming Language :: Python :: 3.7', - 'Programming Language :: Python :: 3.8', - 'Programming Language :: Python :: 3.9', - 'Programming Language :: Python :: 3.10', - 'Programming Language :: Python :: 3.11', - 'Programming Language :: Python :: 3.12', - 'Programming Language :: Python :: Implementation :: CPython', + "Intended Audience :: Developers", + "License :: OSI Approved :: MIT License", + "Programming Language :: Python", + "Programming Language :: Python :: 2.7", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.5", + "Programming Language :: Python :: 3.6", + "Programming Language :: Python :: 3.7", + "Programming Language :: Python :: 3.8", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", + "Programming Language :: Python :: Implementation :: CPython", ], ) - diff --git a/tox.ini b/tox.ini index b6ed98c7..5c1648b3 100644 --- a/tox.ini +++ b/tox.ini @@ -9,6 +9,7 @@ envlist = py310, py311, py312, + py313, mypy [testenv] From 62bd3c6a562b26e4005a012c30a0e86428b8defc Mon Sep 17 00:00:00 2001 From: Adam Hupp Date: Sat, 1 Mar 2025 17:10:13 -0800 Subject: [PATCH 64/78] format with ruff --- magic/__init__.py | 69 +++++++++++++++++++++++++++---------------- magic/__init__.pyi | 20 ++++++++++++- ruff.toml | 3 ++ test/libmagic_test.py | 22 +++++++------- 4 files changed, 78 insertions(+), 36 deletions(-) create mode 100644 ruff.toml diff --git a/magic/__init__.py b/magic/__init__.py index d56caafc..851b717f 100644 --- a/magic/__init__.py +++ b/magic/__init__.py @@ -38,12 +38,27 @@ class Magic: Magic is a wrapper around the libmagic C library. """ - def __init__(self, mime=False, magic_file=None, mime_encoding=False, - keep_going=False, uncompress=False, raw=False, extension=False, - follow_symlinks=False, check_tar=True, check_soft=True, - check_apptype=True, check_elf=True, check_text=True, - check_cdf=True, check_csv=True, check_encoding=True, - check_json=True, check_simh=True): + def __init__( + self, + mime=False, + magic_file=None, + mime_encoding=False, + keep_going=False, + uncompress=False, + raw=False, + extension=False, + follow_symlinks=False, + check_tar=True, + check_soft=True, + check_apptype=True, + check_elf=True, + check_text=True, + check_cdf=True, + check_csv=True, + check_encoding=True, + check_json=True, + check_simh=True, + ): """ Create a new libmagic wrapper. @@ -101,7 +116,9 @@ def __init__(self, mime=False, magic_file=None, mime_encoding=False, # MAGIC_EXTENSION was added in 523 or 524, so bail if # it doesn't appear to be available if extension and (not _has_version or version() < 524): - raise NotImplementedError('MAGIC_EXTENSION is not supported in this version of libmagic') + raise NotImplementedError( + "MAGIC_EXTENSION is not supported in this version of libmagic" + ) # For https://github.com/ahupp/python-magic/issues/190 # libmagic has fixed internal limits that some files exceed, causing @@ -128,7 +145,7 @@ def from_buffer(self, buf): # which is not what libmagic expects # NEXTBREAK: only take bytes if type(buf) == str and str != bytes: - buf = buf.encode('utf-8', errors='replace') + buf = buf.encode("utf-8", errors="replace") return maybe_decode(magic_buffer(self.cookie, buf)) except MagicException as e: return self._handle509Bug(e) @@ -176,7 +193,7 @@ def __del__(self): # incorrect fix for a threading problem, however I'm leaving # it in because it's harmless and I'm slightly afraid to # remove it. - if hasattr(self, 'cookie') and self.cookie and magic_close: + if hasattr(self, "cookie") and self.cookie and magic_close: magic_close(self.cookie) self.cookie = None @@ -192,7 +209,7 @@ def _get_magic_type(mime): def from_file(filename, mime=False): - """" + """ Accepts a filename and returns the detected filetype. Return value is the mimetype if mime=True, otherwise a human readable name. @@ -230,7 +247,9 @@ def from_descriptor(fd, mime=False): m = _get_magic_type(mime) return m.from_descriptor(fd) + from . import loader + libmagic = loader.load_lib() magic_t = ctypes.c_void_p @@ -261,20 +280,23 @@ def maybe_decode(s): else: # backslashreplace here because sometimes libmagic will return metadata in the charset # of the file, which is unknown to us (e.g the title of a Word doc) - return s.decode('utf-8', 'backslashreplace') + return s.decode("utf-8", "backslashreplace") try: from os import PathLike + def unpath(filename): if isinstance(filename, PathLike): return filename.__fspath__() else: return filename except ImportError: + def unpath(filename): return filename + def coerce_filename(filename): if filename is None: return None @@ -286,12 +308,11 @@ def coerce_filename(filename): # then you'll get inconsistent behavior (crashes) depending on the user's # LANG environment variable # NEXTBREAK: remove - is_unicode = (sys.version_info[0] <= 2 and - isinstance(filename, unicode)) or \ - (sys.version_info[0] >= 3 and - isinstance(filename, str)) + is_unicode = (sys.version_info[0] <= 2 and isinstance(filename, unicode)) or ( + sys.version_info[0] >= 3 and isinstance(filename, str) + ) if is_unicode: - return filename.encode('utf-8', 'surrogateescape') + return filename.encode("utf-8", "surrogateescape") else: return filename @@ -370,7 +391,7 @@ def magic_load(cookie, filename): magic_compile.argtypes = [magic_t, c_char_p] _has_param = False -if hasattr(libmagic, 'magic_setparam') and hasattr(libmagic, 'magic_getparam'): +if hasattr(libmagic, "magic_setparam") and hasattr(libmagic, "magic_getparam"): _has_param = True _magic_setparam = libmagic.magic_setparam _magic_setparam.restype = c_int @@ -443,8 +464,8 @@ def version(): MAGIC_NO_CHECK_CDF = 0x0040000 # Don't check for CDF files MAGIC_NO_CHECK_CSV = 0x0080000 # Don't check for CSV files MAGIC_NO_CHECK_ENCODING = 0x0200000 # Don't check text encodings -MAGIC_NO_CHECK_JSON = 0x0400000 # Don't check for JSON files -MAGIC_NO_CHECK_SIMH = 0x0800000 # Don't check for SIMH tape files +MAGIC_NO_CHECK_JSON = 0x0400000 # Don't check for JSON files +MAGIC_NO_CHECK_SIMH = 0x0800000 # Don't check for SIMH tape files MAGIC_PARAM_INDIR_MAX = 0 # Recursion limit for indirect magic MAGIC_PARAM_NAME_MAX = 1 # Use count limit for name/use magic @@ -468,22 +489,20 @@ def _(*args, **kwargs): warnings.warn( "Using compatibility mode with libmagic's python binding. " "See https://github.com/ahupp/python-magic/blob/master/COMPAT.md for details.", - PendingDeprecationWarning) + PendingDeprecationWarning, + ) return fn(*args, **kwargs) return _ - fn = ['detect_from_filename', - 'detect_from_content', - 'detect_from_fobj', - 'open'] + fn = ["detect_from_filename", "detect_from_content", "detect_from_fobj", "open"] for fname in fn: to_module[fname] = deprecation_wrapper(compat.__dict__[fname]) # copy constants over, ensuring there's no conflicts is_const_re = re.compile("^[A-Z_]+$") - allowed_inconsistent = set(['MAGIC_MIME']) + allowed_inconsistent = set(["MAGIC_MIME"]) for name, value in compat.__dict__.items(): if is_const_re.match(name): if name in to_module: diff --git a/magic/__init__.pyi b/magic/__init__.pyi index 0e375881..bea800a4 100644 --- a/magic/__init__.pyi +++ b/magic/__init__.pyi @@ -11,7 +11,25 @@ class Magic: flags: int = ... cookie: Any = ... lock: threading.Lock = ... - def __init__(self, mime: bool = ..., magic_file: Optional[Any] = ..., mime_encoding: bool = ..., keep_going: bool = ..., uncompress: bool = ..., raw: bool = ..., extension: bool = ..., follow_symlinks: bool = ..., check_tar: bool = ..., check_soft: bool = ..., check_apptype: bool = ..., check_elf: bool = ..., check_text: bool = ..., check_encoding: bool = ..., check_json: bool = ..., check_simh: bool = ...) -> None: ... + def __init__( + self, + mime: bool = ..., + magic_file: Optional[Any] = ..., + mime_encoding: bool = ..., + keep_going: bool = ..., + uncompress: bool = ..., + raw: bool = ..., + extension: bool = ..., + follow_symlinks: bool = ..., + check_tar: bool = ..., + check_soft: bool = ..., + check_apptype: bool = ..., + check_elf: bool = ..., + check_text: bool = ..., + check_encoding: bool = ..., + check_json: bool = ..., + check_simh: bool = ..., + ) -> None: ... def from_buffer(self, buf: Union[bytes, str]) -> Text: ... def from_file(self, filename: Union[bytes, str, PathLike]) -> Text: ... def from_descriptor(self, fd: int, mime: bool = ...) -> Text: ... diff --git a/ruff.toml b/ruff.toml new file mode 100644 index 00000000..fe365518 --- /dev/null +++ b/ruff.toml @@ -0,0 +1,3 @@ +exclude = ["magic/compat.py"] + + diff --git a/test/libmagic_test.py b/test/libmagic_test.py index 7b4665b5..fff71cda 100644 --- a/test/libmagic_test.py +++ b/test/libmagic_test.py @@ -6,16 +6,20 @@ import os.path # magic_descriptor is broken (?) in centos 7, so don't run those tests -SKIP_FROM_DESCRIPTOR = bool(os.environ.get('SKIP_FROM_DESCRIPTOR')) +SKIP_FROM_DESCRIPTOR = bool(os.environ.get("SKIP_FROM_DESCRIPTOR")) -TESTDATA_DIR = os.path.abspath(os.path.join(os.path.dirname(__file__), 'testdata')) +TESTDATA_DIR = os.path.abspath(os.path.join(os.path.dirname(__file__), "testdata")) class MagicTestCase(unittest.TestCase): - filename = os.path.join(TESTDATA_DIR, 'test.pdf') - expected_mime_type = 'application/pdf' - expected_encoding = 'us-ascii' - expected_name = ('PDF document, version 1.2', 'PDF document, version 1.2, 2 pages', 'PDF document, version 1.2, 2 page(s)') + filename = os.path.join(TESTDATA_DIR, "test.pdf") + expected_mime_type = "application/pdf" + expected_encoding = "us-ascii" + expected_name = ( + "PDF document, version 1.2", + "PDF document, version 1.2, 2 pages", + "PDF document, version 1.2, 2 page(s)", + ) def assert_result(self, result): self.assertEqual(result.mime_type, self.expected_mime_type) @@ -27,11 +31,9 @@ def test_detect_from_filename(self): self.assert_result(result) def test_detect_from_fobj(self): - if SKIP_FROM_DESCRIPTOR: self.skipTest("magic_descriptor is broken in this version of libmagic") - with open(self.filename) as fobj: result = magic.detect_from_fobj(fobj) self.assert_result(result) @@ -41,10 +43,10 @@ def test_detect_from_content(self): # this avoids hitting a bug in python3+libfile bindings # see https://github.com/ahupp/python-magic/issues/152 # for a similar issue - with open(self.filename, 'rb') as fobj: + with open(self.filename, "rb") as fobj: result = magic.detect_from_content(fobj.read(4096)) self.assert_result(result) -if __name__ == '__main__': +if __name__ == "__main__": unittest.main() From fac66155b77aa0d216085911a71be190b623c6c3 Mon Sep 17 00:00:00 2001 From: Name <87663453+Dodf12@users.noreply.github.com> Date: Tue, 1 Jul 2025 22:18:39 -0700 Subject: [PATCH 65/78] Small Fix to ReadMe that makes pip install command easier to see/find I wasn't able to really find the pip command quickly, so I thought this addition would help with readability and help people find the pip install command easier --- README.md | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 010cc8f2..b7dc10e8 100644 --- a/README.md +++ b/README.md @@ -45,8 +45,11 @@ You can also combine the flag options: ## Installation -The current stable version of python-magic is available on PyPI and -can be installed by running `pip install python-magic`. +The current stable version of Python-Magic is available on PyPI and +can be installed by running: +``` +pip install python-magic +``` Other sources: From 8361a3333b73cd5c06fa6dc067b6d31c5b4245df Mon Sep 17 00:00:00 2001 From: Adam Hupp Date: Sun, 6 Jul 2025 15:23:54 -0700 Subject: [PATCH 66/78] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index b7dc10e8..cbe6aa6b 100644 --- a/README.md +++ b/README.md @@ -45,7 +45,7 @@ You can also combine the flag options: ## Installation -The current stable version of Python-Magic is available on PyPI and +The current stable version of python-magic is available on PyPI and can be installed by running: ``` pip install python-magic From 7cbbc99c613608423eaf97c74de2cdbab177e667 Mon Sep 17 00:00:00 2001 From: Adam Hupp Date: Thu, 14 Aug 2025 20:10:59 -0700 Subject: [PATCH 67/78] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index cbe6aa6b..c55f87c1 100644 --- a/README.md +++ b/README.md @@ -73,7 +73,7 @@ sudo apt-get install libmagic1 If python-magic fails to load the library it may be in a non-standard location, in which case you can set the environment variable `DYLD_LIBRARY_PATH` to point to it. ### SmartOS: -- Install libmagic for source https://github.com/threatstack/libmagic/ +- Install libmagic for source: https://github.com/file/file - Depending on your ./configure --prefix settings set your LD_LIBRARY_PATH to /lib ### Troubleshooting From f8fb0ee1f36988e2ba9eb1c2fe196427f79c2728 Mon Sep 17 00:00:00 2001 From: Christian Clauss Date: Wed, 24 Sep 2025 13:55:15 +0200 Subject: [PATCH 68/78] Add Python 3.14 to the testing Python v3.14 -- October 7th * https://www.python.org/download/pre-releases * https://www.python.org/downloads/release/python-3140rc3 * https://docs.python.org/3.14/whatsnew/3.14.html Like: * #347 --- .github/workflows/ci.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 251eb0b7..049880d4 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -6,16 +6,16 @@ jobs: fail-fast: false matrix: os: ["ubuntu-latest"] - python-version: ["3.8", "3.9", "3.10", "3.11", "3.12", "3.13"] + python-version: ["3.8", "3.9", "3.10", "3.11", "3.12", "3.13", "3.14"] include: - os: macos-latest - python-version: "3.13" + python-version: "3.x" # - os: windows-latest # TODO: Fix the Windows test that runs in an infinite loop # python-version: '3.13' runs-on: ${{ matrix.os }} steps: - - uses: actions/checkout@v4 - - uses: actions/setup-python@v5 + - uses: actions/checkout@v5 + - uses: actions/setup-python@v6 with: python-version: ${{ matrix.python-version }} allow-prereleases: true From 07bd5dd0ed651465d086e57abc629b1071162ea3 Mon Sep 17 00:00:00 2001 From: Christian Clauss Date: Wed, 24 Sep 2025 14:18:52 +0200 Subject: [PATCH 69/78] Keep GitHub Actions up to date with GitHub's Dependabot * [Keeping your software supply chain secure with Dependabot](https://docs.github.com/en/code-security/dependabot) * [Keeping your actions up to date with Dependabot](https://docs.github.com/en/code-security/dependabot/working-with-dependabot/keeping-your-actions-up-to-date-with-dependabot) * [Configuration options for the `dependabot.yml` file - package-ecosystem](https://docs.github.com/en/code-security/dependabot/dependabot-version-updates/configuration-options-for-the-dependabot.yml-file#package-ecosystem) --- .github/dependabot.yml | 13 +++++++++++++ 1 file changed, 13 insertions(+) create mode 100644 .github/dependabot.yml diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 00000000..be006de9 --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,13 @@ +# Keep GitHub Actions up to date with GitHub's Dependabot... +# https://docs.github.com/en/code-security/dependabot/working-with-dependabot/keeping-your-actions-up-to-date-with-dependabot +# https://docs.github.com/en/code-security/dependabot/dependabot-version-updates/configuration-options-for-the-dependabot.yml-file#package-ecosystem +version: 2 +updates: + - package-ecosystem: github-actions + directory: / + groups: + github-actions: + patterns: + - "*" # Group all Actions updates into a single larger pull request + schedule: + interval: weekly From 5cffa79c84f4657e80c53923b654756e16b38297 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 24 Nov 2025 14:26:19 +0000 Subject: [PATCH 70/78] Bump actions/checkout from 5 to 6 in the github-actions group Bumps the github-actions group with 1 update: [actions/checkout](https://github.com/actions/checkout). Updates `actions/checkout` from 5 to 6 - [Release notes](https://github.com/actions/checkout/releases) - [Changelog](https://github.com/actions/checkout/blob/main/CHANGELOG.md) - [Commits](https://github.com/actions/checkout/compare/v5...v6) --- updated-dependencies: - dependency-name: actions/checkout dependency-version: '6' dependency-type: direct:production update-type: version-update:semver-major dependency-group: github-actions ... Signed-off-by: dependabot[bot] --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 049880d4..ab0c6435 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -14,7 +14,7 @@ jobs: # python-version: '3.13' runs-on: ${{ matrix.os }} steps: - - uses: actions/checkout@v5 + - uses: actions/checkout@v6 - uses: actions/setup-python@v6 with: python-version: ${{ matrix.python-version }} From ff3e0498e87a4e4f19c90d8e20456aee78dc37a8 Mon Sep 17 00:00:00 2001 From: Semyon Pupkov Date: Tue, 3 Mar 2026 10:05:56 +0500 Subject: [PATCH 71/78] Drop unused imports --- magic/__init__.py | 6 +----- magic/compat.py | 1 - 2 files changed, 1 insertion(+), 6 deletions(-) diff --git a/magic/__init__.py b/magic/__init__.py index 851b717f..14d18968 100644 --- a/magic/__init__.py +++ b/magic/__init__.py @@ -18,11 +18,7 @@ import sys import os -import glob -import ctypes -import ctypes.util import threading -import logging from ctypes import c_char_p, c_int, c_size_t, c_void_p, byref, POINTER @@ -252,7 +248,7 @@ def from_descriptor(fd, mime=False): libmagic = loader.load_lib() -magic_t = ctypes.c_void_p +magic_t = c_void_p def errorcheck_null(result, func, args): diff --git a/magic/compat.py b/magic/compat.py index 6ab9400e..32a7b93b 100644 --- a/magic/compat.py +++ b/magic/compat.py @@ -8,7 +8,6 @@ from collections import namedtuple from ctypes import * -from ctypes.util import find_library from . import loader From a1fad4334ca36f3263dc9e30d6dbb01dec1eed71 Mon Sep 17 00:00:00 2001 From: Adam Thompson-Sharpe Date: Wed, 6 May 2026 12:15:43 -0400 Subject: [PATCH 72/78] Fix test for Apache Parquet files for file 5.47 The output for Parquet files changed in 5.47. This commit edits the test to accept both the old and new output. ```sh # Old $ file example.parquet example.parquet: Apache Parquet $ file --mime example.parquet example.parquet: application/octet-stream; charset=binary # New $ file example.parquet example.parquet: Apache Parquet file $ file --mime example.parquet example.parquet: application/vnd.apache.parquet; charset=binary ``` --- test/python_magic_test.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/python_magic_test.py b/test/python_magic_test.py index 50760440..3719b9f6 100755 --- a/test/python_magic_test.py +++ b/test/python_magic_test.py @@ -89,8 +89,8 @@ class TestFile: (NO_SOFT, ["data"]), ], b"test.snappy.parquet": [ - (COMMON_MIME, ["application/octet-stream"]), - (COMMON_PLAIN, ["Apache Parquet", "Par archive data"]), + (COMMON_MIME, ["application/octet-stream", "application/vnd.apache.parquet"]), + (COMMON_PLAIN, ["Apache Parquet", "Apache Parquet file", "Par archive data"]), (NO_SOFT, ["data"]), ], b"test.json": [ From 71301b0d4f84734116b8d834cb873548dbf09b51 Mon Sep 17 00:00:00 2001 From: ddelange <14880945+ddelange@users.noreply.github.com> Date: Tue, 14 Oct 2025 14:09:18 +0300 Subject: [PATCH 73/78] Add python 3.14 to CI --- .github/workflows/ci.yml | 2 +- tox.ini | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ab0c6435..83d84d9c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -6,7 +6,7 @@ jobs: fail-fast: false matrix: os: ["ubuntu-latest"] - python-version: ["3.8", "3.9", "3.10", "3.11", "3.12", "3.13", "3.14"] + python-version: ["3.8", "3.9", "3.10", "3.11", "3.12", "3.13", "3.13t", "3.14", "3.14t"] include: - os: macos-latest python-version: "3.x" diff --git a/tox.ini b/tox.ini index 5c1648b3..51b12cfd 100644 --- a/tox.ini +++ b/tox.ini @@ -10,6 +10,9 @@ envlist = py311, py312, py313, + py313t, + py314, + py314t, mypy [testenv] From 8e7d98e11632f33f714b057ff1052cb13bd69ac2 Mon Sep 17 00:00:00 2001 From: ddelange <14880945+ddelange@users.noreply.github.com> Date: Tue, 14 Oct 2025 14:23:53 +0300 Subject: [PATCH 74/78] Move lock to global scope --- magic/__init__.py | 67 +++++++++++++++++++++++++---------------------- 1 file changed, 36 insertions(+), 31 deletions(-) diff --git a/magic/__init__.py b/magic/__init__.py index 14d18968..fbdc3889 100644 --- a/magic/__init__.py +++ b/magic/__init__.py @@ -105,7 +105,6 @@ def __init__( self.flags |= MAGIC_NO_CHECK_SIMH self.cookie = magic_open(self.flags) - self.lock = threading.Lock() magic_load(self.cookie, magic_file) @@ -134,34 +133,31 @@ def from_buffer(self, buf): """ Identify the contents of `buf` """ - with self.lock: - try: - # if we're on python3, convert buf to bytes - # otherwise this string is passed as wchar* - # which is not what libmagic expects - # NEXTBREAK: only take bytes - if type(buf) == str and str != bytes: - buf = buf.encode("utf-8", errors="replace") - return maybe_decode(magic_buffer(self.cookie, buf)) - except MagicException as e: - return self._handle509Bug(e) + try: + # if we're on python3, convert buf to bytes + # otherwise this string is passed as wchar* + # which is not what libmagic expects + # NEXTBREAK: only take bytes + if type(buf) == str and str != bytes: + buf = buf.encode("utf-8", errors="replace") + return maybe_decode(magic_buffer(self.cookie, buf)) + except MagicException as e: + return self._handle509Bug(e) def from_file(self, filename): # raise FileNotFoundException or IOError if the file does not exist os.stat(filename, follow_symlinks=self.flags & MAGIC_SYMLINK) - with self.lock: - try: - return maybe_decode(magic_file(self.cookie, filename)) - except MagicException as e: - return self._handle509Bug(e) + try: + return maybe_decode(magic_file(self.cookie, filename)) + except MagicException as e: + return self._handle509Bug(e) def from_descriptor(self, fd): - with self.lock: - try: - return maybe_decode(magic_descriptor(self.cookie, fd)) - except MagicException as e: - return self._handle509Bug(e) + try: + return maybe_decode(magic_descriptor(self.cookie, fd)) + except MagicException as e: + return self._handle509Bug(e) def _handle509Bug(self, e): # libmagic 5.09 has a bug where it might fail to identify the @@ -313,6 +309,9 @@ def coerce_filename(filename): return filename +# libmagic is not thread-safe: guard for concurrent calls on a global scope +LOCK = threading.Lock() + magic_open = libmagic.magic_open magic_open.restype = magic_t magic_open.argtypes = [c_int] @@ -336,7 +335,8 @@ def coerce_filename(filename): def magic_file(cookie, filename): - return _magic_file(cookie, coerce_filename(filename)) + with LOCK: + return _magic_file(cookie, coerce_filename(filename)) _magic_buffer = libmagic.magic_buffer @@ -346,7 +346,8 @@ def magic_file(cookie, filename): def magic_buffer(cookie, buf): - return _magic_buffer(cookie, buf, len(buf)) + with LOCK: + return _magic_buffer(cookie, buf, len(buf)) magic_descriptor = libmagic.magic_descriptor @@ -361,7 +362,8 @@ def magic_buffer(cookie, buf): def magic_descriptor(cookie, fd): - return _magic_descriptor(cookie, fd) + with LOCK: + return _magic_descriptor(cookie, fd) _magic_load = libmagic.magic_load @@ -371,7 +373,8 @@ def magic_descriptor(cookie, fd): def magic_load(cookie, filename): - return _magic_load(cookie, coerce_filename(filename)) + with LOCK: + return _magic_load(cookie, coerce_filename(filename)) magic_setflags = libmagic.magic_setflags @@ -404,15 +407,16 @@ def magic_setparam(cookie, param, val): if not _has_param: raise NotImplementedError("magic_setparam not implemented") v = c_size_t(val) - return _magic_setparam(cookie, param, byref(v)) + with LOCK: + return _magic_setparam(cookie, param, byref(v)) def magic_getparam(cookie, param): if not _has_param: raise NotImplementedError("magic_getparam not implemented") val = c_size_t() - _magic_getparam(cookie, param, byref(val)) - return val.value + with LOCK: + return _magic_getparam(cookie, param, byref(val)).value _has_version = False @@ -423,10 +427,11 @@ def magic_getparam(cookie, param): magic_version.argtypes = [] -def version(): +def version(lock=None): if not _has_version: raise NotImplementedError("magic_version not implemented") - return magic_version() + with LOCK: + return magic_version() MAGIC_NONE = 0x000000 # No flags From 892543d4c575c31eea1dd68220e287a67ee98bee Mon Sep 17 00:00:00 2001 From: ddelange <14880945+ddelange@users.noreply.github.com> Date: Tue, 14 Oct 2025 14:39:45 +0300 Subject: [PATCH 75/78] Add test --- magic/__init__.py | 5 +++-- test/python_magic_test.py | 25 +++++++++++++++++++++++++ 2 files changed, 28 insertions(+), 2 deletions(-) diff --git a/magic/__init__.py b/magic/__init__.py index fbdc3889..21af9c45 100644 --- a/magic/__init__.py +++ b/magic/__init__.py @@ -416,7 +416,8 @@ def magic_getparam(cookie, param): raise NotImplementedError("magic_getparam not implemented") val = c_size_t() with LOCK: - return _magic_getparam(cookie, param, byref(val)).value + _magic_getparam(cookie, param, byref(val)) + return val.value _has_version = False @@ -427,7 +428,7 @@ def magic_getparam(cookie, param): magic_version.argtypes = [] -def version(lock=None): +def version(): if not _has_version: raise NotImplementedError("magic_version not implemented") with LOCK: diff --git a/test/python_magic_test.py b/test/python_magic_test.py index 3719b9f6..26398614 100755 --- a/test/python_magic_test.py +++ b/test/python_magic_test.py @@ -10,6 +10,12 @@ import pytest +try: + from concurrent.futures import ThreadPoolExecutor + HAS_CONCURRENT_FUTURES = True +except ImportError: # python 2.7 + HAS_CONCURRENT_FUTURES = False + # for output which reports a local time os.environ["TZ"] = "GMT" @@ -321,6 +327,25 @@ def test_symlink(self): self.assertRaises(IOError, m_follow.from_file, tmp_broken) + @unittest.skipIf(not HAS_CONCURRENT_FUTURES, "concurrent.futures not available in Python 2.7") + def test_thread_safety(self): + """Test that concurrent from_file calls don't crash (would SEGV without global lock)""" + filename = os.path.join(self.TESTDATA_DIR, "test.pdf") + + m = magic.Magic(mime=True) + + def check_file(_): + result = m.from_file(filename) + self.assertEqual(result, "application/pdf") + return result + + with ThreadPoolExecutor(100) as executor: + results = list(executor.map(check_file, range(100))) + + # All calls should complete successfully + self.assertEqual(len(results), 100) + self.assertTrue(all(r == "application/pdf" for r in results)) + if __name__ == "__main__": unittest.main() From f3cef270ce31c64d451dfd6cd0784a610f78addb Mon Sep 17 00:00:00 2001 From: ddelange <14880945+ddelange@users.noreply.github.com> Date: Tue, 14 Oct 2025 13:44:09 +0200 Subject: [PATCH 76/78] Apply suggestions from code review --- .github/workflows/ci.yml | 2 +- tox.ini | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 83d84d9c..ddcbd25c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -6,7 +6,7 @@ jobs: fail-fast: false matrix: os: ["ubuntu-latest"] - python-version: ["3.8", "3.9", "3.10", "3.11", "3.12", "3.13", "3.13t", "3.14", "3.14t"] + python-version: ["3.8", "3.9", "3.10", "3.11", "3.12", "3.13", "3.14", "3.14t"] include: - os: macos-latest python-version: "3.x" diff --git a/tox.ini b/tox.ini index 51b12cfd..01cb7b23 100644 --- a/tox.ini +++ b/tox.ini @@ -10,7 +10,6 @@ envlist = py311, py312, py313, - py313t, py314, py314t, mypy From 4043553f3d4116bcf27fc4acb64da55b25dc7f65 Mon Sep 17 00:00:00 2001 From: ddelange <14880945+ddelange@users.noreply.github.com> Date: Fri, 17 Oct 2025 20:04:26 +0300 Subject: [PATCH 77/78] Revert "Move lock to global scope" This reverts commit f2ac98d8aa7464165984068de9e484d0321cd4f3. --- magic/__init__.py | 64 +++++++++++++++++++++-------------------------- 1 file changed, 29 insertions(+), 35 deletions(-) diff --git a/magic/__init__.py b/magic/__init__.py index 21af9c45..14d18968 100644 --- a/magic/__init__.py +++ b/magic/__init__.py @@ -105,6 +105,7 @@ def __init__( self.flags |= MAGIC_NO_CHECK_SIMH self.cookie = magic_open(self.flags) + self.lock = threading.Lock() magic_load(self.cookie, magic_file) @@ -133,31 +134,34 @@ def from_buffer(self, buf): """ Identify the contents of `buf` """ - try: - # if we're on python3, convert buf to bytes - # otherwise this string is passed as wchar* - # which is not what libmagic expects - # NEXTBREAK: only take bytes - if type(buf) == str and str != bytes: - buf = buf.encode("utf-8", errors="replace") - return maybe_decode(magic_buffer(self.cookie, buf)) - except MagicException as e: - return self._handle509Bug(e) + with self.lock: + try: + # if we're on python3, convert buf to bytes + # otherwise this string is passed as wchar* + # which is not what libmagic expects + # NEXTBREAK: only take bytes + if type(buf) == str and str != bytes: + buf = buf.encode("utf-8", errors="replace") + return maybe_decode(magic_buffer(self.cookie, buf)) + except MagicException as e: + return self._handle509Bug(e) def from_file(self, filename): # raise FileNotFoundException or IOError if the file does not exist os.stat(filename, follow_symlinks=self.flags & MAGIC_SYMLINK) - try: - return maybe_decode(magic_file(self.cookie, filename)) - except MagicException as e: - return self._handle509Bug(e) + with self.lock: + try: + return maybe_decode(magic_file(self.cookie, filename)) + except MagicException as e: + return self._handle509Bug(e) def from_descriptor(self, fd): - try: - return maybe_decode(magic_descriptor(self.cookie, fd)) - except MagicException as e: - return self._handle509Bug(e) + with self.lock: + try: + return maybe_decode(magic_descriptor(self.cookie, fd)) + except MagicException as e: + return self._handle509Bug(e) def _handle509Bug(self, e): # libmagic 5.09 has a bug where it might fail to identify the @@ -309,9 +313,6 @@ def coerce_filename(filename): return filename -# libmagic is not thread-safe: guard for concurrent calls on a global scope -LOCK = threading.Lock() - magic_open = libmagic.magic_open magic_open.restype = magic_t magic_open.argtypes = [c_int] @@ -335,8 +336,7 @@ def coerce_filename(filename): def magic_file(cookie, filename): - with LOCK: - return _magic_file(cookie, coerce_filename(filename)) + return _magic_file(cookie, coerce_filename(filename)) _magic_buffer = libmagic.magic_buffer @@ -346,8 +346,7 @@ def magic_file(cookie, filename): def magic_buffer(cookie, buf): - with LOCK: - return _magic_buffer(cookie, buf, len(buf)) + return _magic_buffer(cookie, buf, len(buf)) magic_descriptor = libmagic.magic_descriptor @@ -362,8 +361,7 @@ def magic_buffer(cookie, buf): def magic_descriptor(cookie, fd): - with LOCK: - return _magic_descriptor(cookie, fd) + return _magic_descriptor(cookie, fd) _magic_load = libmagic.magic_load @@ -373,8 +371,7 @@ def magic_descriptor(cookie, fd): def magic_load(cookie, filename): - with LOCK: - return _magic_load(cookie, coerce_filename(filename)) + return _magic_load(cookie, coerce_filename(filename)) magic_setflags = libmagic.magic_setflags @@ -407,16 +404,14 @@ def magic_setparam(cookie, param, val): if not _has_param: raise NotImplementedError("magic_setparam not implemented") v = c_size_t(val) - with LOCK: - return _magic_setparam(cookie, param, byref(v)) + return _magic_setparam(cookie, param, byref(v)) def magic_getparam(cookie, param): if not _has_param: raise NotImplementedError("magic_getparam not implemented") val = c_size_t() - with LOCK: - _magic_getparam(cookie, param, byref(val)) + _magic_getparam(cookie, param, byref(val)) return val.value @@ -431,8 +426,7 @@ def magic_getparam(cookie, param): def version(): if not _has_version: raise NotImplementedError("magic_version not implemented") - with LOCK: - return magic_version() + return magic_version() MAGIC_NONE = 0x000000 # No flags From 9e0d5a780208567f06bbe47b042c43ac359c57bc Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 20 Jul 2026 13:24:23 +0000 Subject: [PATCH 78/78] Bump the github-actions group across 1 directory with 2 updates Bumps the github-actions group with 2 updates in the / directory: [actions/checkout](https://github.com/actions/checkout) and [actions/setup-python](https://github.com/actions/setup-python). Updates `actions/checkout` from 6 to 7 - [Release notes](https://github.com/actions/checkout/releases) - [Changelog](https://github.com/actions/checkout/blob/main/CHANGELOG.md) - [Commits](https://github.com/actions/checkout/compare/v6...v7) Updates `actions/setup-python` from 6 to 7 - [Release notes](https://github.com/actions/setup-python/releases) - [Commits](https://github.com/actions/setup-python/compare/v6...v7) --- updated-dependencies: - dependency-name: actions/checkout dependency-version: '7' dependency-type: direct:production update-type: version-update:semver-major dependency-group: github-actions - dependency-name: actions/setup-python dependency-version: '7' dependency-type: direct:production update-type: version-update:semver-major dependency-group: github-actions ... Signed-off-by: dependabot[bot] --- .github/workflows/ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ddcbd25c..772132e5 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -14,8 +14,8 @@ jobs: # python-version: '3.13' runs-on: ${{ matrix.os }} steps: - - uses: actions/checkout@v6 - - uses: actions/setup-python@v6 + - uses: actions/checkout@v7 + - uses: actions/setup-python@v7 with: python-version: ${{ matrix.python-version }} allow-prereleases: true