diff --git a/.dockerignore b/.dockerignore new file mode 120000 index 00000000..3e4e48b0 --- /dev/null +++ b/.dockerignore @@ -0,0 +1 @@ +.gitignore \ No newline at end of file diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 00000000..be006de9 --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,13 @@ +# Keep GitHub Actions up to date with GitHub's Dependabot... +# https://docs.github.com/en/code-security/dependabot/working-with-dependabot/keeping-your-actions-up-to-date-with-dependabot +# https://docs.github.com/en/code-security/dependabot/dependabot-version-updates/configuration-options-for-the-dependabot.yml-file#package-ecosystem +version: 2 +updates: + - package-ecosystem: github-actions + directory: / + groups: + github-actions: + patterns: + - "*" # Group all Actions updates into a single larger pull request + schedule: + interval: weekly diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 00000000..772132e5 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,31 @@ +name: ci +on: [push, pull_request] +jobs: + ci: + strategy: + fail-fast: false + matrix: + os: ["ubuntu-latest"] + python-version: ["3.8", "3.9", "3.10", "3.11", "3.12", "3.13", "3.14", "3.14t"] + include: + - os: macos-latest + python-version: "3.x" + # - os: windows-latest # TODO: Fix the Windows test that runs in an infinite loop + # python-version: '3.13' + runs-on: ${{ matrix.os }} + steps: + - uses: actions/checkout@v7 + - uses: actions/setup-python@v7 + with: + python-version: ${{ matrix.python-version }} + allow-prereleases: true + - run: pip install --upgrade pip + - run: pip install --upgrade pytest + - run: pip install --editable . + - if: runner.os == 'macOS' + run: brew install libmagic + - if: runner.os == 'Windows' + run: pip install python-magic-bin + - run: LC_ALL=en_US.UTF-8 pytest + shell: bash + timeout-minutes: 15 # Limit Windows infinite loop. diff --git a/.gitignore b/.gitignore index 111565e3..1f961bbb 100644 --- a/.gitignore +++ b/.gitignore @@ -4,10 +4,11 @@ bin/ deb_dist htmlcov/ lib/ -__pycache__/ +**/__pycache__ python_magic.egg-info pip-selfcheck.json pyvenv.cfg *.pyc *~ dist/ +.vscode/ diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index 8c306d9a..00000000 --- a/.travis.yml +++ /dev/null @@ -1,22 +0,0 @@ -language: python -dist: xenial -cache: pip - -python: - - "2.7" - - "3.5" - - "3.6" - - "3.7" - - "3.8" - - "3.9" - -install: - - pip install coverage coveralls codecov - - pip install . - -script: - - LC_ALL=en_US.UTF-8 coverage run -m unittest test - -after_success: - - coveralls - - codecov diff --git a/CHANGELOG b/CHANGELOG index 26e01f48..a8370c68 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -1,26 +1,51 @@ +Changes to 0.4.29: + +- support MAGIC_SYMLINK (via follow_symlink flag on Magic constructor) +- correctly throw FileNotFoundException depending on flag + +Changes to 0.4.28: + +- support "magic-1.dll" on Windows, which is produced by vcpkg +- add python 3.10 to tox config +- update test for upstream gzip extensions + +Changes to 0.4.27: + +- remove spurious pyproject.toml that breaks source builds + +Changes to 0.4.26: + +- Use tox for all multi-version testing +- Fix use of pytest, use it via tox + Changes to 0.4.25: - - Support os.PathLike values in Magic.from_file and magic.from_file + +- Support os.PathLike values in Magic.from_file and magic.from_file +- Handle some versions of libmagic that return mime string without charset +- Fix tests for file 5.41 +- Include typing stub in package Changes to 0.4.24: - - Fix regression in library loading on some Alpine docker images. + +- Fix regression in library loading on some Alpine docker images. Changes to 0.4.23 - - Include a `py.typed` sentinal to enable type checking - - Improve fix for attribute error during destruction - - Cleanup library loading logic - - Add new homebrew library dir for OSX +- Include a `py.typed` sentinel to enable type checking +- Improve fix for attribute error during destruction +- Cleanup library loading logic +- Add new homebrew library dir for OSX Changes to 0.4.21, 0.4.22 - - Unify dll loader between the standard and compat library, fixing load - failures on some previously supported platforms. +- Unify dll loader between the standard and compat library, fixing load + failures on some previously supported platforms. Changes to 0.4.20 - merge in a compatibility layer for the upstream libmagic python binding. Since both this package and that one are called 'magic', this compat layer - removes a very common source of runtime errors. Use of that libmagic API will + removes a very common source of runtime errors. Use of that libmagic API will produce a deprecation warning. - support python 3.9 in tests and pypi metadata @@ -29,9 +54,9 @@ Changes to 0.4.20 rather than a filename. - sometimes the returned description includes snippets of the file, e.g a title - for MS Word docs. Since this is in an unknown encoding, we would throw a - unicode decode error trying to decode. Now, it decodes with - 'backslashreplace' to handle this more gracefully. The undecodable characters + for MS Word docs. Since this is in an unknown encoding, we would throw a + unicode decode error trying to decode. Now, it decodes with + 'backslashreplace' to handle this more gracefully. The undecodable characters are replaced with hex escapes. - add support for MAGIC_EXTENSION, to return possible file extensions. @@ -40,18 +65,18 @@ Changes to 0.4.20 Changes in 0.4.18 -- Make bindings for magic_[set|get]param optional, and throw NotImplementedError -if they are used but not supported. Only call setparam() in the constructor if -it's supported. This prevents breakage on CentOS7 which uses an old version of -libmagic. +- Make bindings for magic\_[set|get]param optional, and throw NotImplementedError + if they are used but not supported. Only call setparam() in the constructor if + it's supported. This prevents breakage on CentOS7 which uses an old version of + libmagic. - Add tests for CentOS 7 & 8 Changes in 0.4.16 and 0.4.17 - add MAGIC_MIME_TYPE constant, use that in preference to MAGIC_MIME internally. -This sets up for a breaking change in a future major version bump where -MAGIC_MIME will change to mathch magic.h. + This sets up for a breaking change in a future major version bump where + MAGIC_MIME will change to match magic.h. - add magic.version() function to return library version - add setparam/getparam to control internal behavior - increase internal limits with setparam to prevent spurious error on some jpeg files @@ -61,12 +86,12 @@ MAGIC_MIME will change to mathch magic.h. - include tests in source distribution - many test improvements: --- tox runner support --- remove deprecated test_suite field from setup.py --- docker tests that cover all LTS ubuntu versions --- add test for snapp file identification + -- tox runner support + -- remove deprecated test_suite field from setup.py + -- docker tests that cover all LTS ubuntu versions + -- add test for snapp file identification - doc improvements --- document dependency install process for debian --- various typos --- document test running process + -- document dependency install process for debian + -- various typos + -- document test running process diff --git a/README.md b/README.md index 9eb70e8a..c55f87c1 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,7 @@ # python-magic [![PyPI version](https://badge.fury.io/py/python-magic.svg)](https://badge.fury.io/py/python-magic) -[![Build Status](https://travis-ci.org/ahupp/python-magic.svg?branch=master)](https://travis-ci.org/ahupp/python-magic) [![Join the chat at https://gitter.im/ahupp/python-magic](https://badges.gitter.im/ahupp/python-magic.svg)](https://gitter.im/ahupp/python-magic?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge) +[![ci](https://github.com/ahupp/python-magic/actions/workflows/ci.yml/badge.svg)](https://github.com/ahupp/python-magic/actions/workflows/ci.yml) +[![Join the chat at https://gitter.im/ahupp/python-magic](https://badges.gitter.im/ahupp/python-magic.svg)](https://gitter.im/ahupp/python-magic?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge) python-magic is a Python interface to the libmagic file type identification library. libmagic identifies file types by checking @@ -45,7 +46,10 @@ You can also combine the flag options: ## Installation The current stable version of python-magic is available on PyPI and -can be installed by running `pip install python-magic`. +can be installed by running: +``` +pip install python-magic +``` Other sources: @@ -61,19 +65,17 @@ that must be installed as well: sudo apt-get install libmagic1 ``` -### Windows - -You'll need DLLs for libmagic. @julian-r maintains a pypi package with the DLLs, you can fetch it with: - -``` -pip install python-magic-bin -``` - ### OSX - When using Homebrew: `brew install libmagic` - When using macports: `port install file` +If python-magic fails to load the library it may be in a non-standard location, in which case you can set the environment variable `DYLD_LIBRARY_PATH` to point to it. + +### SmartOS: +- Install libmagic for source: https://github.com/file/file +- Depending on your ./configure --prefix settings set your LD_LIBRARY_PATH to /lib + ### Troubleshooting - 'MagicException: could not find any magic files!': some @@ -101,22 +103,23 @@ triage it. ## Running the tests -To run the tests across a variety of linux distributions (depends on Docker): +We use the `tox` test runner which can be installed with `python -m pip install tox`. + +To run tests locally across all available python versions: ``` -./test_docker.sh +python -m tox ``` -To run tests locally across all available python versions: +Or to run just against a single version: ``` -./test/run.py +python -m tox py ``` - -To run against a specific python version: +To run the tests across a variety of linux distributions (depends on Docker): ``` -LC_ALL=en_US.UTF-8 python3 test/test.py +./test/run_all_docker_test.sh ``` ## libmagic python API compatibility diff --git a/__init__.py b/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/magic/__init__.py b/magic/__init__.py index bab7c7b1..14d18968 100644 --- a/magic/__init__.py +++ b/magic/__init__.py @@ -17,17 +17,11 @@ """ import sys -import glob -import ctypes -import ctypes.util +import os import threading -import logging from ctypes import c_char_p, c_int, c_size_t, c_void_p, byref, POINTER -# avoid shadowing the real open with the version from compat.py -_real_open = open - class MagicException(Exception): def __init__(self, message): @@ -40,8 +34,27 @@ class Magic: Magic is a wrapper around the libmagic C library. """ - def __init__(self, mime=False, magic_file=None, mime_encoding=False, - keep_going=False, uncompress=False, raw=False, extension=False): + def __init__( + self, + mime=False, + magic_file=None, + mime_encoding=False, + keep_going=False, + uncompress=False, + raw=False, + extension=False, + follow_symlinks=False, + check_tar=True, + check_soft=True, + check_apptype=True, + check_elf=True, + check_text=True, + check_cdf=True, + check_csv=True, + check_encoding=True, + check_json=True, + check_simh=True, + ): """ Create a new libmagic wrapper. @@ -67,6 +80,30 @@ def __init__(self, mime=False, magic_file=None, mime_encoding=False, if extension: self.flags |= MAGIC_EXTENSION + if follow_symlinks: + self.flags |= MAGIC_SYMLINK + + if not check_tar: + self.flags |= MAGIC_NO_CHECK_TAR + if not check_soft: + self.flags |= MAGIC_NO_CHECK_SOFT + if not check_apptype: + self.flags |= MAGIC_NO_CHECK_APPTYPE + if not check_elf: + self.flags |= MAGIC_NO_CHECK_ELF + if not check_text: + self.flags |= MAGIC_NO_CHECK_TEXT + if not check_cdf: + self.flags |= MAGIC_NO_CHECK_CDF + if not check_csv: + self.flags |= MAGIC_NO_CHECK_CSV + if not check_encoding: + self.flags |= MAGIC_NO_CHECK_ENCODING + if not check_json: + self.flags |= MAGIC_NO_CHECK_JSON + if not check_simh: + self.flags |= MAGIC_NO_CHECK_SIMH + self.cookie = magic_open(self.flags) self.lock = threading.Lock() @@ -75,7 +112,9 @@ def __init__(self, mime=False, magic_file=None, mime_encoding=False, # MAGIC_EXTENSION was added in 523 or 524, so bail if # it doesn't appear to be available if extension and (not _has_version or version() < 524): - raise NotImplementedError('MAGIC_EXTENSION is not supported in this version of libmagic') + raise NotImplementedError( + "MAGIC_EXTENSION is not supported in this version of libmagic" + ) # For https://github.com/ahupp/python-magic/issues/190 # libmagic has fixed internal limits that some files exceed, causing @@ -102,15 +141,14 @@ def from_buffer(self, buf): # which is not what libmagic expects # NEXTBREAK: only take bytes if type(buf) == str and str != bytes: - buf = buf.encode('utf-8', errors='replace') + buf = buf.encode("utf-8", errors="replace") return maybe_decode(magic_buffer(self.cookie, buf)) except MagicException as e: return self._handle509Bug(e) def from_file(self, filename): # raise FileNotFoundException or IOError if the file does not exist - with _real_open(filename): - pass + os.stat(filename, follow_symlinks=self.flags & MAGIC_SYMLINK) with self.lock: try: @@ -151,7 +189,7 @@ def __del__(self): # incorrect fix for a threading problem, however I'm leaving # it in because it's harmless and I'm slightly afraid to # remove it. - if hasattr(self, 'cookie') and self.cookie and magic_close: + if hasattr(self, "cookie") and self.cookie and magic_close: magic_close(self.cookie) self.cookie = None @@ -167,7 +205,7 @@ def _get_magic_type(mime): def from_file(filename, mime=False): - """" + """ Accepts a filename and returns the detected filetype. Return value is the mimetype if mime=True, otherwise a human readable name. @@ -205,10 +243,12 @@ def from_descriptor(fd, mime=False): m = _get_magic_type(mime) return m.from_descriptor(fd) + from . import loader + libmagic = loader.load_lib() -magic_t = ctypes.c_void_p +magic_t = c_void_p def errorcheck_null(result, func, args): @@ -236,20 +276,23 @@ def maybe_decode(s): else: # backslashreplace here because sometimes libmagic will return metadata in the charset # of the file, which is unknown to us (e.g the title of a Word doc) - return s.decode('utf-8', 'backslashreplace') + return s.decode("utf-8", "backslashreplace") try: from os import PathLike + def unpath(filename): if isinstance(filename, PathLike): return filename.__fspath__() else: return filename except ImportError: + def unpath(filename): return filename + def coerce_filename(filename): if filename is None: return None @@ -261,12 +304,11 @@ def coerce_filename(filename): # then you'll get inconsistent behavior (crashes) depending on the user's # LANG environment variable # NEXTBREAK: remove - is_unicode = (sys.version_info[0] <= 2 and - isinstance(filename, unicode)) or \ - (sys.version_info[0] >= 3 and - isinstance(filename, str)) + is_unicode = (sys.version_info[0] <= 2 and isinstance(filename, unicode)) or ( + sys.version_info[0] >= 3 and isinstance(filename, str) + ) if is_unicode: - return filename.encode('utf-8', 'surrogateescape') + return filename.encode("utf-8", "surrogateescape") else: return filename @@ -345,7 +387,7 @@ def magic_load(cookie, filename): magic_compile.argtypes = [magic_t, c_char_p] _has_param = False -if hasattr(libmagic, 'magic_setparam') and hasattr(libmagic, 'magic_getparam'): +if hasattr(libmagic, "magic_setparam") and hasattr(libmagic, "magic_getparam"): _has_param = True _magic_setparam = libmagic.magic_setparam _magic_setparam.restype = c_int @@ -410,10 +452,16 @@ def version(): MAGIC_NO_CHECK_SOFT = 0x004000 # Don't check magic entries MAGIC_NO_CHECK_APPTYPE = 0x008000 # Don't check application type MAGIC_NO_CHECK_ELF = 0x010000 # Don't check for elf details -MAGIC_NO_CHECK_ASCII = 0x020000 # Don't check for ascii files -MAGIC_NO_CHECK_TROFF = 0x040000 # Don't check ascii/troff -MAGIC_NO_CHECK_FORTRAN = 0x080000 # Don't check ascii/fortran -MAGIC_NO_CHECK_TOKENS = 0x100000 # Don't check ascii/tokens +MAGIC_NO_CHECK_TEXT = 0x020000 # Don't check for ascii files +MAGIC_NO_CHECK_ASCII = 0x020000 # Deprecated alias for MAGIC_NO_CHECK_TEXT +MAGIC_NO_CHECK_TROFF = 0x040000 # Don't check ascii/troff (deprecated) +MAGIC_NO_CHECK_FORTRAN = 0x080000 # Don't check ascii/fortran (deprecated) +MAGIC_NO_CHECK_TOKENS = 0x100000 # Don't check ascii/tokens (deprecated) +MAGIC_NO_CHECK_CDF = 0x0040000 # Don't check for CDF files +MAGIC_NO_CHECK_CSV = 0x0080000 # Don't check for CSV files +MAGIC_NO_CHECK_ENCODING = 0x0200000 # Don't check text encodings +MAGIC_NO_CHECK_JSON = 0x0400000 # Don't check for JSON files +MAGIC_NO_CHECK_SIMH = 0x0800000 # Don't check for SIMH tape files MAGIC_PARAM_INDIR_MAX = 0 # Recursion limit for indirect magic MAGIC_PARAM_NAME_MAX = 1 # Use count limit for name/use magic @@ -437,22 +485,20 @@ def _(*args, **kwargs): warnings.warn( "Using compatibility mode with libmagic's python binding. " "See https://github.com/ahupp/python-magic/blob/master/COMPAT.md for details.", - PendingDeprecationWarning) + PendingDeprecationWarning, + ) return fn(*args, **kwargs) return _ - fn = ['detect_from_filename', - 'detect_from_content', - 'detect_from_fobj', - 'open'] + fn = ["detect_from_filename", "detect_from_content", "detect_from_fobj", "open"] for fname in fn: to_module[fname] = deprecation_wrapper(compat.__dict__[fname]) # copy constants over, ensuring there's no conflicts is_const_re = re.compile("^[A-Z_]+$") - allowed_inconsistent = set(['MAGIC_MIME']) + allowed_inconsistent = set(["MAGIC_MIME"]) for name, value in compat.__dict__.items(): if is_const_re.match(name): if name in to_module: diff --git a/magic/__init__.pyi b/magic/__init__.pyi index b6b5489c..bea800a4 100644 --- a/magic/__init__.pyi +++ b/magic/__init__.pyi @@ -11,7 +11,25 @@ class Magic: flags: int = ... cookie: Any = ... lock: threading.Lock = ... - def __init__(self, mime: bool = ..., magic_file: Optional[Any] = ..., mime_encoding: bool = ..., keep_going: bool = ..., uncompress: bool = ..., raw: bool = ...) -> None: ... + def __init__( + self, + mime: bool = ..., + magic_file: Optional[Any] = ..., + mime_encoding: bool = ..., + keep_going: bool = ..., + uncompress: bool = ..., + raw: bool = ..., + extension: bool = ..., + follow_symlinks: bool = ..., + check_tar: bool = ..., + check_soft: bool = ..., + check_apptype: bool = ..., + check_elf: bool = ..., + check_text: bool = ..., + check_encoding: bool = ..., + check_json: bool = ..., + check_simh: bool = ..., + ) -> None: ... def from_buffer(self, buf: Union[bytes, str]) -> Text: ... def from_file(self, filename: Union[bytes, str, PathLike]) -> Text: ... def from_descriptor(self, fd: int, mime: bool = ...) -> Text: ... @@ -74,10 +92,16 @@ MAGIC_NO_CHECK_TAR: int MAGIC_NO_CHECK_SOFT: int MAGIC_NO_CHECK_APPTYPE: int MAGIC_NO_CHECK_ELF: int +MAGIC_NO_CHECK_TEXT: int MAGIC_NO_CHECK_ASCII: int MAGIC_NO_CHECK_TROFF: int MAGIC_NO_CHECK_FORTRAN: int +MAGIC_NO_CHECK_CDF: int +MAGIC_NO_CHECK_CSV: int MAGIC_NO_CHECK_TOKENS: int +MAGIC_NO_CHECK_ENCODING: int +MAGIC_NO_CHECK_JSON: int +MAGIC_NO_CHECK_SIMH: int MAGIC_PARAM_INDIR_MAX: int MAGIC_PARAM_NAME_MAX: int MAGIC_PARAM_ELF_PHNUM_MAX: int diff --git a/magic/compat.py b/magic/compat.py index 07fad45a..32a7b93b 100644 --- a/magic/compat.py +++ b/magic/compat.py @@ -4,13 +4,10 @@ Python bindings for libmagic ''' -import ctypes - +import threading from collections import namedtuple from ctypes import * -from ctypes.util import find_library - from . import loader @@ -45,13 +42,19 @@ MAGIC_NO_CHECK_BUILTIN = NO_CHECK_BUILTIN = 4173824 +MAGIC_PARAM_INDIR_MAX = PARAM_INDIR_MAX = 0 +MAGIC_PARAM_NAME_MAX = PARAM_NAME_MAX = 1 +MAGIC_PARAM_ELF_PHNUM_MAX = PARAM_ELF_PHNUM_MAX = 2 +MAGIC_PARAM_ELF_SHNUM_MAX = PARAM_ELF_SHNUM_MAX = 3 +MAGIC_PARAM_ELF_NOTES_MAX = PARAM_ELF_NOTES_MAX = 4 +MAGIC_PARAM_REGEX_MAX = PARAM_REGEX_MAX = 5 +MAGIC_PARAM_BYTES_MAX = PARAM_BYTES_MAX = 6 + FileMagic = namedtuple('FileMagic', ('mime_type', 'encoding', 'name')) class magic_set(Structure): pass - - magic_set._fields_ = [] magic_t = POINTER(magic_set) @@ -103,6 +106,14 @@ class magic_set(Structure): _errno.restype = c_int _errno.argtypes = [magic_t] +_getparam = _libraries['magic'].magic_getparam +_getparam.restype = c_int +_getparam.argtypes = [magic_t, c_int, c_void_p] + +_setparam = _libraries['magic'].magic_setparam +_setparam.restype = c_int +_setparam.argtypes = [magic_t, c_int, c_void_p] + class Magic(object): def __init__(self, ms): @@ -228,29 +239,81 @@ def errno(self): """ return _errno(self._magic_t) + def getparam(self, param): + """ + Returns the param value if successful and -1 if the parameter + was unknown. + """ + v = c_int() + i = _getparam(self._magic_t, param, byref(v)) + if i == -1: + return -1 + return v.value + + def setparam(self, param, value): + """ + Returns 0 if successful and -1 if the parameter was unknown. + """ + v = c_int(value) + return _setparam(self._magic_t, param, byref(v)) + def open(flags): """ Returns a magic object on success and None on failure. Flags argument as for setflags. """ - return Magic(_open(flags)) + magic_t = _open(flags) + if magic_t is None: + return None + return Magic(magic_t) # Objects used by `detect_from_` functions -mime_magic = Magic(_open(MAGIC_MIME)) -mime_magic.load() -none_magic = Magic(_open(MAGIC_NONE)) -none_magic.load() +class error(Exception): + pass +class MagicDetect(object): + def __init__(self): + self.mime_magic = open(MAGIC_MIME) + if self.mime_magic is None: + raise error + if self.mime_magic.load() == -1: + self.mime_magic.close() + self.mime_magic = None + raise error + self.none_magic = open(MAGIC_NONE) + if self.none_magic is None: + self.mime_magic.close() + self.mime_magic = None + raise error + if self.none_magic.load() == -1: + self.none_magic.close() + self.none_magic = None + self.mime_magic.close() + self.mime_magic = None + raise error + + def __del__(self): + if self.mime_magic is not None: + self.mime_magic.close() + if self.none_magic is not None: + self.none_magic.close() + +threadlocal = threading.local() + +def _detect_make(): + v = getattr(threadlocal, "magic_instance", None) + if v is None: + v = MagicDetect() + setattr(threadlocal, "magic_instance", v) + return v def _create_filemagic(mime_detected, type_detected): - splat = mime_detected.split('; ') - mime_type = splat[0] - if len(splat) == 2: - mime_encoding = splat[1] - else: - mime_encoding = '' + try: + mime_type, mime_encoding = mime_detected.split('; ') + except ValueError: + raise ValueError(mime_detected) return FileMagic(name=type_detected, mime_type=mime_type, encoding=mime_encoding.replace('charset=', '')) @@ -261,9 +324,9 @@ def detect_from_filename(filename): Returns a `FileMagic` namedtuple. ''' - - return _create_filemagic(mime_magic.file(filename), - none_magic.file(filename)) + x = _detect_make() + return _create_filemagic(x.mime_magic.file(filename), + x.none_magic.file(filename)) def detect_from_fobj(fobj): @@ -273,8 +336,9 @@ def detect_from_fobj(fobj): ''' file_descriptor = fobj.fileno() - return _create_filemagic(mime_magic.descriptor(file_descriptor), - none_magic.descriptor(file_descriptor)) + x = _detect_make() + return _create_filemagic(x.mime_magic.descriptor(file_descriptor), + x.none_magic.descriptor(file_descriptor)) def detect_from_content(byte_content): @@ -283,5 +347,6 @@ def detect_from_content(byte_content): Returns a `FileMagic` namedtuple. ''' - return _create_filemagic(mime_magic.buffer(byte_content), - none_magic.buffer(byte_content)) + x = _detect_make() + return _create_filemagic(x.mime_magic.buffer(byte_content), + x.none_magic.buffer(byte_content)) diff --git a/magic/loader.py b/magic/loader.py index 931f1619..f8d59faf 100644 --- a/magic/loader.py +++ b/magic/loader.py @@ -3,48 +3,79 @@ import sys import glob import os.path +import logging + +logger = logging.getLogger(__name__) -def _lib_candidates(): - yield find_library('magic') +def _lib_candidates_linux(): + """Yield possible libmagic library names on Linux. - if sys.platform == 'darwin': + This is necessary because alpine is bad + """ + yield "libmagic.so.1" + +def _lib_candidates_macos(): + """Yield possible libmagic library names on macOS.""" paths = [ - '/opt/local/lib', - '/usr/local/lib', - '/opt/homebrew/lib', - ] + glob.glob('/usr/local/Cellar/libmagic/*/lib') + "/opt/homebrew/lib", + "/opt/local/lib", + "/usr/local/lib", + ] + glob.glob("/usr/local/Cellar/libmagic/*/lib") + for path in paths: + yield os.path.join(path, "libmagic.dylib") - for i in paths: - yield os.path.join(i, 'libmagic.dylib') - elif sys.platform in ('win32', 'cygwin'): +def _lib_candidates_windows(): + """Yield possible libmagic library names on Windows.""" + prefixes = ( + "libmagic", + "magic1", + "magic-1", + "cygmagic-1", + "libmagic-1", + "msys-magic-1", + ) + for prefix in prefixes: + # find_library searches in %PATH% but not the current directory, + # so look for both + yield "./%s.dll" % (prefix,) + yield find_library(prefix) - prefixes = ['libmagic', 'magic1', 'cygmagic-1', 'libmagic-1', 'msys-magic-1'] - for i in prefixes: - # find_library searches in %PATH% but not the current directory, - # so look for both - yield './%s.dll' % (i,) - yield find_library(i) +def _lib_candidates(): + yield find_library("magic") - elif sys.platform == 'linux': - # This is necessary because alpine is bad - yield 'libmagic.so.1' + func = { + "cygwin": _lib_candidates_windows, + "darwin": _lib_candidates_macos, + "linux": _lib_candidates_linux, + "win32": _lib_candidates_windows, + "sunos5": _lib_candidates_linux, + }.get(sys.platform) + if func is None: + raise ImportError("python-magic: Unsupported platform: " + sys.platform) + # When we drop legacy Python, we can just `yield from func()` + for path in func(): + yield path def load_lib(): + exc = [] + for lib in _lib_candidates(): + # find_library returns None when lib not found + if lib is None: + continue - for lib in _lib_candidates(): - # find_library returns None when lib not found - if lib is None: - continue - try: - return ctypes.CDLL(lib) - except OSError: - pass - else: - # It is better to raise an ImportError since we are importing magic module - raise ImportError('failed to find libmagic. Check your installation') + try: + return ctypes.CDLL(lib) + except OSError as e: + exc.append(e) + + msg = "\n".join([str(e) for e in exc]) + # It is better to raise an ImportError since we are importing magic module + raise ImportError( + "python-magic: failed to find libmagic. Check your installation: \n" + msg + ) diff --git a/ruff.toml b/ruff.toml new file mode 100644 index 00000000..fe365518 --- /dev/null +++ b/ruff.toml @@ -0,0 +1,3 @@ +exclude = ["magic/compat.py"] + + diff --git a/setup.py b/setup.py index 295b8db7..54aff089 100644 --- a/setup.py +++ b/setup.py @@ -8,38 +8,43 @@ def read(file_name): """Read a text file and return the content as a string.""" - with io.open(os.path.join(os.path.dirname(__file__), file_name), - encoding='utf-8') as f: + with io.open( + os.path.join(os.path.dirname(__file__), file_name), encoding="utf-8" + ) as f: return f.read() + setuptools.setup( - name='python-magic', - description='File type identification using libmagic', - author='Adam Hupp', - author_email='adam@hupp.org', + name="python-magic", + description="File type identification using libmagic", + author="Adam Hupp", + author_email="adam@hupp.org", url="http://github.com/ahupp/python-magic", - version='0.4.24', - long_description=read('README.md'), - long_description_content_type='text/markdown', - packages=['magic'], + version="0.4.28", + long_description=read("README.md"), + long_description_content_type="text/markdown", + packages=["magic"], package_data={ - 'magic': ['py.typed'], + "magic": ["py.typed", "*.pyi", "**/*.pyi"], }, keywords="mime magic file", license="MIT", - python_requires='>=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*', + python_requires=">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*", classifiers=[ - 'Intended Audience :: Developers', - 'License :: OSI Approved :: MIT License', - 'Programming Language :: Python', - 'Programming Language :: Python :: 2.7', - 'Programming Language :: Python :: 3', - 'Programming Language :: Python :: 3.5', - 'Programming Language :: Python :: 3.6', - 'Programming Language :: Python :: 3.7', - 'Programming Language :: Python :: 3.8', - 'Programming Language :: Python :: 3.9', - 'Programming Language :: Python :: Implementation :: CPython', + "Intended Audience :: Developers", + "License :: OSI Approved :: MIT License", + "Programming Language :: Python", + "Programming Language :: Python :: 2.7", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.5", + "Programming Language :: Python :: 3.6", + "Programming Language :: Python :: 3.7", + "Programming Language :: Python :: 3.8", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", + "Programming Language :: Python :: Implementation :: CPython", ], ) - diff --git a/test/README b/test/README index 12d4e4fc..215ee43a 100644 --- a/test/README +++ b/test/README @@ -1,10 +1,4 @@ -To run the tests across a selection of Ubuntu LTS versions: - -docker build -t "python_magic/xenial:latest" -f test/Dockerfile_xenial . -docker build -t "python_magic/bionic:latest" -f test/Dockerfile_bionic . -docker build -t "python_magic/focal:latest" -f test/Dockerfile_focal . - -docker run python_magic/xenial:latest -docker run python_magic/bionic:latest -docker run python_magic/focal:latest +There are a few ways to run the python-magic tests +1. `tox` will run the tests against all installed versions of python +2. `./test/run_all_docker_test.sh` will run against a variety of different Linux distributions, using docker. diff --git a/test/docker/alpine b/test/docker/alpine index eb511d79..60b0698d 100755 --- a/test/docker/alpine +++ b/test/docker/alpine @@ -1,4 +1,5 @@ FROM python:3.8-alpine3.12 RUN apk add python3 python2 libmagic -COPY . /python-magic -CMD cd /python-magic/test && python3 ./run.py +WORKDIR /python-magic +COPY . . +RUN python3 -m pip install tox diff --git a/test/docker/archlinux b/test/docker/archlinux index fc96c447..6592ffc8 100755 --- a/test/docker/archlinux +++ b/test/docker/archlinux @@ -1,5 +1,6 @@ FROM archlinux:latest RUN yes | pacman -Syyu --overwrite '*' -RUN yes | pacman -S python python2 file which -COPY . /python-magic -CMD cd /python-magic/test && python3 ./run.py +RUN yes | pacman -S python python-pip file which +WORKDIR /python-magic +COPY . . +RUN python3 -m pip install tox diff --git a/test/docker/bionic b/test/docker/bionic index e335b8ee..a37b2534 100755 --- a/test/docker/bionic +++ b/test/docker/bionic @@ -1,8 +1,8 @@ FROM ubuntu:bionic RUN apt-get update -RUN apt-get -y install python -RUN apt-get -y install python3 -RUN apt-get -y install locales +RUN apt-get -y install python python3 locales python3-pip libmagic1 RUN locale-gen en_US.UTF-8 -COPY . /python-magic -CMD cd /python-magic/test && python3 ./run.py + +WORKDIR /python-magic +COPY . . +RUN python3 -m pip install tox diff --git a/test/docker/centos7 b/test/docker/centos7 index f2ac6e40..9caa9898 100644 --- a/test/docker/centos7 +++ b/test/docker/centos7 @@ -1,5 +1,8 @@ FROM centos:7 RUN yum -y update RUN yum -y install file-devel python3 python2 which -COPY . /python-magic -CMD cd /python-magic/test && SKIP_FROM_DESCRIPTOR=1 python3 ./run.py +ENV SKIP_FROM_DESCRIPTOR=1 + +WORKDIR /python-magic +COPY . . +RUN python3 -m pip install tox diff --git a/test/docker/centos8 b/test/docker/centos8 index 968f6b65..7f2dbd06 100644 --- a/test/docker/centos8 +++ b/test/docker/centos8 @@ -4,6 +4,7 @@ RUN yum -y install file-libs python3 python2 which glibc-locale-source RUN yum reinstall glibc-common -y && \ localedef -i en_US -f UTF-8 en_US.UTF-8 && \ echo "LANG=en_US.UTF-8" > /etc/locale.conf -ENV LANG en_US.UTF-8 -COPY . /python-magic -CMD cd /python-magic/test && python3 ./run.py + +WORKDIR /python-magic +COPY . . +RUN python3 -m pip install tox diff --git a/test/docker/focal b/test/docker/focal index 74e4d78a..f24d2317 100755 --- a/test/docker/focal +++ b/test/docker/focal @@ -1,8 +1,10 @@ FROM ubuntu:focal RUN apt-get update -RUN apt-get -y install python2 -RUN apt-get -y install python3 -RUN apt-get -y install locales +RUN apt-get -y install python python3 locales python3-pip libmagic1 RUN locale-gen en_US.UTF-8 -COPY . /python-magic -CMD cd /python-magic/test && python3 ./run.py + +WORKDIR /python-magic +COPY . . +RUN python3 -m pip install tox + + diff --git a/test/docker/xenial b/test/docker/xenial index bc0440be..fe7829be 100755 --- a/test/docker/xenial +++ b/test/docker/xenial @@ -1,8 +1,8 @@ FROM ubuntu:xenial RUN apt-get update -RUN apt-get -y install python -RUN apt-get -y install python3 -RUN apt-get -y install locales +RUN apt-get -y install python python3 locales python3-pip libmagic1 RUN locale-gen en_US.UTF-8 -COPY . /python-magic -CMD cd /python-magic/test && python3 ./run.py + +WORKDIR /python-magic +COPY . . +RUN python3 -m pip install tox diff --git a/test/libmagic_test.py b/test/libmagic_test.py index 88f1254e..fff71cda 100644 --- a/test/libmagic_test.py +++ b/test/libmagic_test.py @@ -3,15 +3,23 @@ import unittest import os import magic +import os.path # magic_descriptor is broken (?) in centos 7, so don't run those tests -SKIP_FROM_DESCRIPTOR = bool(os.environ.get('SKIP_FROM_DESCRIPTOR')) +SKIP_FROM_DESCRIPTOR = bool(os.environ.get("SKIP_FROM_DESCRIPTOR")) + +TESTDATA_DIR = os.path.abspath(os.path.join(os.path.dirname(__file__), "testdata")) + class MagicTestCase(unittest.TestCase): - filename = 'testdata/test.pdf' - expected_mime_type = 'application/pdf' - expected_encoding = 'us-ascii' - expected_name = ('PDF document, version 1.2', 'PDF document, version 1.2, 2 pages') + filename = os.path.join(TESTDATA_DIR, "test.pdf") + expected_mime_type = "application/pdf" + expected_encoding = "us-ascii" + expected_name = ( + "PDF document, version 1.2", + "PDF document, version 1.2, 2 pages", + "PDF document, version 1.2, 2 page(s)", + ) def assert_result(self, result): self.assertEqual(result.mime_type, self.expected_mime_type) @@ -23,11 +31,9 @@ def test_detect_from_filename(self): self.assert_result(result) def test_detect_from_fobj(self): - if SKIP_FROM_DESCRIPTOR: self.skipTest("magic_descriptor is broken in this version of libmagic") - with open(self.filename) as fobj: result = magic.detect_from_fobj(fobj) self.assert_result(result) @@ -37,10 +43,10 @@ def test_detect_from_content(self): # this avoids hitting a bug in python3+libfile bindings # see https://github.com/ahupp/python-magic/issues/152 # for a similar issue - with open(self.filename, 'rb') as fobj: + with open(self.filename, "rb") as fobj: result = magic.detect_from_content(fobj.read(4096)) self.assert_result(result) -if __name__ == '__main__': +if __name__ == "__main__": unittest.main() diff --git a/test/python_magic_test.py b/test/python_magic_test.py new file mode 100755 index 00000000..26398614 --- /dev/null +++ b/test/python_magic_test.py @@ -0,0 +1,351 @@ +from dataclasses import dataclass +from enum import Enum +import os +import os.path +import shutil +import sys +import tempfile +from typing import List, Union +import unittest + +import pytest + +try: + from concurrent.futures import ThreadPoolExecutor + HAS_CONCURRENT_FUTURES = True +except ImportError: # python 2.7 + HAS_CONCURRENT_FUTURES = False + +# for output which reports a local time +os.environ["TZ"] = "GMT" + +if os.environ.get("LC_ALL", "") != "en_US.UTF-8": + # this ensure we're in a utf-8 default filesystem encoding which is + # necessary for some tests + raise Exception("must run `export LC_ALL=en_US.UTF-8` before running test suite") + +import magic + + +@dataclass +class TestFile: + file_name: str + mime_results: List[str] + text_results: List[str] + no_check_elf_results: Union[List[str], None] + buf_equals_file: bool = True + + +# magic_descriptor is broken (?) in centos 7, so don't run those tests +SKIP_FROM_DESCRIPTOR = bool(os.environ.get("SKIP_FROM_DESCRIPTOR")) + + +COMMON_PLAIN = [{}] +NO_SOFT = [{"check_soft": False}] +COMMON_MIME = [{"mime": True}] + +CASES = { + b"magic._pyc_": [ + ( + COMMON_MIME, + [ + "application/octet-stream", + "text/x-bytecode.python", + "application/x-bytecode.python", + ], + ), + (COMMON_PLAIN, ["python 2.4 byte-compiled"]), + (NO_SOFT, ["data"]), + ], + b"test.pdf": [ + (COMMON_MIME, ["application/pdf"]), + ( + COMMON_PLAIN, + [ + "PDF document, version 1.2", + "PDF document, version 1.2, 2 pages", + "PDF document, version 1.2, 2 page(s)", + ], + ), + (NO_SOFT, ["ASCII text"]), + ], + b"test.gz": [ + (COMMON_MIME, ["application/gzip", "application/x-gzip"]), + ( + COMMON_PLAIN, + [ + 'gzip compressed data, was "test", from Unix, last modified: Sun Jun 29 01:32:52 2008', + 'gzip compressed data, was "test", last modified: Sun Jun 29 01:32:52 2008, from Unix', + 'gzip compressed data, was "test", last modified: Sun Jun 29 01:32:52 2008, from Unix, original size 15', + 'gzip compressed data, was "test", last modified: Sun Jun 29 01:32:52 2008, from Unix, original size modulo 2^32 15', + 'gzip compressed data, was "test", last modified: Sun Jun 29 01:32:52 2008, from Unix, truncated', + ], + ), + ( + [{"extension": True}], + [ + # some versions return '' for the extensions of a gz file, + # including w/ the command line. Who knows... + "gz/tgz/tpz/zabw/svgz/adz/kmy/xcfgz", + "gz/tgz/tpz/zabw/svgz", + "", + "???", + ], + ), + (NO_SOFT, ["data"]), + ], + b"test.snappy.parquet": [ + (COMMON_MIME, ["application/octet-stream", "application/vnd.apache.parquet"]), + (COMMON_PLAIN, ["Apache Parquet", "Apache Parquet file", "Par archive data"]), + (NO_SOFT, ["data"]), + ], + b"test.json": [ + (COMMON_MIME, ["application/json"]), + (COMMON_PLAIN, ["JSON text data"]), + ( + [{"mime": True, "check_json": False}], + [ + "text/plain", + ], + ), + (NO_SOFT, ["JSON text data"]), + ], + b"elf-NetBSD-x86_64-echo": [ + # TODO: soft, no elf + ( + COMMON_PLAIN, + [ + "ELF 64-bit LSB shared object, x86-64, version 1 (SYSV)", + "ELF 64-bit LSB pie executable, x86-64, version 1 (SYSV), dynamically linked, interpreter /libexec/ld.elf_so, for NetBSD 8.0, not stripped", + ], + ), + ( + COMMON_MIME, + [ + "application/x-pie-executable", + "application/x-sharedlib", + ], + ), + ( + [{"check_elf": False}], + [ + "ELF 64-bit LSB shared object, x86-64, version 1 (SYSV)", + ], + ), + # TODO: sometimes + # "ELF 64-bit LSB pie executable, x86-64, version 1 (SYSV), dynamically linked, interpreter /libexec/ld.elf_so, for NetBSD 8.0, not stripped", + (NO_SOFT, ["data"]), + ], + b"text.txt": [ + (COMMON_MIME, ["text/plain"]), + (COMMON_PLAIN, ["ASCII text"]), + ( + [{"mime_encoding": True}], + [ + "us-ascii", + ], + ), + (NO_SOFT, ["ASCII text"]), + ], + b"text-iso8859-1.txt": [ + ( + [{"mime_encoding": True}], + [ + "iso-8859-1", + ], + ), + ], + b"\xce\xbb": [ + (COMMON_MIME, ["text/plain"]), + ], + b"name_use.jpg": [ + ([{"extension": True}], ["jpeg/jpg/jpe/jfif"]), + ], + b"keep-going.jpg": [ + (COMMON_MIME, ["image/jpeg"]), + ( + [{"mime": True, "keep_going": True}], + [ + "image/jpeg\\012- application/octet-stream", + ], + ), + ], + b"../../magic/loader.py": [ + ( + COMMON_MIME, + [ + "text/x-python", + "text/x-script.python", + ], + ) + ], +} + + +class MagicTest(unittest.TestCase): + TESTDATA_DIR = os.path.abspath(os.path.join(os.path.dirname(__file__), "testdata")) + + def test_version(self): + try: + self.assertTrue(magic.version() > 0) + except NotImplementedError: + pass + + def test_fs_encoding(self): + self.assertEqual("utf-8", sys.getfilesystemencoding().lower()) + + def test_from_file_str_and_bytes(self): + filename = os.path.join(self.TESTDATA_DIR, "test.pdf") + + self.assertEqual("application/pdf", magic.from_file(filename, mime=True)) + self.assertEqual( + "application/pdf", magic.from_file(filename.encode("utf-8"), mime=True) + ) + + def test_all_cases(self): + # TODO: + # * MAGIC_EXTENSION not supported + # * keep_going not supported + # * buffer checks + dest = os.path.join(MagicTest.TESTDATA_DIR, b"\xce\xbb".decode("utf-8")) + shutil.copyfile(os.path.join(MagicTest.TESTDATA_DIR, "lambda"), dest) + os.environ["TZ"] = "UTC" + try: + for filename, cases in CASES.items(): + filename = os.path.join(self.TESTDATA_DIR.encode("utf-8"), filename) + print("test case ", filename, file=sys.stderr) + for flag_variants, outputs in cases: + for flags in flag_variants: + print("flags", flags, file=sys.stderr) + m = magic.Magic(**flags) + with open(filename) as f: + self.assertIn(m.from_descriptor(f.fileno()), outputs) + + self.assertIn(m.from_file(filename), outputs) + + fname_str = filename.decode("utf-8") + self.assertIn(m.from_file(fname_str), outputs) + + with open(filename, "rb") as f: + buf_result = m.from_buffer(f.read(1024)) + self.assertIn(buf_result, outputs) + finally: + del os.environ["TZ"] + os.unlink(dest) + + def test_unicode_result_nonraw(self): + m = magic.Magic(raw=False) + src = os.path.join(MagicTest.TESTDATA_DIR, "pgpunicode") + result = m.from_file(src) + # NOTE: This check is added as otherwise some magic files don't identify the test case as a PGP key. + if "PGP" in result: + assert r"PGP\011Secret Sub-key -" == result + else: + raise unittest.SkipTest("Magic file doesn't return expected type.") + + def test_unicode_result_raw(self): + m = magic.Magic(raw=True) + src = os.path.join(MagicTest.TESTDATA_DIR, "pgpunicode") + result = m.from_file(src) + if "PGP" in result: + assert b"PGP\tSecret Sub-key -" == result.encode("utf-8") + else: + raise unittest.SkipTest("Magic file doesn't return expected type.") + + def test_errors(self): + m = magic.Magic() + self.assertRaises(IOError, m.from_file, "nonexistent") + self.assertRaises(magic.MagicException, magic.Magic, magic_file="nonexistent") + os.environ["MAGIC"] = "nonexistent" + try: + self.assertRaises(magic.MagicException, magic.Magic) + finally: + del os.environ["MAGIC"] + + def test_rethrow(self): + old = magic.magic_buffer + try: + + def t(x, y): + raise magic.MagicException("passthrough") + + magic.magic_buffer = t + + with self.assertRaises(magic.MagicException): + magic.from_buffer("hello", True) + finally: + magic.magic_buffer = old + + def test_getparam(self): + m = magic.Magic(mime=True) + try: + m.setparam(magic.MAGIC_PARAM_INDIR_MAX, 1) + self.assertEqual(m.getparam(magic.MAGIC_PARAM_INDIR_MAX), 1) + except NotImplementedError: + pass + + def test_name_count(self): + m = magic.Magic() + with open(os.path.join(self.TESTDATA_DIR, "name_use.jpg"), "rb") as f: + m.from_buffer(f.read()) + + def test_pathlike(self): + if sys.version_info < (3, 6): + return + from pathlib import Path + + path = Path(self.TESTDATA_DIR, "test.pdf") + m = magic.Magic(mime=True) + self.assertEqual("application/pdf", m.from_file(path)) + + def test_symlink(self): + # TODO: 3.0 + if not hasattr(tempfile, "TemporaryDirectory"): + return + + with tempfile.TemporaryDirectory() as tmp: + tmp_link = os.path.join(tmp, "test_link") + tmp_broken = os.path.join(tmp, "nonexistent") + + os.symlink( + os.path.join(self.TESTDATA_DIR, "test.pdf"), + tmp_link, + ) + + os.symlink("/nonexistent", tmp_broken) + + m = magic.Magic() + m_follow = magic.Magic(follow_symlinks=True) + self.assertTrue(m.from_file(tmp_link).startswith("symbolic link to ")) + self.assertTrue(m_follow.from_file(tmp_link).startswith("PDF document")) + + self.assertTrue( + m.from_file(tmp_broken).startswith( + "broken symbolic link to /nonexistent" + ) + ) + + self.assertRaises(IOError, m_follow.from_file, tmp_broken) + + @unittest.skipIf(not HAS_CONCURRENT_FUTURES, "concurrent.futures not available in Python 2.7") + def test_thread_safety(self): + """Test that concurrent from_file calls don't crash (would SEGV without global lock)""" + filename = os.path.join(self.TESTDATA_DIR, "test.pdf") + + m = magic.Magic(mime=True) + + def check_file(_): + result = m.from_file(filename) + self.assertEqual(result, "application/pdf") + return result + + with ThreadPoolExecutor(100) as executor: + results = list(executor.map(check_file, range(100))) + + # All calls should complete successfully + self.assertEqual(len(results), 100) + self.assertTrue(all(r == "application/pdf" for r in results)) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/run.py b/test/run.py deleted file mode 100644 index cf62eee7..00000000 --- a/test/run.py +++ /dev/null @@ -1,35 +0,0 @@ -import subprocess -import os.path -import sys - -this_dir = os.path.dirname(sys.argv[0]) - -new_env = dict(os.environ) -new_env.update({ - 'LC_ALL': 'en_US.UTF-8', - 'PYTHONPATH': os.path.join(this_dir, ".."), -}) - - -def has_py(version): - ret = subprocess.run("which %s" % version, shell=True, stdout=subprocess.DEVNULL) - return ret.returncode == 0 - - -def run_test(versions): - found = False - for i in versions: - if not has_py(i): - # if this version doesn't exist in path, skip - continue - found = True - print("Testing %s" % i) - subprocess.run([i, os.path.join(this_dir, "test.py")], env=new_env, check=True) - subprocess.run([i, os.path.join(this_dir, "libmagic_test.py")], env=new_env, check=True) - - if not found: - sys.exit("No versions found: " + str(versions)) - -run_test(["python2", "python2.7"]) -run_test(["python3.5", "python3.6", "python3.7", "python3.8", "python3.9"]) - diff --git a/test/run_all_docker_test.sh b/test/run_all_docker_test.sh new file mode 100755 index 00000000..dce930b7 --- /dev/null +++ b/test/run_all_docker_test.sh @@ -0,0 +1,13 @@ +#!/bin/sh + +set -e +set -x + +ROOT=$(dirname $0)/.. +cd $ROOT + +for f in test/docker/*; do + H=$(docker build -q -f ${f} .) + docker run --rm $H python3 -m tox +done + diff --git a/test/test.py b/test/test.py deleted file mode 100755 index 0c4621c5..00000000 --- a/test/test.py +++ /dev/null @@ -1,232 +0,0 @@ -import os - -# for output which reports a local time -os.environ['TZ'] = 'GMT' - -if os.environ.get('LC_ALL', '') != 'en_US.UTF-8': - # this ensure we're in a utf-8 default filesystem encoding which is - # necessary for some tests - raise Exception("must run `export LC_ALL=en_US.UTF-8` before running test suite") - -import shutil -import os.path -import unittest - -import magic -import sys - -# magic_descriptor is broken (?) in centos 7, so don't run those tests -SKIP_FROM_DESCRIPTOR = bool(os.environ.get('SKIP_FROM_DESCRIPTOR')) - -class MagicTest(unittest.TestCase): - TESTDATA_DIR = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'testdata') - - def test_version(self): - try: - self.assertTrue(magic.version() > 0) - except NotImplementedError: - pass - - def test_fs_encoding(self): - self.assertEqual('utf-8', sys.getfilesystemencoding().lower()) - - def assert_values(self, m, expected_values, buf_equals_file=True): - for filename, expected_value in expected_values.items(): - try: - filename = os.path.join(self.TESTDATA_DIR, filename) - except TypeError: - filename = os.path.join( - self.TESTDATA_DIR.encode('utf-8'), filename) - - if type(expected_value) is not tuple: - expected_value = (expected_value,) - - with open(filename, 'rb') as f: - buf_value = m.from_buffer(f.read()) - - file_value = m.from_file(filename) - - if buf_equals_file: - self.assertEqual(buf_value, file_value) - - for value in (buf_value, file_value): - self.assertIn(value, expected_value) - - def test_from_file_str_and_bytes(self): - filename = os.path.join(self.TESTDATA_DIR, "test.pdf") - - self.assertEqual('application/pdf', - magic.from_file(filename, mime=True)) - self.assertEqual('application/pdf', - magic.from_file(filename.encode('utf-8'), mime=True)) - - def test_from_descriptor_str_and_bytes(self): - if SKIP_FROM_DESCRIPTOR: - self.skipTest("magic_descriptor is broken in this version of libmagic") - - filename = os.path.join(self.TESTDATA_DIR, "test.pdf") - with open(filename) as f: - self.assertEqual('application/pdf', - magic.from_descriptor(f.fileno(), mime=True)) - self.assertEqual('application/pdf', - magic.from_descriptor(f.fileno(), mime=True)) - - def test_from_buffer_str_and_bytes(self): - if SKIP_FROM_DESCRIPTOR: - self.skipTest("magic_descriptor is broken in this version of libmagic") - m = magic.Magic(mime=True) - - self.assertTrue( - m.from_buffer('#!/usr/bin/env python\nprint("foo")') - in ("text/x-python", "text/x-script.python")) - self.assertTrue( - m.from_buffer(b'#!/usr/bin/env python\nprint("foo")') - in ("text/x-python", "text/x-script.python")) - - def test_mime_types(self): - dest = os.path.join(MagicTest.TESTDATA_DIR, - b'\xce\xbb'.decode('utf-8')) - shutil.copyfile(os.path.join(MagicTest.TESTDATA_DIR, 'lambda'), dest) - try: - m = magic.Magic(mime=True) - self.assert_values(m, { - 'magic._pyc_': ('application/octet-stream', 'text/x-bytecode.python'), - 'test.pdf': 'application/pdf', - 'test.gz': ('application/gzip', 'application/x-gzip'), - 'test.snappy.parquet': 'application/octet-stream', - 'text.txt': 'text/plain', - b'\xce\xbb'.decode('utf-8'): 'text/plain', - b'\xce\xbb': 'text/plain', - }) - finally: - os.unlink(dest) - - def test_descriptions(self): - m = magic.Magic() - os.environ['TZ'] = 'UTC' # To get last modified date of test.gz in UTC - try: - self.assert_values(m, { - 'magic._pyc_': 'python 2.4 byte-compiled', - 'test.pdf': ('PDF document, version 1.2', - 'PDF document, version 1.2, 2 pages'), - 'test.gz': - ('gzip compressed data, was "test", from Unix, last ' - 'modified: Sun Jun 29 01:32:52 2008', - 'gzip compressed data, was "test", last modified' - ': Sun Jun 29 01:32:52 2008, from Unix', - 'gzip compressed data, was "test", last modified' - ': Sun Jun 29 01:32:52 2008, from Unix, original size 15', - 'gzip compressed data, was "test", ' - 'last modified: Sun Jun 29 01:32:52 2008, ' - 'from Unix, original size modulo 2^32 15', - 'gzip compressed data, was "test", last modified' - ': Sun Jun 29 01:32:52 2008, from Unix, truncated' - ), - 'text.txt': 'ASCII text', - 'test.snappy.parquet': ('Apache Parquet', 'Par archive data'), - }, buf_equals_file=False) - finally: - del os.environ['TZ'] - - def test_extension(self): - try: - m = magic.Magic(extension=True) - self.assert_values(m, { - # some versions return '' for the extensions of a gz file, - # including w/ the command line. Who knows... - 'test.gz': ('gz/tgz/tpz/zabw/svgz', '', '???'), - 'name_use.jpg': 'jpeg/jpg/jpe/jfif', - }) - except NotImplementedError: - self.skipTest('MAGIC_EXTENSION not supported in this version') - - def test_unicode_result_nonraw(self): - m = magic.Magic(raw=False) - src = os.path.join(MagicTest.TESTDATA_DIR, 'pgpunicode') - result = m.from_file(src) - # NOTE: This check is added as otherwise some magic files don't identify the test case as a PGP key. - if 'PGP' in result: - assert r"PGP\011Secret Sub-key -" == result - else: - raise unittest.SkipTest("Magic file doesn't return expected type.") - - def test_unicode_result_raw(self): - m = magic.Magic(raw=True) - src = os.path.join(MagicTest.TESTDATA_DIR, 'pgpunicode') - result = m.from_file(src) - if 'PGP' in result: - assert b'PGP\tSecret Sub-key -' == result.encode('utf-8') - else: - raise unittest.SkipTest("Magic file doesn't return expected type.") - - def test_mime_encodings(self): - m = magic.Magic(mime_encoding=True) - self.assert_values(m, { - 'text-iso8859-1.txt': 'iso-8859-1', - 'text.txt': 'us-ascii', - }) - - def test_errors(self): - m = magic.Magic() - self.assertRaises(IOError, m.from_file, 'nonexistent') - self.assertRaises(magic.MagicException, magic.Magic, - magic_file='nonexistent') - os.environ['MAGIC'] = 'nonexistent' - try: - self.assertRaises(magic.MagicException, magic.Magic) - finally: - del os.environ['MAGIC'] - - def test_keep_going(self): - filename = os.path.join(self.TESTDATA_DIR, 'keep-going.jpg') - - m = magic.Magic(mime=True) - self.assertEqual(m.from_file(filename), 'image/jpeg') - - try: - # this will throw if you have an "old" version of the library - # I'm otherwise not sure how to query if keep_going is supported - magic.version() - m = magic.Magic(mime=True, keep_going=True) - self.assertEqual(m.from_file(filename), - 'image/jpeg\\012- application/octet-stream') - except NotImplementedError: - pass - - def test_rethrow(self): - old = magic.magic_buffer - try: - def t(x, y): - raise magic.MagicException("passthrough") - - magic.magic_buffer = t - - with self.assertRaises(magic.MagicException): - magic.from_buffer("hello", True) - finally: - magic.magic_buffer = old - - def test_getparam(self): - m = magic.Magic(mime=True) - try: - m.setparam(magic.MAGIC_PARAM_INDIR_MAX, 1) - self.assertEqual(m.getparam(magic.MAGIC_PARAM_INDIR_MAX), 1) - except NotImplementedError: - pass - - def test_name_count(self): - m = magic.Magic() - with open(os.path.join(self.TESTDATA_DIR, 'name_use.jpg'), 'rb') as f: - m.from_buffer(f.read()) - - def test_pathlike(self): - if sys.version_info < (3, 6): - return - from pathlib import Path - path = Path(self.TESTDATA_DIR, "test.pdf") - m = magic.Magic(mime=True) - self.assertEqual('application/pdf', m.from_file(path)) - - -if __name__ == '__main__': - unittest.main() diff --git a/test/testdata/elf-NetBSD-x86_64-echo b/test/testdata/elf-NetBSD-x86_64-echo new file mode 100644 index 00000000..74affab9 Binary files /dev/null and b/test/testdata/elf-NetBSD-x86_64-echo differ diff --git a/test/testdata/test.json b/test/testdata/test.json new file mode 100644 index 00000000..cbd40300 --- /dev/null +++ b/test/testdata/test.json @@ -0,0 +1,7 @@ +[ + { + "one": 2, + "three": null, + "four": [5, "six", false] + } +] diff --git a/test_docker.sh b/test_docker.sh deleted file mode 100755 index ad2bc5d5..00000000 --- a/test_docker.sh +++ /dev/null @@ -1,18 +0,0 @@ -#!/bin/bash - -# Test with various versions of ubuntu. This more or less re-creates the -# Travis CI test environment - -set -e - -DEFAULT_TARGETS="xenial bionic focal centos7 centos8 archlinux alpine" - -TARGETS=${1:-${DEFAULT_TARGETS}} - -HERE=`dirname $0` - -for i in $TARGETS; do - TAG="python_magic/${i}:latest" - docker build -t $TAG -f ${HERE}/test/docker/$i . - docker run $TAG -done diff --git a/tox.ini b/tox.ini index 65595983..01cb7b23 100644 --- a/tox.ini +++ b/tox.ini @@ -1,26 +1,30 @@ [tox] envlist = - coverage-clean, py27, py35, py36, py37, py38, py39, - coverage-report, + py310, + py311, + py312, + py313, + py314, + py314t, mypy [testenv] commands = - coverage run --source=magic ./test/test.py + coverage run -m pytest setenv = COVERAGE_FILE=.coverage.{envname} - LC_ALL=en_US.UTF-8 + LC_ALL=en_US.UTF-8 deps = .[test] - zope.testrunner coverage + pytest [testenv:coverage-clean] deps = coverage @@ -44,4 +48,5 @@ commands = deps = mypy skip_install = true commands = - mypy magic.pyi + mypy -p magic +