diff --git a/.dockerignore b/.dockerignore new file mode 120000 index 00000000..3e4e48b0 --- /dev/null +++ b/.dockerignore @@ -0,0 +1 @@ +.gitignore \ No newline at end of file diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 00000000..be006de9 --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,13 @@ +# Keep GitHub Actions up to date with GitHub's Dependabot... +# https://docs.github.com/en/code-security/dependabot/working-with-dependabot/keeping-your-actions-up-to-date-with-dependabot +# https://docs.github.com/en/code-security/dependabot/dependabot-version-updates/configuration-options-for-the-dependabot.yml-file#package-ecosystem +version: 2 +updates: + - package-ecosystem: github-actions + directory: / + groups: + github-actions: + patterns: + - "*" # Group all Actions updates into a single larger pull request + schedule: + interval: weekly diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 00000000..772132e5 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,31 @@ +name: ci +on: [push, pull_request] +jobs: + ci: + strategy: + fail-fast: false + matrix: + os: ["ubuntu-latest"] + python-version: ["3.8", "3.9", "3.10", "3.11", "3.12", "3.13", "3.14", "3.14t"] + include: + - os: macos-latest + python-version: "3.x" + # - os: windows-latest # TODO: Fix the Windows test that runs in an infinite loop + # python-version: '3.13' + runs-on: ${{ matrix.os }} + steps: + - uses: actions/checkout@v7 + - uses: actions/setup-python@v7 + with: + python-version: ${{ matrix.python-version }} + allow-prereleases: true + - run: pip install --upgrade pip + - run: pip install --upgrade pytest + - run: pip install --editable . + - if: runner.os == 'macOS' + run: brew install libmagic + - if: runner.os == 'Windows' + run: pip install python-magic-bin + - run: LC_ALL=en_US.UTF-8 pytest + shell: bash + timeout-minutes: 15 # Limit Windows infinite loop. diff --git a/.gitignore b/.gitignore index 0346a859..1f961bbb 100644 --- a/.gitignore +++ b/.gitignore @@ -4,9 +4,11 @@ bin/ deb_dist htmlcov/ lib/ -__pycache__/ +**/__pycache__ python_magic.egg-info pip-selfcheck.json pyvenv.cfg *.pyc *~ +dist/ +.vscode/ diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index 8c306d9a..00000000 --- a/.travis.yml +++ /dev/null @@ -1,22 +0,0 @@ -language: python -dist: xenial -cache: pip - -python: - - "2.7" - - "3.5" - - "3.6" - - "3.7" - - "3.8" - - "3.9" - -install: - - pip install coverage coveralls codecov - - pip install . - -script: - - LC_ALL=en_US.UTF-8 coverage run -m unittest test - -after_success: - - coveralls - - codecov diff --git a/CHANGELOG b/CHANGELOG index fbabad62..a8370c68 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -1,13 +1,51 @@ +Changes to 0.4.29: + +- support MAGIC_SYMLINK (via follow_symlink flag on Magic constructor) +- correctly throw FileNotFoundException depending on flag + +Changes to 0.4.28: + +- support "magic-1.dll" on Windows, which is produced by vcpkg +- add python 3.10 to tox config +- update test for upstream gzip extensions + +Changes to 0.4.27: + +- remove spurious pyproject.toml that breaks source builds + +Changes to 0.4.26: + +- Use tox for all multi-version testing +- Fix use of pytest, use it via tox + +Changes to 0.4.25: + +- Support os.PathLike values in Magic.from_file and magic.from_file +- Handle some versions of libmagic that return mime string without charset +- Fix tests for file 5.41 +- Include typing stub in package + +Changes to 0.4.24: + +- Fix regression in library loading on some Alpine docker images. + +Changes to 0.4.23 + +- Include a `py.typed` sentinel to enable type checking +- Improve fix for attribute error during destruction +- Cleanup library loading logic +- Add new homebrew library dir for OSX + Changes to 0.4.21, 0.4.22 - - Unify dll loader between the standard and compat library, fixing load - failures on some previously supported platforms. +- Unify dll loader between the standard and compat library, fixing load + failures on some previously supported platforms. Changes to 0.4.20 -- merge in a compatability layer for the upstream libmagic python binding. +- merge in a compatibility layer for the upstream libmagic python binding. Since both this package and that one are called 'magic', this compat layer - removes a very common source of runtime errors. Use of that libmagic API will + removes a very common source of runtime errors. Use of that libmagic API will produce a deprecation warning. - support python 3.9 in tests and pypi metadata @@ -16,9 +54,9 @@ Changes to 0.4.20 rather than a filename. - sometimes the returned description includes snippets of the file, e.g a title - for MS Word docs. Since this is in an unknown encoding, we would throw a - unicode decode error trying to decode. Now, it decodes with - 'backslashreplace' to handle this more gracefully. The undecodable characters + for MS Word docs. Since this is in an unknown encoding, we would throw a + unicode decode error trying to decode. Now, it decodes with + 'backslashreplace' to handle this more gracefully. The undecodable characters are replaced with hex escapes. - add support for MAGIC_EXTENSION, to return possible file extensions. @@ -27,18 +65,18 @@ Changes to 0.4.20 Changes in 0.4.18 -- Make bindings for magic_[set|get]param optional, and throw NotImplementedError -if they are used but not supported. Only call setparam() in the constructor if -it's supported. This prevents breakage on CentOS7 which uses an old version of -libmagic. +- Make bindings for magic\_[set|get]param optional, and throw NotImplementedError + if they are used but not supported. Only call setparam() in the constructor if + it's supported. This prevents breakage on CentOS7 which uses an old version of + libmagic. - Add tests for CentOS 7 & 8 Changes in 0.4.16 and 0.4.17 - add MAGIC_MIME_TYPE constant, use that in preference to MAGIC_MIME internally. -This sets up for a breaking change in a future major version bump where -MAGIC_MIME will change to mathch magic.h. + This sets up for a breaking change in a future major version bump where + MAGIC_MIME will change to match magic.h. - add magic.version() function to return library version - add setparam/getparam to control internal behavior - increase internal limits with setparam to prevent spurious error on some jpeg files @@ -48,12 +86,12 @@ MAGIC_MIME will change to mathch magic.h. - include tests in source distribution - many test improvements: --- tox runner support --- remove deprecated test_suite field from setup.py --- docker tests that cover all LTS ubuntu versions --- add test for snapp file identification + -- tox runner support + -- remove deprecated test_suite field from setup.py + -- docker tests that cover all LTS ubuntu versions + -- add test for snapp file identification - doc improvements --- document dependency install process for debian --- various typos --- document test running process + -- document dependency install process for debian + -- various typos + -- document test running process diff --git a/COMPAT.md b/COMPAT.md index 21b35a17..921abafa 100644 --- a/COMPAT.md +++ b/COMPAT.md @@ -1,10 +1,10 @@ There are two python modules named 'magic' that do the same thing, but with incompatible APIs. One of these ships with libmagic, and (this one) is distributed through pypi. Both have been around for many years and have -substantial user bases. This incompatability is a major source of pain for +substantial user bases. This incompatibility is a major source of pain for users, and bug reports for me. -To mitigate this pain, python-magic has added a compatability layer to export +To mitigate this pain, python-magic has added a compatibility layer to export the libmagic python API parallel to the existing one. The mapping between the libmagic and python-magic functions is: diff --git a/README.md b/README.md index 378594ba..c55f87c1 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,7 @@ # python-magic [![PyPI version](https://badge.fury.io/py/python-magic.svg)](https://badge.fury.io/py/python-magic) -[![Build Status](https://travis-ci.org/ahupp/python-magic.svg?branch=master)](https://travis-ci.org/ahupp/python-magic) +[![ci](https://github.com/ahupp/python-magic/actions/workflows/ci.yml/badge.svg)](https://github.com/ahupp/python-magic/actions/workflows/ci.yml) +[![Join the chat at https://gitter.im/ahupp/python-magic](https://badges.gitter.im/ahupp/python-magic.svg)](https://gitter.im/ahupp/python-magic?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge) python-magic is a Python interface to the libmagic file type identification library. libmagic identifies file types by checking @@ -45,7 +46,10 @@ You can also combine the flag options: ## Installation The current stable version of python-magic is available on PyPI and -can be installed by running `pip install python-magic`. +can be installed by running: +``` +pip install python-magic +``` Other sources: @@ -61,19 +65,17 @@ that must be installed as well: sudo apt-get install libmagic1 ``` -### Windows - -You'll need DLLs for libmagic. @julian-r maintains a pypi package with the DLLs, you can fetch it with: - -``` -pip install python-magic-bin -``` - ### OSX - When using Homebrew: `brew install libmagic` - When using macports: `port install file` +If python-magic fails to load the library it may be in a non-standard location, in which case you can set the environment variable `DYLD_LIBRARY_PATH` to point to it. + +### SmartOS: +- Install libmagic for source: https://github.com/file/file +- Depending on your ./configure --prefix settings set your LD_LIBRARY_PATH to /lib + ### Troubleshooting - 'MagicException: could not find any magic files!': some @@ -101,27 +103,28 @@ triage it. ## Running the tests -To run the tests across a variety of linux distributions (depends on Docker): +We use the `tox` test runner which can be installed with `python -m pip install tox`. + +To run tests locally across all available python versions: ``` -./test_docker.sh +python -m tox ``` -To run tests locally across all available python versions: +Or to run just against a single version: ``` -./test/run.py +python -m tox py ``` - -To run against a specific python version: +To run the tests across a variety of linux distributions (depends on Docker): ``` -LC_ALL=en_US.UTF-8 python3 test/test.py +./test/run_all_docker_test.sh ``` -## libmagic and python-magic +## libmagic python API compatibility -See [COMPAT.md](COMPAT.md) for a guide to libmagic / python-magic compatability. +The python bindings shipped with libmagic use a module name that conflicts with this package. To work around this, python-magic includes a compatibility layer for the libmagic API. See [COMPAT.md](COMPAT.md) for a guide to libmagic / python-magic compatibility. ## Versioning diff --git a/magic/__init__.py b/magic/__init__.py index 7a75b751..14d18968 100644 --- a/magic/__init__.py +++ b/magic/__init__.py @@ -17,17 +17,11 @@ """ import sys -import glob -import ctypes -import ctypes.util +import os import threading -import logging from ctypes import c_char_p, c_int, c_size_t, c_void_p, byref, POINTER -# avoid shadowing the real open with the version from compat.py -_real_open = open - class MagicException(Exception): def __init__(self, message): @@ -40,8 +34,27 @@ class Magic: Magic is a wrapper around the libmagic C library. """ - def __init__(self, mime=False, magic_file=None, mime_encoding=False, - keep_going=False, uncompress=False, raw=False, extension=False): + def __init__( + self, + mime=False, + magic_file=None, + mime_encoding=False, + keep_going=False, + uncompress=False, + raw=False, + extension=False, + follow_symlinks=False, + check_tar=True, + check_soft=True, + check_apptype=True, + check_elf=True, + check_text=True, + check_cdf=True, + check_csv=True, + check_encoding=True, + check_json=True, + check_simh=True, + ): """ Create a new libmagic wrapper. @@ -53,8 +66,6 @@ def __init__(self, mime=False, magic_file=None, mime_encoding=False, raw - Do not try to decode "non-printable" chars. extension - Print a slash-separated list of valid extensions for the file type found. """ - - self.cookie = None self.flags = MAGIC_NONE if mime: self.flags |= MAGIC_MIME_TYPE @@ -69,6 +80,30 @@ def __init__(self, mime=False, magic_file=None, mime_encoding=False, if extension: self.flags |= MAGIC_EXTENSION + if follow_symlinks: + self.flags |= MAGIC_SYMLINK + + if not check_tar: + self.flags |= MAGIC_NO_CHECK_TAR + if not check_soft: + self.flags |= MAGIC_NO_CHECK_SOFT + if not check_apptype: + self.flags |= MAGIC_NO_CHECK_APPTYPE + if not check_elf: + self.flags |= MAGIC_NO_CHECK_ELF + if not check_text: + self.flags |= MAGIC_NO_CHECK_TEXT + if not check_cdf: + self.flags |= MAGIC_NO_CHECK_CDF + if not check_csv: + self.flags |= MAGIC_NO_CHECK_CSV + if not check_encoding: + self.flags |= MAGIC_NO_CHECK_ENCODING + if not check_json: + self.flags |= MAGIC_NO_CHECK_JSON + if not check_simh: + self.flags |= MAGIC_NO_CHECK_SIMH + self.cookie = magic_open(self.flags) self.lock = threading.Lock() @@ -77,7 +112,9 @@ def __init__(self, mime=False, magic_file=None, mime_encoding=False, # MAGIC_EXTENSION was added in 523 or 524, so bail if # it doesn't appear to be available if extension and (not _has_version or version() < 524): - raise NotImplementedError('MAGIC_EXTENSION is not supported in this version of libmagic') + raise NotImplementedError( + "MAGIC_EXTENSION is not supported in this version of libmagic" + ) # For https://github.com/ahupp/python-magic/issues/190 # libmagic has fixed internal limits that some files exceed, causing @@ -102,16 +139,16 @@ def from_buffer(self, buf): # if we're on python3, convert buf to bytes # otherwise this string is passed as wchar* # which is not what libmagic expects + # NEXTBREAK: only take bytes if type(buf) == str and str != bytes: - buf = buf.encode('utf-8', errors='replace') + buf = buf.encode("utf-8", errors="replace") return maybe_decode(magic_buffer(self.cookie, buf)) except MagicException as e: return self._handle509Bug(e) def from_file(self, filename): # raise FileNotFoundException or IOError if the file does not exist - with _real_open(filename): - pass + os.stat(filename, follow_symlinks=self.flags & MAGIC_SYMLINK) with self.lock: try: @@ -152,7 +189,7 @@ def __del__(self): # incorrect fix for a threading problem, however I'm leaving # it in because it's harmless and I'm slightly afraid to # remove it. - if self.cookie and magic_close: + if hasattr(self, "cookie") and self.cookie and magic_close: magic_close(self.cookie) self.cookie = None @@ -168,7 +205,7 @@ def _get_magic_type(mime): def from_file(filename, mime=False): - """" + """ Accepts a filename and returns the detected filetype. Return value is the mimetype if mime=True, otherwise a human readable name. @@ -206,10 +243,12 @@ def from_descriptor(fd, mime=False): m = _get_magic_type(mime) return m.from_descriptor(fd) + from . import loader + libmagic = loader.load_lib() -magic_t = ctypes.c_void_p +magic_t = c_void_p def errorcheck_null(result, func, args): @@ -231,27 +270,45 @@ def errorcheck_negative_one(result, func, args): # return str on python3. Don't want to unconditionally # decode because that results in unicode on python2 def maybe_decode(s): + # NEXTBREAK: remove if str == bytes: return s else: # backslashreplace here because sometimes libmagic will return metadata in the charset # of the file, which is unknown to us (e.g the title of a Word doc) - return s.decode('utf-8', 'backslashreplace') + return s.decode("utf-8", "backslashreplace") + + +try: + from os import PathLike + + def unpath(filename): + if isinstance(filename, PathLike): + return filename.__fspath__() + else: + return filename +except ImportError: + + def unpath(filename): + return filename def coerce_filename(filename): if filename is None: return None + + filename = unpath(filename) + # ctypes will implicitly convert unicode strings to bytes with # .encode('ascii'). If you use the filesystem encoding # then you'll get inconsistent behavior (crashes) depending on the user's # LANG environment variable - is_unicode = (sys.version_info[0] <= 2 and - isinstance(filename, unicode)) or \ - (sys.version_info[0] >= 3 and - isinstance(filename, str)) + # NEXTBREAK: remove + is_unicode = (sys.version_info[0] <= 2 and isinstance(filename, unicode)) or ( + sys.version_info[0] >= 3 and isinstance(filename, str) + ) if is_unicode: - return filename.encode('utf-8', 'surrogateescape') + return filename.encode("utf-8", "surrogateescape") else: return filename @@ -330,7 +387,7 @@ def magic_load(cookie, filename): magic_compile.argtypes = [magic_t, c_char_p] _has_param = False -if hasattr(libmagic, 'magic_setparam') and hasattr(libmagic, 'magic_getparam'): +if hasattr(libmagic, "magic_setparam") and hasattr(libmagic, "magic_getparam"): _has_param = True _magic_setparam = libmagic.magic_setparam _magic_setparam.restype = c_int @@ -395,10 +452,16 @@ def version(): MAGIC_NO_CHECK_SOFT = 0x004000 # Don't check magic entries MAGIC_NO_CHECK_APPTYPE = 0x008000 # Don't check application type MAGIC_NO_CHECK_ELF = 0x010000 # Don't check for elf details -MAGIC_NO_CHECK_ASCII = 0x020000 # Don't check for ascii files -MAGIC_NO_CHECK_TROFF = 0x040000 # Don't check ascii/troff -MAGIC_NO_CHECK_FORTRAN = 0x080000 # Don't check ascii/fortran -MAGIC_NO_CHECK_TOKENS = 0x100000 # Don't check ascii/tokens +MAGIC_NO_CHECK_TEXT = 0x020000 # Don't check for ascii files +MAGIC_NO_CHECK_ASCII = 0x020000 # Deprecated alias for MAGIC_NO_CHECK_TEXT +MAGIC_NO_CHECK_TROFF = 0x040000 # Don't check ascii/troff (deprecated) +MAGIC_NO_CHECK_FORTRAN = 0x080000 # Don't check ascii/fortran (deprecated) +MAGIC_NO_CHECK_TOKENS = 0x100000 # Don't check ascii/tokens (deprecated) +MAGIC_NO_CHECK_CDF = 0x0040000 # Don't check for CDF files +MAGIC_NO_CHECK_CSV = 0x0080000 # Don't check for CSV files +MAGIC_NO_CHECK_ENCODING = 0x0200000 # Don't check text encodings +MAGIC_NO_CHECK_JSON = 0x0400000 # Don't check for JSON files +MAGIC_NO_CHECK_SIMH = 0x0800000 # Don't check for SIMH tape files MAGIC_PARAM_INDIR_MAX = 0 # Recursion limit for indirect magic MAGIC_PARAM_NAME_MAX = 1 # Use count limit for name/use magic @@ -420,24 +483,22 @@ def _add_compat(to_module): def deprecation_wrapper(fn): def _(*args, **kwargs): warnings.warn( - "Using compatability mode with libmagic's python binding. " + "Using compatibility mode with libmagic's python binding. " "See https://github.com/ahupp/python-magic/blob/master/COMPAT.md for details.", - PendingDeprecationWarning) + PendingDeprecationWarning, + ) return fn(*args, **kwargs) return _ - fn = ['detect_from_filename', - 'detect_from_content', - 'detect_from_fobj', - 'open'] + fn = ["detect_from_filename", "detect_from_content", "detect_from_fobj", "open"] for fname in fn: to_module[fname] = deprecation_wrapper(compat.__dict__[fname]) # copy constants over, ensuring there's no conflicts is_const_re = re.compile("^[A-Z_]+$") - allowed_inconsistent = set(['MAGIC_MIME']) + allowed_inconsistent = set(["MAGIC_MIME"]) for name, value in compat.__dict__.items(): if is_const_re.match(name): if name in to_module: diff --git a/magic/__init__.pyi b/magic/__init__.pyi index 8d5f38f5..bea800a4 100644 --- a/magic/__init__.pyi +++ b/magic/__init__.pyi @@ -1,6 +1,7 @@ import ctypes.util import threading from typing import Any, Text, Optional, Union +from os import PathLike class MagicException(Exception): message: Any = ... @@ -10,15 +11,33 @@ class Magic: flags: int = ... cookie: Any = ... lock: threading.Lock = ... - def __init__(self, mime: bool = ..., magic_file: Optional[Any] = ..., mime_encoding: bool = ..., keep_going: bool = ..., uncompress: bool = ..., raw: bool = ...) -> None: ... + def __init__( + self, + mime: bool = ..., + magic_file: Optional[Any] = ..., + mime_encoding: bool = ..., + keep_going: bool = ..., + uncompress: bool = ..., + raw: bool = ..., + extension: bool = ..., + follow_symlinks: bool = ..., + check_tar: bool = ..., + check_soft: bool = ..., + check_apptype: bool = ..., + check_elf: bool = ..., + check_text: bool = ..., + check_encoding: bool = ..., + check_json: bool = ..., + check_simh: bool = ..., + ) -> None: ... def from_buffer(self, buf: Union[bytes, str]) -> Text: ... - def from_file(self, filename: Union[bytes, str]) -> Text: ... + def from_file(self, filename: Union[bytes, str, PathLike]) -> Text: ... def from_descriptor(self, fd: int, mime: bool = ...) -> Text: ... def setparam(self, param: Any, val: Any): ... def getparam(self, param: Any): ... def __del__(self) -> None: ... -def from_file(filename: Union[bytes, str], mime: bool = ...) -> Text: ... +def from_file(filename: Union[bytes, str, PathLike], mime: bool = ...) -> Text: ... def from_buffer(buffer: Union[bytes, str], mime: bool = ...) -> Text: ... def from_descriptor(fd: int, mime: bool = ...) -> Text: ... @@ -73,10 +92,16 @@ MAGIC_NO_CHECK_TAR: int MAGIC_NO_CHECK_SOFT: int MAGIC_NO_CHECK_APPTYPE: int MAGIC_NO_CHECK_ELF: int +MAGIC_NO_CHECK_TEXT: int MAGIC_NO_CHECK_ASCII: int MAGIC_NO_CHECK_TROFF: int MAGIC_NO_CHECK_FORTRAN: int +MAGIC_NO_CHECK_CDF: int +MAGIC_NO_CHECK_CSV: int MAGIC_NO_CHECK_TOKENS: int +MAGIC_NO_CHECK_ENCODING: int +MAGIC_NO_CHECK_JSON: int +MAGIC_NO_CHECK_SIMH: int MAGIC_PARAM_INDIR_MAX: int MAGIC_PARAM_NAME_MAX: int MAGIC_PARAM_ELF_PHNUM_MAX: int diff --git a/magic/compat.py b/magic/compat.py index e2d71ee4..32a7b93b 100644 --- a/magic/compat.py +++ b/magic/compat.py @@ -4,13 +4,10 @@ Python bindings for libmagic ''' -import ctypes - +import threading from collections import namedtuple from ctypes import * -from ctypes.util import find_library - from . import loader @@ -45,13 +42,19 @@ MAGIC_NO_CHECK_BUILTIN = NO_CHECK_BUILTIN = 4173824 +MAGIC_PARAM_INDIR_MAX = PARAM_INDIR_MAX = 0 +MAGIC_PARAM_NAME_MAX = PARAM_NAME_MAX = 1 +MAGIC_PARAM_ELF_PHNUM_MAX = PARAM_ELF_PHNUM_MAX = 2 +MAGIC_PARAM_ELF_SHNUM_MAX = PARAM_ELF_SHNUM_MAX = 3 +MAGIC_PARAM_ELF_NOTES_MAX = PARAM_ELF_NOTES_MAX = 4 +MAGIC_PARAM_REGEX_MAX = PARAM_REGEX_MAX = 5 +MAGIC_PARAM_BYTES_MAX = PARAM_BYTES_MAX = 6 + FileMagic = namedtuple('FileMagic', ('mime_type', 'encoding', 'name')) class magic_set(Structure): pass - - magic_set._fields_ = [] magic_t = POINTER(magic_set) @@ -103,6 +106,14 @@ class magic_set(Structure): _errno.restype = c_int _errno.argtypes = [magic_t] +_getparam = _libraries['magic'].magic_getparam +_getparam.restype = c_int +_getparam.argtypes = [magic_t, c_int, c_void_p] + +_setparam = _libraries['magic'].magic_setparam +_setparam.restype = c_int +_setparam.argtypes = [magic_t, c_int, c_void_p] + class Magic(object): def __init__(self, ms): @@ -228,24 +239,81 @@ def errno(self): """ return _errno(self._magic_t) + def getparam(self, param): + """ + Returns the param value if successful and -1 if the parameter + was unknown. + """ + v = c_int() + i = _getparam(self._magic_t, param, byref(v)) + if i == -1: + return -1 + return v.value + + def setparam(self, param, value): + """ + Returns 0 if successful and -1 if the parameter was unknown. + """ + v = c_int(value) + return _setparam(self._magic_t, param, byref(v)) + def open(flags): """ Returns a magic object on success and None on failure. Flags argument as for setflags. """ - return Magic(_open(flags)) + magic_t = _open(flags) + if magic_t is None: + return None + return Magic(magic_t) # Objects used by `detect_from_` functions -mime_magic = Magic(_open(MAGIC_MIME)) -mime_magic.load() -none_magic = Magic(_open(MAGIC_NONE)) -none_magic.load() +class error(Exception): + pass +class MagicDetect(object): + def __init__(self): + self.mime_magic = open(MAGIC_MIME) + if self.mime_magic is None: + raise error + if self.mime_magic.load() == -1: + self.mime_magic.close() + self.mime_magic = None + raise error + self.none_magic = open(MAGIC_NONE) + if self.none_magic is None: + self.mime_magic.close() + self.mime_magic = None + raise error + if self.none_magic.load() == -1: + self.none_magic.close() + self.none_magic = None + self.mime_magic.close() + self.mime_magic = None + raise error + + def __del__(self): + if self.mime_magic is not None: + self.mime_magic.close() + if self.none_magic is not None: + self.none_magic.close() + +threadlocal = threading.local() + +def _detect_make(): + v = getattr(threadlocal, "magic_instance", None) + if v is None: + v = MagicDetect() + setattr(threadlocal, "magic_instance", v) + return v def _create_filemagic(mime_detected, type_detected): - mime_type, mime_encoding = mime_detected.split('; ') + try: + mime_type, mime_encoding = mime_detected.split('; ') + except ValueError: + raise ValueError(mime_detected) return FileMagic(name=type_detected, mime_type=mime_type, encoding=mime_encoding.replace('charset=', '')) @@ -256,9 +324,9 @@ def detect_from_filename(filename): Returns a `FileMagic` namedtuple. ''' - - return _create_filemagic(mime_magic.file(filename), - none_magic.file(filename)) + x = _detect_make() + return _create_filemagic(x.mime_magic.file(filename), + x.none_magic.file(filename)) def detect_from_fobj(fobj): @@ -268,8 +336,9 @@ def detect_from_fobj(fobj): ''' file_descriptor = fobj.fileno() - return _create_filemagic(mime_magic.descriptor(file_descriptor), - none_magic.descriptor(file_descriptor)) + x = _detect_make() + return _create_filemagic(x.mime_magic.descriptor(file_descriptor), + x.none_magic.descriptor(file_descriptor)) def detect_from_content(byte_content): @@ -278,5 +347,6 @@ def detect_from_content(byte_content): Returns a `FileMagic` namedtuple. ''' - return _create_filemagic(mime_magic.buffer(byte_content), - none_magic.buffer(byte_content)) + x = _detect_make() + return _create_filemagic(x.mime_magic.buffer(byte_content), + x.none_magic.buffer(byte_content)) diff --git a/magic/loader.py b/magic/loader.py index 6b2bfcb3..f8d59faf 100644 --- a/magic/loader.py +++ b/magic/loader.py @@ -1,40 +1,81 @@ +from ctypes.util import find_library import ctypes import sys import glob +import os.path +import logging + +logger = logging.getLogger(__name__) + + +def _lib_candidates_linux(): + """Yield possible libmagic library names on Linux. + + This is necessary because alpine is bad + """ + yield "libmagic.so.1" + + +def _lib_candidates_macos(): + """Yield possible libmagic library names on macOS.""" + paths = [ + "/opt/homebrew/lib", + "/opt/local/lib", + "/usr/local/lib", + ] + glob.glob("/usr/local/Cellar/libmagic/*/lib") + for path in paths: + yield os.path.join(path, "libmagic.dylib") + + +def _lib_candidates_windows(): + """Yield possible libmagic library names on Windows.""" + prefixes = ( + "libmagic", + "magic1", + "magic-1", + "cygmagic-1", + "libmagic-1", + "msys-magic-1", + ) + for prefix in prefixes: + # find_library searches in %PATH% but not the current directory, + # so look for both + yield "./%s.dll" % (prefix,) + yield find_library(prefix) + + +def _lib_candidates(): + yield find_library("magic") + + func = { + "cygwin": _lib_candidates_windows, + "darwin": _lib_candidates_macos, + "linux": _lib_candidates_linux, + "win32": _lib_candidates_windows, + "sunos5": _lib_candidates_linux, + }.get(sys.platform) + if func is None: + raise ImportError("python-magic: Unsupported platform: " + sys.platform) + # When we drop legacy Python, we can just `yield from func()` + for path in func(): + yield path + def load_lib(): - libmagic = None - # Let's try to find magic or magic1 - dll = ctypes.util.find_library('magic') \ - or ctypes.util.find_library('magic1') \ - or ctypes.util.find_library('cygmagic-1') \ - or ctypes.util.find_library('libmagic-1') \ - or ctypes.util.find_library('msys-magic-1') # for MSYS2 - - # necessary because find_library returns None if it doesn't find the library - if dll: - libmagic = ctypes.CDLL(dll) - - if not libmagic or not libmagic._name: - windows_dlls = ['magic1.dll', 'cygmagic-1.dll', 'libmagic-1.dll', 'msys-magic-1.dll'] - platform_to_lib = {'darwin': ['/opt/local/lib/libmagic.dylib', - '/usr/local/lib/libmagic.dylib'] + - # Assumes there will only be one version installed - glob.glob('/usr/local/Cellar/libmagic/*/lib/libmagic.dylib'), # flake8:noqa - 'win32': windows_dlls, - 'cygwin': windows_dlls, - 'linux': ['libmagic.so.1'], - # fallback for some Linuxes (e.g. Alpine) where library search does not work # flake8:noqa - } - platform = 'linux' if sys.platform.startswith('linux') else sys.platform - for dll in platform_to_lib.get(platform, []): - try: - libmagic = ctypes.CDLL(dll) - break - except OSError: - pass - - if not libmagic or not libmagic._name: - # It is better to raise an ImportError since we are importing magic module - raise ImportError('failed to find libmagic. Check your installation') - return libmagic \ No newline at end of file + exc = [] + for lib in _lib_candidates(): + # find_library returns None when lib not found + if lib is None: + continue + + try: + return ctypes.CDLL(lib) + except OSError as e: + exc.append(e) + + msg = "\n".join([str(e) for e in exc]) + + # It is better to raise an ImportError since we are importing magic module + raise ImportError( + "python-magic: failed to find libmagic. Check your installation: \n" + msg + ) diff --git a/__init__.py b/magic/py.typed similarity index 100% rename from __init__.py rename to magic/py.typed diff --git a/ruff.toml b/ruff.toml new file mode 100644 index 00000000..fe365518 --- /dev/null +++ b/ruff.toml @@ -0,0 +1,3 @@ +exclude = ["magic/compat.py"] + + diff --git a/setup.py b/setup.py index 3b5489af..54aff089 100644 --- a/setup.py +++ b/setup.py @@ -8,35 +8,43 @@ def read(file_name): """Read a text file and return the content as a string.""" - with io.open(os.path.join(os.path.dirname(__file__), file_name), - encoding='utf-8') as f: + with io.open( + os.path.join(os.path.dirname(__file__), file_name), encoding="utf-8" + ) as f: return f.read() + setuptools.setup( - name='python-magic', - description='File type identification using libmagic', - author='Adam Hupp', - author_email='adam@hupp.org', + name="python-magic", + description="File type identification using libmagic", + author="Adam Hupp", + author_email="adam@hupp.org", url="http://github.com/ahupp/python-magic", - version='0.4.22', - long_description=read('README.md'), - long_description_content_type='text/markdown', - packages=['magic'], + version="0.4.28", + long_description=read("README.md"), + long_description_content_type="text/markdown", + packages=["magic"], + package_data={ + "magic": ["py.typed", "*.pyi", "**/*.pyi"], + }, keywords="mime magic file", license="MIT", - python_requires='>=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*', + python_requires=">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*", classifiers=[ - 'Intended Audience :: Developers', - 'License :: OSI Approved :: MIT License', - 'Programming Language :: Python', - 'Programming Language :: Python :: 2.7', - 'Programming Language :: Python :: 3', - 'Programming Language :: Python :: 3.5', - 'Programming Language :: Python :: 3.6', - 'Programming Language :: Python :: 3.7', - 'Programming Language :: Python :: 3.8', - 'Programming Language :: Python :: 3.9', - 'Programming Language :: Python :: Implementation :: CPython', + "Intended Audience :: Developers", + "License :: OSI Approved :: MIT License", + "Programming Language :: Python", + "Programming Language :: Python :: 2.7", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.5", + "Programming Language :: Python :: 3.6", + "Programming Language :: Python :: 3.7", + "Programming Language :: Python :: 3.8", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", + "Programming Language :: Python :: Implementation :: CPython", ], ) - diff --git a/test/Dockerfile_alpine b/test/Dockerfile_alpine deleted file mode 100755 index f8ec79e8..00000000 --- a/test/Dockerfile_alpine +++ /dev/null @@ -1,4 +0,0 @@ -FROM alpine:latest -RUN apk add python3 python2 libmagic -COPY . /python-magic -CMD cd /python-magic/test && python3 ./run.py diff --git a/test/Dockerfile_archlinux b/test/Dockerfile_archlinux deleted file mode 100755 index 677add4d..00000000 --- a/test/Dockerfile_archlinux +++ /dev/null @@ -1,5 +0,0 @@ -FROM archlinux:20200505 -RUN yes | pacman -Syyu --overwrite '*' -RUN yes | pacman -S python python2 file which -COPY . /python-magic -CMD cd /python-magic/test && python3 ./run.py diff --git a/test/Dockerfile_bionic b/test/Dockerfile_bionic deleted file mode 100755 index e335b8ee..00000000 --- a/test/Dockerfile_bionic +++ /dev/null @@ -1,8 +0,0 @@ -FROM ubuntu:bionic -RUN apt-get update -RUN apt-get -y install python -RUN apt-get -y install python3 -RUN apt-get -y install locales -RUN locale-gen en_US.UTF-8 -COPY . /python-magic -CMD cd /python-magic/test && python3 ./run.py diff --git a/test/Dockerfile_centos7 b/test/Dockerfile_centos7 deleted file mode 100644 index f2ac6e40..00000000 --- a/test/Dockerfile_centos7 +++ /dev/null @@ -1,5 +0,0 @@ -FROM centos:7 -RUN yum -y update -RUN yum -y install file-devel python3 python2 which -COPY . /python-magic -CMD cd /python-magic/test && SKIP_FROM_DESCRIPTOR=1 python3 ./run.py diff --git a/test/Dockerfile_centos8 b/test/Dockerfile_centos8 deleted file mode 100644 index 505221b8..00000000 --- a/test/Dockerfile_centos8 +++ /dev/null @@ -1,5 +0,0 @@ -FROM centos:8 -RUN yum -y update -RUN yum -y install file-libs python3 python2 which -COPY . /python-magic -CMD cd /python-magic/test && python3 ./run.py diff --git a/test/Dockerfile_focal b/test/Dockerfile_focal deleted file mode 100755 index 74e4d78a..00000000 --- a/test/Dockerfile_focal +++ /dev/null @@ -1,8 +0,0 @@ -FROM ubuntu:focal -RUN apt-get update -RUN apt-get -y install python2 -RUN apt-get -y install python3 -RUN apt-get -y install locales -RUN locale-gen en_US.UTF-8 -COPY . /python-magic -CMD cd /python-magic/test && python3 ./run.py diff --git a/test/Dockerfile_xenial b/test/Dockerfile_xenial deleted file mode 100755 index bc0440be..00000000 --- a/test/Dockerfile_xenial +++ /dev/null @@ -1,8 +0,0 @@ -FROM ubuntu:xenial -RUN apt-get update -RUN apt-get -y install python -RUN apt-get -y install python3 -RUN apt-get -y install locales -RUN locale-gen en_US.UTF-8 -COPY . /python-magic -CMD cd /python-magic/test && python3 ./run.py diff --git a/test/README b/test/README index 12d4e4fc..215ee43a 100644 --- a/test/README +++ b/test/README @@ -1,10 +1,4 @@ -To run the tests across a selection of Ubuntu LTS versions: - -docker build -t "python_magic/xenial:latest" -f test/Dockerfile_xenial . -docker build -t "python_magic/bionic:latest" -f test/Dockerfile_bionic . -docker build -t "python_magic/focal:latest" -f test/Dockerfile_focal . - -docker run python_magic/xenial:latest -docker run python_magic/bionic:latest -docker run python_magic/focal:latest +There are a few ways to run the python-magic tests +1. `tox` will run the tests against all installed versions of python +2. `./test/run_all_docker_test.sh` will run against a variety of different Linux distributions, using docker. diff --git a/test/docker/alpine b/test/docker/alpine new file mode 100755 index 00000000..60b0698d --- /dev/null +++ b/test/docker/alpine @@ -0,0 +1,5 @@ +FROM python:3.8-alpine3.12 +RUN apk add python3 python2 libmagic +WORKDIR /python-magic +COPY . . +RUN python3 -m pip install tox diff --git a/test/docker/archlinux b/test/docker/archlinux new file mode 100755 index 00000000..6592ffc8 --- /dev/null +++ b/test/docker/archlinux @@ -0,0 +1,6 @@ +FROM archlinux:latest +RUN yes | pacman -Syyu --overwrite '*' +RUN yes | pacman -S python python-pip file which +WORKDIR /python-magic +COPY . . +RUN python3 -m pip install tox diff --git a/test/docker/bionic b/test/docker/bionic new file mode 100755 index 00000000..a37b2534 --- /dev/null +++ b/test/docker/bionic @@ -0,0 +1,8 @@ +FROM ubuntu:bionic +RUN apt-get update +RUN apt-get -y install python python3 locales python3-pip libmagic1 +RUN locale-gen en_US.UTF-8 + +WORKDIR /python-magic +COPY . . +RUN python3 -m pip install tox diff --git a/test/docker/centos7 b/test/docker/centos7 new file mode 100644 index 00000000..9caa9898 --- /dev/null +++ b/test/docker/centos7 @@ -0,0 +1,8 @@ +FROM centos:7 +RUN yum -y update +RUN yum -y install file-devel python3 python2 which +ENV SKIP_FROM_DESCRIPTOR=1 + +WORKDIR /python-magic +COPY . . +RUN python3 -m pip install tox diff --git a/test/docker/centos8 b/test/docker/centos8 new file mode 100644 index 00000000..7f2dbd06 --- /dev/null +++ b/test/docker/centos8 @@ -0,0 +1,10 @@ +FROM centos:8 +RUN yum -y update +RUN yum -y install file-libs python3 python2 which glibc-locale-source +RUN yum reinstall glibc-common -y && \ + localedef -i en_US -f UTF-8 en_US.UTF-8 && \ + echo "LANG=en_US.UTF-8" > /etc/locale.conf + +WORKDIR /python-magic +COPY . . +RUN python3 -m pip install tox diff --git a/test/docker/focal b/test/docker/focal new file mode 100755 index 00000000..f24d2317 --- /dev/null +++ b/test/docker/focal @@ -0,0 +1,10 @@ +FROM ubuntu:focal +RUN apt-get update +RUN apt-get -y install python python3 locales python3-pip libmagic1 +RUN locale-gen en_US.UTF-8 + +WORKDIR /python-magic +COPY . . +RUN python3 -m pip install tox + + diff --git a/test/docker/xenial b/test/docker/xenial new file mode 100755 index 00000000..fe7829be --- /dev/null +++ b/test/docker/xenial @@ -0,0 +1,8 @@ +FROM ubuntu:xenial +RUN apt-get update +RUN apt-get -y install python python3 locales python3-pip libmagic1 +RUN locale-gen en_US.UTF-8 + +WORKDIR /python-magic +COPY . . +RUN python3 -m pip install tox diff --git a/test/libmagic_test.py b/test/libmagic_test.py index 64b7ec4f..fff71cda 100644 --- a/test/libmagic_test.py +++ b/test/libmagic_test.py @@ -3,31 +3,37 @@ import unittest import os import magic +import os.path # magic_descriptor is broken (?) in centos 7, so don't run those tests -SKIP_FROM_DESCRIPTOR = bool(os.environ.get('SKIP_FROM_DESCRIPTOR')) +SKIP_FROM_DESCRIPTOR = bool(os.environ.get("SKIP_FROM_DESCRIPTOR")) + +TESTDATA_DIR = os.path.abspath(os.path.join(os.path.dirname(__file__), "testdata")) + class MagicTestCase(unittest.TestCase): - filename = 'testdata/test.pdf' - expected_mime_type = 'application/pdf' - expected_encoding = 'us-ascii' - expected_name = 'PDF document, version 1.2' + filename = os.path.join(TESTDATA_DIR, "test.pdf") + expected_mime_type = "application/pdf" + expected_encoding = "us-ascii" + expected_name = ( + "PDF document, version 1.2", + "PDF document, version 1.2, 2 pages", + "PDF document, version 1.2, 2 page(s)", + ) def assert_result(self, result): self.assertEqual(result.mime_type, self.expected_mime_type) self.assertEqual(result.encoding, self.expected_encoding) - self.assertEqual(result.name, self.expected_name) + self.assertIn(result.name, self.expected_name) def test_detect_from_filename(self): result = magic.detect_from_filename(self.filename) self.assert_result(result) def test_detect_from_fobj(self): - if SKIP_FROM_DESCRIPTOR: self.skipTest("magic_descriptor is broken in this version of libmagic") - with open(self.filename) as fobj: result = magic.detect_from_fobj(fobj) self.assert_result(result) @@ -37,10 +43,10 @@ def test_detect_from_content(self): # this avoids hitting a bug in python3+libfile bindings # see https://github.com/ahupp/python-magic/issues/152 # for a similar issue - with open(self.filename, 'rb') as fobj: + with open(self.filename, "rb") as fobj: result = magic.detect_from_content(fobj.read(4096)) self.assert_result(result) -if __name__ == '__main__': +if __name__ == "__main__": unittest.main() diff --git a/test/python_magic_test.py b/test/python_magic_test.py new file mode 100755 index 00000000..26398614 --- /dev/null +++ b/test/python_magic_test.py @@ -0,0 +1,351 @@ +from dataclasses import dataclass +from enum import Enum +import os +import os.path +import shutil +import sys +import tempfile +from typing import List, Union +import unittest + +import pytest + +try: + from concurrent.futures import ThreadPoolExecutor + HAS_CONCURRENT_FUTURES = True +except ImportError: # python 2.7 + HAS_CONCURRENT_FUTURES = False + +# for output which reports a local time +os.environ["TZ"] = "GMT" + +if os.environ.get("LC_ALL", "") != "en_US.UTF-8": + # this ensure we're in a utf-8 default filesystem encoding which is + # necessary for some tests + raise Exception("must run `export LC_ALL=en_US.UTF-8` before running test suite") + +import magic + + +@dataclass +class TestFile: + file_name: str + mime_results: List[str] + text_results: List[str] + no_check_elf_results: Union[List[str], None] + buf_equals_file: bool = True + + +# magic_descriptor is broken (?) in centos 7, so don't run those tests +SKIP_FROM_DESCRIPTOR = bool(os.environ.get("SKIP_FROM_DESCRIPTOR")) + + +COMMON_PLAIN = [{}] +NO_SOFT = [{"check_soft": False}] +COMMON_MIME = [{"mime": True}] + +CASES = { + b"magic._pyc_": [ + ( + COMMON_MIME, + [ + "application/octet-stream", + "text/x-bytecode.python", + "application/x-bytecode.python", + ], + ), + (COMMON_PLAIN, ["python 2.4 byte-compiled"]), + (NO_SOFT, ["data"]), + ], + b"test.pdf": [ + (COMMON_MIME, ["application/pdf"]), + ( + COMMON_PLAIN, + [ + "PDF document, version 1.2", + "PDF document, version 1.2, 2 pages", + "PDF document, version 1.2, 2 page(s)", + ], + ), + (NO_SOFT, ["ASCII text"]), + ], + b"test.gz": [ + (COMMON_MIME, ["application/gzip", "application/x-gzip"]), + ( + COMMON_PLAIN, + [ + 'gzip compressed data, was "test", from Unix, last modified: Sun Jun 29 01:32:52 2008', + 'gzip compressed data, was "test", last modified: Sun Jun 29 01:32:52 2008, from Unix', + 'gzip compressed data, was "test", last modified: Sun Jun 29 01:32:52 2008, from Unix, original size 15', + 'gzip compressed data, was "test", last modified: Sun Jun 29 01:32:52 2008, from Unix, original size modulo 2^32 15', + 'gzip compressed data, was "test", last modified: Sun Jun 29 01:32:52 2008, from Unix, truncated', + ], + ), + ( + [{"extension": True}], + [ + # some versions return '' for the extensions of a gz file, + # including w/ the command line. Who knows... + "gz/tgz/tpz/zabw/svgz/adz/kmy/xcfgz", + "gz/tgz/tpz/zabw/svgz", + "", + "???", + ], + ), + (NO_SOFT, ["data"]), + ], + b"test.snappy.parquet": [ + (COMMON_MIME, ["application/octet-stream", "application/vnd.apache.parquet"]), + (COMMON_PLAIN, ["Apache Parquet", "Apache Parquet file", "Par archive data"]), + (NO_SOFT, ["data"]), + ], + b"test.json": [ + (COMMON_MIME, ["application/json"]), + (COMMON_PLAIN, ["JSON text data"]), + ( + [{"mime": True, "check_json": False}], + [ + "text/plain", + ], + ), + (NO_SOFT, ["JSON text data"]), + ], + b"elf-NetBSD-x86_64-echo": [ + # TODO: soft, no elf + ( + COMMON_PLAIN, + [ + "ELF 64-bit LSB shared object, x86-64, version 1 (SYSV)", + "ELF 64-bit LSB pie executable, x86-64, version 1 (SYSV), dynamically linked, interpreter /libexec/ld.elf_so, for NetBSD 8.0, not stripped", + ], + ), + ( + COMMON_MIME, + [ + "application/x-pie-executable", + "application/x-sharedlib", + ], + ), + ( + [{"check_elf": False}], + [ + "ELF 64-bit LSB shared object, x86-64, version 1 (SYSV)", + ], + ), + # TODO: sometimes + # "ELF 64-bit LSB pie executable, x86-64, version 1 (SYSV), dynamically linked, interpreter /libexec/ld.elf_so, for NetBSD 8.0, not stripped", + (NO_SOFT, ["data"]), + ], + b"text.txt": [ + (COMMON_MIME, ["text/plain"]), + (COMMON_PLAIN, ["ASCII text"]), + ( + [{"mime_encoding": True}], + [ + "us-ascii", + ], + ), + (NO_SOFT, ["ASCII text"]), + ], + b"text-iso8859-1.txt": [ + ( + [{"mime_encoding": True}], + [ + "iso-8859-1", + ], + ), + ], + b"\xce\xbb": [ + (COMMON_MIME, ["text/plain"]), + ], + b"name_use.jpg": [ + ([{"extension": True}], ["jpeg/jpg/jpe/jfif"]), + ], + b"keep-going.jpg": [ + (COMMON_MIME, ["image/jpeg"]), + ( + [{"mime": True, "keep_going": True}], + [ + "image/jpeg\\012- application/octet-stream", + ], + ), + ], + b"../../magic/loader.py": [ + ( + COMMON_MIME, + [ + "text/x-python", + "text/x-script.python", + ], + ) + ], +} + + +class MagicTest(unittest.TestCase): + TESTDATA_DIR = os.path.abspath(os.path.join(os.path.dirname(__file__), "testdata")) + + def test_version(self): + try: + self.assertTrue(magic.version() > 0) + except NotImplementedError: + pass + + def test_fs_encoding(self): + self.assertEqual("utf-8", sys.getfilesystemencoding().lower()) + + def test_from_file_str_and_bytes(self): + filename = os.path.join(self.TESTDATA_DIR, "test.pdf") + + self.assertEqual("application/pdf", magic.from_file(filename, mime=True)) + self.assertEqual( + "application/pdf", magic.from_file(filename.encode("utf-8"), mime=True) + ) + + def test_all_cases(self): + # TODO: + # * MAGIC_EXTENSION not supported + # * keep_going not supported + # * buffer checks + dest = os.path.join(MagicTest.TESTDATA_DIR, b"\xce\xbb".decode("utf-8")) + shutil.copyfile(os.path.join(MagicTest.TESTDATA_DIR, "lambda"), dest) + os.environ["TZ"] = "UTC" + try: + for filename, cases in CASES.items(): + filename = os.path.join(self.TESTDATA_DIR.encode("utf-8"), filename) + print("test case ", filename, file=sys.stderr) + for flag_variants, outputs in cases: + for flags in flag_variants: + print("flags", flags, file=sys.stderr) + m = magic.Magic(**flags) + with open(filename) as f: + self.assertIn(m.from_descriptor(f.fileno()), outputs) + + self.assertIn(m.from_file(filename), outputs) + + fname_str = filename.decode("utf-8") + self.assertIn(m.from_file(fname_str), outputs) + + with open(filename, "rb") as f: + buf_result = m.from_buffer(f.read(1024)) + self.assertIn(buf_result, outputs) + finally: + del os.environ["TZ"] + os.unlink(dest) + + def test_unicode_result_nonraw(self): + m = magic.Magic(raw=False) + src = os.path.join(MagicTest.TESTDATA_DIR, "pgpunicode") + result = m.from_file(src) + # NOTE: This check is added as otherwise some magic files don't identify the test case as a PGP key. + if "PGP" in result: + assert r"PGP\011Secret Sub-key -" == result + else: + raise unittest.SkipTest("Magic file doesn't return expected type.") + + def test_unicode_result_raw(self): + m = magic.Magic(raw=True) + src = os.path.join(MagicTest.TESTDATA_DIR, "pgpunicode") + result = m.from_file(src) + if "PGP" in result: + assert b"PGP\tSecret Sub-key -" == result.encode("utf-8") + else: + raise unittest.SkipTest("Magic file doesn't return expected type.") + + def test_errors(self): + m = magic.Magic() + self.assertRaises(IOError, m.from_file, "nonexistent") + self.assertRaises(magic.MagicException, magic.Magic, magic_file="nonexistent") + os.environ["MAGIC"] = "nonexistent" + try: + self.assertRaises(magic.MagicException, magic.Magic) + finally: + del os.environ["MAGIC"] + + def test_rethrow(self): + old = magic.magic_buffer + try: + + def t(x, y): + raise magic.MagicException("passthrough") + + magic.magic_buffer = t + + with self.assertRaises(magic.MagicException): + magic.from_buffer("hello", True) + finally: + magic.magic_buffer = old + + def test_getparam(self): + m = magic.Magic(mime=True) + try: + m.setparam(magic.MAGIC_PARAM_INDIR_MAX, 1) + self.assertEqual(m.getparam(magic.MAGIC_PARAM_INDIR_MAX), 1) + except NotImplementedError: + pass + + def test_name_count(self): + m = magic.Magic() + with open(os.path.join(self.TESTDATA_DIR, "name_use.jpg"), "rb") as f: + m.from_buffer(f.read()) + + def test_pathlike(self): + if sys.version_info < (3, 6): + return + from pathlib import Path + + path = Path(self.TESTDATA_DIR, "test.pdf") + m = magic.Magic(mime=True) + self.assertEqual("application/pdf", m.from_file(path)) + + def test_symlink(self): + # TODO: 3.0 + if not hasattr(tempfile, "TemporaryDirectory"): + return + + with tempfile.TemporaryDirectory() as tmp: + tmp_link = os.path.join(tmp, "test_link") + tmp_broken = os.path.join(tmp, "nonexistent") + + os.symlink( + os.path.join(self.TESTDATA_DIR, "test.pdf"), + tmp_link, + ) + + os.symlink("/nonexistent", tmp_broken) + + m = magic.Magic() + m_follow = magic.Magic(follow_symlinks=True) + self.assertTrue(m.from_file(tmp_link).startswith("symbolic link to ")) + self.assertTrue(m_follow.from_file(tmp_link).startswith("PDF document")) + + self.assertTrue( + m.from_file(tmp_broken).startswith( + "broken symbolic link to /nonexistent" + ) + ) + + self.assertRaises(IOError, m_follow.from_file, tmp_broken) + + @unittest.skipIf(not HAS_CONCURRENT_FUTURES, "concurrent.futures not available in Python 2.7") + def test_thread_safety(self): + """Test that concurrent from_file calls don't crash (would SEGV without global lock)""" + filename = os.path.join(self.TESTDATA_DIR, "test.pdf") + + m = magic.Magic(mime=True) + + def check_file(_): + result = m.from_file(filename) + self.assertEqual(result, "application/pdf") + return result + + with ThreadPoolExecutor(100) as executor: + results = list(executor.map(check_file, range(100))) + + # All calls should complete successfully + self.assertEqual(len(results), 100) + self.assertTrue(all(r == "application/pdf" for r in results)) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/run.py b/test/run.py deleted file mode 100644 index cf62eee7..00000000 --- a/test/run.py +++ /dev/null @@ -1,35 +0,0 @@ -import subprocess -import os.path -import sys - -this_dir = os.path.dirname(sys.argv[0]) - -new_env = dict(os.environ) -new_env.update({ - 'LC_ALL': 'en_US.UTF-8', - 'PYTHONPATH': os.path.join(this_dir, ".."), -}) - - -def has_py(version): - ret = subprocess.run("which %s" % version, shell=True, stdout=subprocess.DEVNULL) - return ret.returncode == 0 - - -def run_test(versions): - found = False - for i in versions: - if not has_py(i): - # if this version doesn't exist in path, skip - continue - found = True - print("Testing %s" % i) - subprocess.run([i, os.path.join(this_dir, "test.py")], env=new_env, check=True) - subprocess.run([i, os.path.join(this_dir, "libmagic_test.py")], env=new_env, check=True) - - if not found: - sys.exit("No versions found: " + str(versions)) - -run_test(["python2", "python2.7"]) -run_test(["python3.5", "python3.6", "python3.7", "python3.8", "python3.9"]) - diff --git a/test/run_all_docker_test.sh b/test/run_all_docker_test.sh new file mode 100755 index 00000000..dce930b7 --- /dev/null +++ b/test/run_all_docker_test.sh @@ -0,0 +1,13 @@ +#!/bin/sh + +set -e +set -x + +ROOT=$(dirname $0)/.. +cd $ROOT + +for f in test/docker/*; do + H=$(docker build -q -f ${f} .) + docker run --rm $H python3 -m tox +done + diff --git a/test/test.py b/test/test.py deleted file mode 100755 index 949c77eb..00000000 --- a/test/test.py +++ /dev/null @@ -1,223 +0,0 @@ -import os - -# for output which reports a local time -os.environ['TZ'] = 'GMT' - -if os.environ.get('LC_ALL', '') != 'en_US.UTF-8': - # this ensure we're in a utf-8 default filesystem encoding which is - # necessary for some tests - raise Exception("must run `export LC_ALL=en_US.UTF-8` before running test suite") - -import shutil -import os.path -import unittest - -import magic -import sys - -# magic_descriptor is broken (?) in centos 7, so don't run those tests -SKIP_FROM_DESCRIPTOR = bool(os.environ.get('SKIP_FROM_DESCRIPTOR')) - -class MagicTest(unittest.TestCase): - TESTDATA_DIR = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'testdata') - - def test_version(self): - try: - self.assertTrue(magic.version() > 0) - except NotImplementedError: - pass - - def test_fs_encoding(self): - self.assertEqual('utf-8', sys.getfilesystemencoding().lower()) - - def assert_values(self, m, expected_values, buf_equals_file=True): - for filename, expected_value in expected_values.items(): - try: - filename = os.path.join(self.TESTDATA_DIR, filename) - except TypeError: - filename = os.path.join( - self.TESTDATA_DIR.encode('utf-8'), filename) - - if type(expected_value) is not tuple: - expected_value = (expected_value,) - - with open(filename, 'rb') as f: - buf_value = m.from_buffer(f.read()) - - file_value = m.from_file(filename) - - if buf_equals_file: - self.assertEqual(buf_value, file_value) - - for value in (buf_value, file_value): - self.assertIn(value, expected_value) - - def test_from_file_str_and_bytes(self): - filename = os.path.join(self.TESTDATA_DIR, "test.pdf") - - self.assertEqual('application/pdf', - magic.from_file(filename, mime=True)) - self.assertEqual('application/pdf', - magic.from_file(filename.encode('utf-8'), mime=True)) - - def test_from_descriptor_str_and_bytes(self): - if SKIP_FROM_DESCRIPTOR: - self.skipTest("magic_descriptor is broken in this version of libmagic") - - filename = os.path.join(self.TESTDATA_DIR, "test.pdf") - with open(filename) as f: - self.assertEqual('application/pdf', - magic.from_descriptor(f.fileno(), mime=True)) - self.assertEqual('application/pdf', - magic.from_descriptor(f.fileno(), mime=True)) - - def test_from_buffer_str_and_bytes(self): - if SKIP_FROM_DESCRIPTOR: - self.skipTest("magic_descriptor is broken in this version of libmagic") - m = magic.Magic(mime=True) - - self.assertTrue( - m.from_buffer('#!/usr/bin/env python\nprint("foo")') - in ("text/x-python", "text/x-script.python")) - self.assertTrue( - m.from_buffer(b'#!/usr/bin/env python\nprint("foo")') - in ("text/x-python", "text/x-script.python")) - - def test_mime_types(self): - dest = os.path.join(MagicTest.TESTDATA_DIR, - b'\xce\xbb'.decode('utf-8')) - shutil.copyfile(os.path.join(MagicTest.TESTDATA_DIR, 'lambda'), dest) - try: - m = magic.Magic(mime=True) - self.assert_values(m, { - 'magic._pyc_': ('application/octet-stream', 'text/x-bytecode.python'), - 'test.pdf': 'application/pdf', - 'test.gz': ('application/gzip', 'application/x-gzip'), - 'test.snappy.parquet': 'application/octet-stream', - 'text.txt': 'text/plain', - b'\xce\xbb'.decode('utf-8'): 'text/plain', - b'\xce\xbb': 'text/plain', - }) - finally: - os.unlink(dest) - - def test_descriptions(self): - m = magic.Magic() - os.environ['TZ'] = 'UTC' # To get last modified date of test.gz in UTC - try: - self.assert_values(m, { - 'magic._pyc_': 'python 2.4 byte-compiled', - 'test.pdf': 'PDF document, version 1.2', - 'test.gz': - ('gzip compressed data, was "test", from Unix, last ' - 'modified: Sun Jun 29 01:32:52 2008', - 'gzip compressed data, was "test", last modified' - ': Sun Jun 29 01:32:52 2008, from Unix', - 'gzip compressed data, was "test", last modified' - ': Sun Jun 29 01:32:52 2008, from Unix, original size 15', - 'gzip compressed data, was "test", ' - 'last modified: Sun Jun 29 01:32:52 2008, ' - 'from Unix, original size modulo 2^32 15', - 'gzip compressed data, was "test", last modified' - ': Sun Jun 29 01:32:52 2008, from Unix, truncated' - ), - 'text.txt': 'ASCII text', - 'test.snappy.parquet': ('Apache Parquet', 'Par archive data'), - }, buf_equals_file=False) - finally: - del os.environ['TZ'] - - def test_extension(self): - try: - m = magic.Magic(extension=True) - self.assert_values(m, { - # some versions return '' for the extensions of a gz file, - # including w/ the command line. Who knows... - 'test.gz': ('gz/tgz/tpz/zabw/svgz', '', '???'), - 'name_use.jpg': 'jpeg/jpg/jpe/jfif', - }) - except NotImplementedError: - self.skipTest('MAGIC_EXTENSION not supported in this version') - - def test_unicode_result_nonraw(self): - m = magic.Magic(raw=False) - src = os.path.join(MagicTest.TESTDATA_DIR, 'pgpunicode') - result = m.from_file(src) - # NOTE: This check is added as otherwise some magic files don't identify the test case as a PGP key. - if 'PGP' in result: - assert r"PGP\011Secret Sub-key -" == result - else: - raise unittest.SkipTest("Magic file doesn't return expected type.") - - def test_unicode_result_raw(self): - m = magic.Magic(raw=True) - src = os.path.join(MagicTest.TESTDATA_DIR, 'pgpunicode') - result = m.from_file(src) - if 'PGP' in result: - assert b'PGP\tSecret Sub-key -' == result.encode('utf-8') - else: - raise unittest.SkipTest("Magic file doesn't return expected type.") - - def test_mime_encodings(self): - m = magic.Magic(mime_encoding=True) - self.assert_values(m, { - 'text-iso8859-1.txt': 'iso-8859-1', - 'text.txt': 'us-ascii', - }) - - def test_errors(self): - m = magic.Magic() - self.assertRaises(IOError, m.from_file, 'nonexistent') - self.assertRaises(magic.MagicException, magic.Magic, - magic_file='nonexistent') - os.environ['MAGIC'] = 'nonexistent' - try: - self.assertRaises(magic.MagicException, magic.Magic) - finally: - del os.environ['MAGIC'] - - def test_keep_going(self): - filename = os.path.join(self.TESTDATA_DIR, 'keep-going.jpg') - - m = magic.Magic(mime=True) - self.assertEqual(m.from_file(filename), 'image/jpeg') - - try: - # this will throw if you have an "old" version of the library - # I'm otherwise not sure how to query if keep_going is supported - magic.version() - m = magic.Magic(mime=True, keep_going=True) - self.assertEqual(m.from_file(filename), - 'image/jpeg\\012- application/octet-stream') - except NotImplementedError: - pass - - def test_rethrow(self): - old = magic.magic_buffer - try: - def t(x, y): - raise magic.MagicException("passthrough") - - magic.magic_buffer = t - - with self.assertRaises(magic.MagicException): - magic.from_buffer("hello", True) - finally: - magic.magic_buffer = old - - def test_getparam(self): - m = magic.Magic(mime=True) - try: - m.setparam(magic.MAGIC_PARAM_INDIR_MAX, 1) - self.assertEqual(m.getparam(magic.MAGIC_PARAM_INDIR_MAX), 1) - except NotImplementedError: - pass - - def test_name_count(self): - m = magic.Magic() - with open(os.path.join(self.TESTDATA_DIR, 'name_use.jpg'), 'rb') as f: - m.from_buffer(f.read()) - - -if __name__ == '__main__': - unittest.main() diff --git a/test/testdata/elf-NetBSD-x86_64-echo b/test/testdata/elf-NetBSD-x86_64-echo new file mode 100644 index 00000000..74affab9 Binary files /dev/null and b/test/testdata/elf-NetBSD-x86_64-echo differ diff --git a/test/testdata/test.json b/test/testdata/test.json new file mode 100644 index 00000000..cbd40300 --- /dev/null +++ b/test/testdata/test.json @@ -0,0 +1,7 @@ +[ + { + "one": 2, + "three": null, + "four": [5, "six", false] + } +] diff --git a/test_docker.sh b/test_docker.sh deleted file mode 100755 index 57d53285..00000000 --- a/test_docker.sh +++ /dev/null @@ -1,21 +0,0 @@ -#!/bin/bash - -# Test with various versions of ubuntu. This more or less re-creates the -# Travis CI test environment - -set -e - -function TestInContainer { - local name="$1" - local TAG="python_magic/${name}:latest" - docker build -t $TAG -f "test/Dockerfile_${name}" . - docker run "python_magic/${name}:latest" -} - -TestInContainer "xenial" -TestInContainer "bionic" -TestInContainer "focal" -TestInContainer "centos7" -TestInContainer "centos8" -TestInContainer "archlinux" -TestInContainer "alpine" diff --git a/tox.ini b/tox.ini index 65595983..01cb7b23 100644 --- a/tox.ini +++ b/tox.ini @@ -1,26 +1,30 @@ [tox] envlist = - coverage-clean, py27, py35, py36, py37, py38, py39, - coverage-report, + py310, + py311, + py312, + py313, + py314, + py314t, mypy [testenv] commands = - coverage run --source=magic ./test/test.py + coverage run -m pytest setenv = COVERAGE_FILE=.coverage.{envname} - LC_ALL=en_US.UTF-8 + LC_ALL=en_US.UTF-8 deps = .[test] - zope.testrunner coverage + pytest [testenv:coverage-clean] deps = coverage @@ -44,4 +48,5 @@ commands = deps = mypy skip_install = true commands = - mypy magic.pyi + mypy -p magic +