diff --git a/src/typecode/magic2.py b/src/typecode/magic2.py index 6f33a58..2027e51 100644 --- a/src/typecode/magic2.py +++ b/src/typecode/magic2.py @@ -31,6 +31,7 @@ import ctypes.util import glob import os +import re import sys import warnings @@ -301,6 +302,18 @@ def encoding(location): return _detect(location, DETECT_ENC) +# For GNU message catalog (.mo) files, some libmagic versions append the +# gettext PO header's "Project-Id-Version" field verbatim to the detected +# type string. That field is free-form content taken from inside the +# scanned file (translators can put arbitrary, including non-ASCII escaped, +# text there) rather than structural information about the file type, so we +# strip it and keep only libmagic's own description. +MO_FILE_TYPE_RE = re.compile( + r"^(GNU message catalog \([^)]*\), revision [\d.]+, \d+ messages)," + r" Project-Id-Version:.*$" +) + + def _detect(location, flags): """ " Return the detected type using `flags` of file at `location` or an empty @@ -314,7 +327,11 @@ def _detect(location, flags): val = detector.get(location) val = val or "" val = val.decode("ascii", "ignore").strip() - return " ".join(val.split()) + val = " ".join(val.split()) + mo_match = MO_FILE_TYPE_RE.match(val) + if mo_match: + val = mo_match.group(1) + return val class MagicException(Exception): diff --git a/tests/test_magic2.py b/tests/test_magic2.py index 249a3e8..edf0f9c 100644 --- a/tests/test_magic2.py +++ b/tests/test_magic2.py @@ -7,8 +7,49 @@ # See https://aboutcode.org for more information about nexB OSS projects. # +from typecode.magic2 import MO_FILE_TYPE_RE from typecode.magic2 import libmagic_version def test_load_lib(): assert float(libmagic_version()) > 5 + + +def _clean_mo_type(val): + match = MO_FILE_TYPE_RE.match(val) + return match.group(1) if match else val + + +def test_mo_file_type_strips_embedded_project_id_version_ascii(): + val = ( + "GNU message catalog (little endian), revision 0.0, 349 messages, " + "Project-Id-Version: django '%(datetime)s non se puido interpretar na " + "zona hora horaria %(current_timezone)s; pode ser ambiguo ou non " + "existir.'" + ) + assert _clean_mo_type(val) == ( + "GNU message catalog (little endian), revision 0.0, 349 messages" + ) + + +def test_mo_file_type_strips_embedded_project_id_version_escaped_bytes(): + val = ( + "GNU message catalog (little endian), revision 0.0, 349 messages, " + "Project-Id-Version: django '%(datetime)s \\343\\201\\257 " + "%(current_timezone)s \\343\\201\\256\\343\\202\\277\\343\\202\\244" + "\\343\\203\\240\\343\\202\\276\\343\\203\\274\\343\\203\\263" + "\\343\\201\\247\\343\\201\\257\\350\\247\\243\\351\\207\\210" + "\\343\\201\\247\\343\\201\\215\\343\\201\\276\\343\\201\\233" + "\\343\\202\\223\\343\\201\\247\\343\\201\\227\\343\\201\\237" + "\\343\\200\\202\\343\\201\\235\\343\\202\\214\\343\\201\\257" + "\\346\\233\\226\\346\\230\\247\\343\\201\\247\\343\\201\\202" + "\\343\\202\\213\\343\\201\\213\\343\\200'" + ) + assert _clean_mo_type(val) == ( + "GNU message catalog (little endian), revision 0.0, 349 messages" + ) + + +def test_mo_file_type_leaves_unrelated_types_unchanged(): + assert _clean_mo_type("ASCII text") == "ASCII text" + assert _clean_mo_type("ELF 64-bit LSB executable") == "ELF 64-bit LSB executable"