From bb3a99a67d0bed6572800e258780dd3ba95985cc Mon Sep 17 00:00:00 2001 From: Jitendra Bansal Date: Sat, 31 Oct 2026 14:36:53 +0530 Subject: [PATCH] Strip embedded Project-Id-Version content from GNU message catalog type libmagic's description for GNU message catalog (.mo) files can include the gettext PO header's Project-Id-Version field verbatim, which is free-form content taken from inside the scanned file (translator comments, escaped non-ASCII text, etc.) rather than structural type information. This leaked raw file content into filetype_file and anything derived from it. Strip that trailing Project-Id-Version content in _detect() right after decoding libmagic's output, keeping only the structural 'GNU message catalog (...), revision X, N messages' description. Added tests/test_magic2.py cases covering both leaked examples from the issue (plain text and escaped-byte Project-Id-Version values) and confirming unrelated file types are left untouched. Fixes #43 Signed-off-by: Jitendra Bansal --- src/typecode/magic2.py | 19 ++++++++++++++++++- tests/test_magic2.py | 41 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 59 insertions(+), 1 deletion(-) diff --git a/src/typecode/magic2.py b/src/typecode/magic2.py index 6f33a58..2027e51 100644 --- a/src/typecode/magic2.py +++ b/src/typecode/magic2.py @@ -31,6 +31,7 @@ import ctypes.util import glob import os +import re import sys import warnings @@ -301,6 +302,18 @@ def encoding(location): return _detect(location, DETECT_ENC) +# For GNU message catalog (.mo) files, some libmagic versions append the +# gettext PO header's "Project-Id-Version" field verbatim to the detected +# type string. That field is free-form content taken from inside the +# scanned file (translators can put arbitrary, including non-ASCII escaped, +# text there) rather than structural information about the file type, so we +# strip it and keep only libmagic's own description. +MO_FILE_TYPE_RE = re.compile( + r"^(GNU message catalog \([^)]*\), revision [\d.]+, \d+ messages)," + r" Project-Id-Version:.*$" +) + + def _detect(location, flags): """ " Return the detected type using `flags` of file at `location` or an empty @@ -314,7 +327,11 @@ def _detect(location, flags): val = detector.get(location) val = val or "" val = val.decode("ascii", "ignore").strip() - return " ".join(val.split()) + val = " ".join(val.split()) + mo_match = MO_FILE_TYPE_RE.match(val) + if mo_match: + val = mo_match.group(1) + return val class MagicException(Exception): diff --git a/tests/test_magic2.py b/tests/test_magic2.py index 249a3e8..edf0f9c 100644 --- a/tests/test_magic2.py +++ b/tests/test_magic2.py @@ -7,8 +7,49 @@ # See https://aboutcode.org for more information about nexB OSS projects. # +from typecode.magic2 import MO_FILE_TYPE_RE from typecode.magic2 import libmagic_version def test_load_lib(): assert float(libmagic_version()) > 5 + + +def _clean_mo_type(val): + match = MO_FILE_TYPE_RE.match(val) + return match.group(1) if match else val + + +def test_mo_file_type_strips_embedded_project_id_version_ascii(): + val = ( + "GNU message catalog (little endian), revision 0.0, 349 messages, " + "Project-Id-Version: django '%(datetime)s non se puido interpretar na " + "zona hora horaria %(current_timezone)s; pode ser ambiguo ou non " + "existir.'" + ) + assert _clean_mo_type(val) == ( + "GNU message catalog (little endian), revision 0.0, 349 messages" + ) + + +def test_mo_file_type_strips_embedded_project_id_version_escaped_bytes(): + val = ( + "GNU message catalog (little endian), revision 0.0, 349 messages, " + "Project-Id-Version: django '%(datetime)s \\343\\201\\257 " + "%(current_timezone)s \\343\\201\\256\\343\\202\\277\\343\\202\\244" + "\\343\\203\\240\\343\\202\\276\\343\\203\\274\\343\\203\\263" + "\\343\\201\\247\\343\\201\\257\\350\\247\\243\\351\\207\\210" + "\\343\\201\\247\\343\\201\\215\\343\\201\\276\\343\\201\\233" + "\\343\\202\\223\\343\\201\\247\\343\\201\\227\\343\\201\\237" + "\\343\\200\\202\\343\\201\\235\\343\\202\\214\\343\\201\\257" + "\\346\\233\\226\\346\\230\\247\\343\\201\\247\\343\\201\\202" + "\\343\\202\\213\\343\\201\\213\\343\\200'" + ) + assert _clean_mo_type(val) == ( + "GNU message catalog (little endian), revision 0.0, 349 messages" + ) + + +def test_mo_file_type_leaves_unrelated_types_unchanged(): + assert _clean_mo_type("ASCII text") == "ASCII text" + assert _clean_mo_type("ELF 64-bit LSB executable") == "ELF 64-bit LSB executable"