diff --git a/.flake8 b/.flake8 new file mode 100644 index 0000000..a23144a --- /dev/null +++ b/.flake8 @@ -0,0 +1,22 @@ +[flake8] +filename = uefi_support/*.py,uefi_support/*.pyi +max-line-length = 90 +accept-encodings = utf-8 +mypy_config = mypy.ini + +# In the pyi files ignore ... on same line, and some keywords shadowing. +per-file-ignores = + uefi_support/*.pyi:E704,A003 + +# Errors to ignore. +ignore = CCR001,E265,E302,E305,E501,W503 + +# CCR001 Cognitive complexity is too high (27 > 7) +# E265 block comment should start with '# ' +# E302 expected 2 blank lines, found 1 +# E241 multiple spaces after ',' +# E202 whitespace before ')' +# E221 multiple spaces before operator +# E305 expected 2 blank lines after class or function definition, found 1 +# E501 line too long (91 > 90 characters) +# W503 line break before binary operator diff --git a/.github/workflows/pyblish.yml b/.github/workflows/pyblish.yml new file mode 100644 index 0000000..9561c1b --- /dev/null +++ b/.github/workflows/pyblish.yml @@ -0,0 +1,124 @@ +name: Build and Publish to PyPI + +on: + push: + tags: + - "v*" + +permissions: + id-token: write + contents: read + +jobs: + build: + name: Build wheels & sdist + strategy: + matrix: + os: [ubuntu-latest, macos-latest, windows-latest] + python-version: ["3.10", "3.11", "3.12", "3.13"] + + runs-on: ${{ matrix.os }} + + steps: + - name: Checkout repository and submodules + uses: actions/checkout@v4 + with: + fetch-depth: 0 + submodules: recursive + + ################################################################# + # LINUX + macOS: cibuildwheel + ################################################################# + - name: Set up Python (Linux/macOS) + if: matrix.os != 'windows-latest' + uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + + - name: Install build tools (Linux/macOS) and verify package + if: matrix.os != 'windows-latest' + run: | + python -m pip install --upgrade pip + pip install build cibuildwheel mypy flake8 + mypy --strict + flake8 + + - name: Build wheels (Linux/macOS) + if: matrix.os != 'windows-latest' + run: cibuildwheel --output-dir dist + + ################################################################# + # WINDOWS: Build with MinGW (avoids MSVC incompatibilities) + ################################################################# + - name: Install MSYS2 + MinGW + if: matrix.os == 'windows-latest' + uses: msys2/setup-msys2@v2 + with: + update: true + install: > + mingw-w64-x86_64-gcc + mingw-w64-x86_64-python + git + + - name: Build wheel (Windows via MinGW) + if: matrix.os == 'windows-latest' + shell: msys2 {0} + env: + XNAME: ${{ github.ref_name }} + run: | + VENV_PATH=$(cygpath -u "$PWD/venv") + python3 -m venv "$VENV_PATH" + source "$VENV_PATH/bin/activate" + XVERSION=${XNAME#v} + export SETUPTOOLS_SCM_PRETEND_VERSION_FOR_CERT_UEFI_SUPPORT="$XVERSION" + python3.exe -m pip install --upgrade pip || true + python -m pip install --upgrade build + python -m build --wheel --outdir dist + cd dist + for whl in *.whl; do + newwhl=$(echo "$whl" | sed 's/mingw_x86_64_msvcrt_gnu/win_amd64/') + mv "$whl" "$newwhl" + echo "$whl" "$newwhl" + done + + ################################################################# + # Source Distribution (only once) + ################################################################# + - name: Build source distribution (Linux only) + if: matrix.os == 'ubuntu-latest' + run: python -m build --sdist --outdir dist + + ################################################################# + # Upload Artifacts + ################################################################# + - name: Upload built artifacts + uses: actions/upload-artifact@v4 + with: + name: python-dist-${{ matrix.os }} + path: dist/* + + + publish: + name: Publish to PyPI via Trusted Publishing + needs: build + runs-on: ubuntu-latest + environment: pypi + + permissions: + id-token: write + contents: read + + steps: + - uses: actions/download-artifact@v4 + with: + pattern: python-dist-* + path: dist + merge-multiple: true + + - name: Update Packaging Tools + run: python -m pip install --upgrade twine pkginfo + + - name: Publish to PyPI + uses: pypa/gh-action-pypi-publish@release/v1 + with: + skip-existing: false diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..eab43d0 --- /dev/null +++ b/.gitignore @@ -0,0 +1,98 @@ +.DS_Store +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class +*~ + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +pip-wheel-metadata/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# PyBuilder +target/ + +# IPython +profile_default/ +ipython_config.py + +# pyenv +.python-version + +# pipenv +# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. +# However, in case of collaboration, if having platform-specific dependencies or dependencies +# having no cross-platform support, pipenv may install dependencies that don't work, or not +# install all needed dependencies. +#Pipfile.lock + +# PEP 582; used by e.g. github.com/David-OConnor/pyflow +__pypackages__/ + +# Celery stuff +celerybeat-schedule +celerybeat.pid + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +*venv/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ +_version.py +node_modules +tmp diff --git a/LICENSE.md b/LICENSE.md index 7348599..1c38bd2 100644 --- a/LICENSE.md +++ b/LICENSE.md @@ -1,60 +1,121 @@ -# CERT UEFI Support Library - -Insert SEI license terms here. - -### Third-Party Software - -This Software includes and wraps the following Third-Party Software -subject to their own licenses: - -1. EDK II (https://github.com/tianocore/edk2) - - Copyright (c) 2019, TianoCore and contributors - - BSD-2-Clause-Patent - - https://github.com/tianocore/edk2/blob/master/License.txt - -2. libmspack (https://www.cabextract.org.uk/libmspack/) - - Copyright (c) 2003-2018 Stuart Caie - - LGPL-2.1 - - https://github.com/kyz/libmspack/blob/master/libmspack/COPYING.LIB - -3. Unhuffme (https://io.netgarage.org/me/) - - Copyright (C) 2015-2015 bla - - All rights reserved. - - Provided with no guarantees, whatsoever. Not even the implied - guarantee of fitness for any particular purpose. - - -4. unME12 (https://github.com/ptresearch/unME12) - - Copyright (c) 2018 Positive Technologies - - Permission is hereby granted, free of charge, to any person - obtaining a copy of this software and associated documentation - files (the "Software"), to deal in the Software without - restriction, including without limitation the rights to use, copy, - modify, merge, publish, distribute, sublicense, and/or sell copies - of the Software, and to permit persons to whom the Software is - furnished to do so, subject to the following conditions: - - The above copyright notice and this permission notice shall be - included in all copies or substantial portions of the Software. - - THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, - EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF - MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND - NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS - BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN - ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN - CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - SOFTWARE. - - https://github.com/ptresearch/unME12/blob/master/LICENSE +# CERT UEFI Support Library — License + +Copyright 2025 +Carnegie Mellon University + +--- + +## 1. Redistribution and Use + +Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: + +1. Redistributions of source code must retain the above copyright notice, this list of conditions, and the following disclaimer. + +2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions, and the following disclaimer in the documentation and/or other materials provided with the distribution. + +3. Products derived from this software may not include “Carnegie Mellon University,” “SEI,” and/or “Software Engineering Institute” in the name of such derived product, nor shall “Carnegie Mellon University,” “SEI,” and/or “Software Engineering Institute” be used to endorse or promote products derived from this software without prior written permission. For written permission, please contact: permission@sei.cmu.edu + +--- + +## 2. Acknowledgments and Disclaimers + +CERT UEFI Parser includes and/or can make use of certain third-party software (“Third Party Software”). +The Third Party Software used by CERT UEFI Parser depends on system configuration, but typically includes the software identified in this license file and/or described in the documentation or readme. + +By using CERT UEFI Parser, you agree to comply with any and all relevant Third Party Software terms and conditions contained in such Third Party Software or in a separate license file distributed with the Third Party Software. + +The owners of the Third Party Software (“Third Party Licensors”) are intended third-party beneficiaries with respect to the terms applicable to their software. +Third-Party Software licenses apply only to the Third-Party components and not to CERT UEFI Parser as a whole. + +--- + +## 3. Government Contract Notice + +This material is based upon work supported by the Department of War under Air Force Contract No. FA8702-15-D-0002 with Carnegie Mellon University for the operation of the Software Engineering Institute, a federally funded research and development center. + +The opinions, findings, conclusions, and/or recommendations contained in this material are those of the authors and should not be construed as an official U.S. Government position, policy, or decision unless so designated by other documentation. + +--- + +## 4. Warranty Disclaimer + +NO WARRANTY. +THIS CARNEGIE MELLON UNIVERSITY AND SOFTWARE ENGINEERING INSTITUTE MATERIAL IS FURNISHED ON AN “AS-IS” BASIS. + +CARNEGIE MELLON UNIVERSITY MAKES NO WARRANTIES OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO: + +- FITNESS FOR A PARTICULAR PURPOSE +- MERCHANTABILITY +- EXCLUSIVITY +- RESULTS OBTAINED FROM USE +- FREEDOM FROM PATENT, TRADEMARK, OR COPYRIGHT INFRINGEMENT + +--- + +## 5. Distribution Statement + +[DISTRIBUTION STATEMENT A] +Approved for public release; unlimited distribution. + +Please see copyright notice for non-U.S. Government use and distribution. + +Reference: DM25-1401 + +--- + +# 6. Third-Party Software and Licenses + +CERT UEFI Support Library includes or wraps the following Third-Party components. +Each third-party license applies only to that component and must be reproduced when distributing binaries. + +--- + +## 6.1 EDK II +Home: https://github.com/tianocore/edk2 + +Copyright 2019 +TianoCore and contributors + +License: BSD-2-Clause-Patent +Full license: https://github.com/tianocore/edk2/blob/master/License.txt + +--- + +## 6.2 libmspack +Home: https://www.cabextract.org.uk/libmspack/ + +Copyright 2003–2018 +Stuart Caie + +License: LGPL-2.1 +Full license: +https://github.com/kyz/libmspack/blob/master/libmspack/COPYING.LIB + +--- + +## 6.3 Unhuffme +Home: https://io.netgarage.org/me/ + +Copyright 2015 +bla + +License: Custom license +(Original text preserved below) + +> All rights reserved. +> Provided with no guarantees whatsoever, including implied warranties of fitness for any particular purpose. + +--- + +## 6.4 unME12 +Home: https://github.com/ptresearch/unME12 + +Copyright 2018 +Positive Technologies + +License: MIT License +Full license: https://github.com/ptresearch/unME12/blob/master/LICENSE + +--- + +# End of License diff --git a/MANIFEST.in b/MANIFEST.in index 2eb13b6..0eda31d 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -14,9 +14,12 @@ include edk2/BaseTools/Source/C/LzmaCompress/Sdk/C/LzmaDec.h include edk2/BaseTools/Source/C/LzmaCompress/Sdk/C/LzmaEnc.h include edk2/BaseTools/Source/C/LzmaCompress/Sdk/C/7zVersion.h include libmspack/libmspack/mspack/system.h +include libmspack/libmspack/mspack/macros.h include libmspack/libmspack/mspack/mspack.h include libmspack/libmspack/mspack/cab.h include libmspack/libmspack/mspack/qtm.h include libmspack/libmspack/mspack/readbits.h include libmspack/libmspack/mspack/readhuff.h +include libmspack/libmspack/mspack/mszip.h +include libmspack/libmspack/mspack/lzx.h include uefi_support/py.typed diff --git a/README.md b/README.md index ec42e3f..bd9ef09 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,10 @@ # CERT UEFI Support Library -This is a python library that wraps a bunch of third-party -decompression libraries for use in parsing UEFI files. +The CERT UEFI Support Library is a Python package that +wraps several third-party decompression libraries for +working with UEFI files. It is also available as +from [PyPI cert-uefi-support](https://pypi.org/project/cert-uefi-support/) + +It is a required dependency for the `cert-uefi-parser` project +project, which can be installed from [PyPI cert-uefi-parser](https://pypi.org/project/cert-uefi-parser/) +or from the [GitHub repository](https://github.com/cmu-sei/cert-uefi-parser). diff --git a/mypy.ini b/mypy.ini new file mode 100644 index 0000000..dfb7452 --- /dev/null +++ b/mypy.ini @@ -0,0 +1,30 @@ +[mypy] +mypy_path = stubs +files = uefi_support +python_version = 3.10 +show_error_codes = True +pretty = True +warn_unused_configs = True +warn_redundant_casts = True +strict_optional = True + +# Strict mode; enables the following flags: --warn-unused-configs, +# --disallow-any-generics, --disallow-subclassing-any, +# --disallow-untyped-calls, --disallow-untyped-defs, +# --disallow-incomplete-defs, --check-untyped-defs, +# --disallow-untyped-decorators, --no-implicit-optional, +# --warn-redundant-casts, --warn-unused-ignores, --warn-return-any, +# --no-implicit-reexport + +strict = True +disallow_any_generics = True +disallow_subclassing_any = True +disallow_untyped_calls = True +disallow_untyped_defs = True +disallow_incomplete_defs = True +check_untyped_defs = True +disallow_untyped_decorators = True +no_implicit_optional = True +warn_unused_ignores = True +warn_return_any = True +no_implicit_reexport = True diff --git a/pyproject.toml b/pyproject.toml index 474a071..91448d5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,9 +1,45 @@ [project] -name = "uefi_support" -version = "0.9" -authors = [{ name = "Michael Duggan", email = "mwd@cert.org"}] +name = "cert-uefi-support" +authors = [{ name = "CERT Threat Analysis Team", email = "cert@cert.org"}] description = "A package for decompressing file formats found in UEFI distributions" +readme = {file="README.md", content-type="text/markdown"} +dynamic = ["version"] +keywords = ["uefi", "firmware", "uefi-decompression", "support-tools"] +license = "BSD-2-Clause-Patent AND LGPL-2.1 AND MIT AND LicenseRef-Custom" +license-files = [ "LICENSE.md" ] +classifiers = [ + "Development Status :: 4 - Beta", + "Programming Language :: Python :: 3", + "Operating System :: MacOS :: MacOS X", + "Operating System :: Microsoft :: Windows", + "Operating System :: POSIX :: Linux", + "Topic :: Security", + "Topic :: System :: Hardware", +] +requires-python = ">=3.10" + +# Explicitly tell setuptools NOT to auto-discover +[tool.setuptools] +packages = ["uefi_support"] + +[tool.setuptools.package-dir] +uefi_support = "uefi_support" + +# Include non-Python files like huff11.bin +[tool.setuptools.package-data] +uefi_support = ["huff11.bin"] [build-system] -requires = ["setuptools"] +requires = ["setuptools>=77.0", "setuptools_scm[toml]>=6.0", "wheel"] build-backend = "setuptools.build_meta" + +[project.urls] +Homepage = "https://github.com/cmu-sei/cert-uefi-support" +Repository = "https://github.com/cmu-sei/cert-uefi-support" + +#version control uisng git tags +[tool.setuptools_scm] +version_scheme = "guess-next-dev" +local_scheme = "node-and-date" +fallback_version = "0.9.0" +write_to = "uefi_support/_version.py" \ No newline at end of file diff --git a/setup.py b/setup.py index d0929cc..bc5b9c8 100644 --- a/setup.py +++ b/setup.py @@ -1,14 +1,67 @@ from setuptools import setup, Extension +from setuptools.command.build_ext import build_ext from pathlib import Path +import tempfile +import subprocess +import os import sys +import shutil -BaseToolsDir = Path("edk2/BaseTools") +# Friendly build_ext that checks for a working C compiler and prints helpful guidance if missing. +class FriendlyBuildExt(build_ext): + def build_extensions(self): + if not self._has_compiler(): + print( + "\nERROR: A C compiler is required to build cert-uefi-support from source.\n" + "Install one of the following for your platform:\n" + " - Debian/Ubuntu: sudo apt install build-essential\n" + " - Fedora: sudo dnf groupinstall 'Development Tools'\n" + " - macOS: xcode-select --install\n" + " - Windows: Install 'Build Tools for Visual Studio' (MSVC)\n" + "Alternatively, install a prebuilt wheel if available on PyPI.\n" + ) + sys.exit(1) + super().build_extensions() + + def _has_compiler(self): + """Try compiling a tiny C file using the detected compiler.""" + compiler = self.compiler + # check for MSVC compiler and initiatlize it + if hasattr(compiler, 'initialize'): + compiler.initialize() + + tmpdir = tempfile.mkdtemp() + src_file = os.path.join(tmpdir, "test.c") + try: + with open(src_file, "w") as f: + f.write("int main(void){return 0;}") + + # Compile the source file + try: + objs = compiler.compile([src_file], output_dir=tmpdir) + except Exception: + return False + + # Optional: try linking (Windows MSVC may need this) + try: + exe_file = os.path.join(tmpdir, "test.exe") + compiler.link_executable(objs, exe_file) + except Exception: + return False + + return True -if not Path(BaseToolsDir).exists(): - print("The edk2 submodule has not been populated.") - print("Populate it using \"git submodule update --init --recursive\"") - sys.exit(1) + finally: + shutil.rmtree(tmpdir, ignore_errors=True) +# BaseTools path under edk2 repository (may be present as a submodule) +BaseToolsDir = Path("edk2/BaseTools") +if not BaseToolsDir.exists(): + print("WARNING: edk2/BaseTools not found. If you need full functionality run:") + print(" git submodule update --init --recursive") + # don't exit; allow builds to proceed if CI provides sources / necessary files + +# Define extension modules (mirror previous setup.py logic) edk2Module = Extension( 'uefi_support.EfiCompressor', sources=[ @@ -22,6 +75,7 @@ ] ) +# LZMA SDK sources (vendored under edk2 path in original layout) LzmaSDK = Path(BaseToolsDir, 'Source', 'C', 'LzmaCompress', 'Sdk', 'C') LzmaSDKFiles = [str(Path(LzmaSDK, x)) for x in ['Alloc.c', 'LzFind.c', 'LzmaDec.c', 'LzmaEnc.c', '7zFile.c', '7zStream.c', 'Bra86.c']] @@ -32,6 +86,7 @@ define_macros = [('_7ZIP_ST', None)] ) +# Huffman/unhuffme sources (vendored in unhuffme/) HuffmanPath = Path('unhuffme') HuffmanFiles = [str(Path(HuffmanPath, x)) for x in ['dict_cpt_1.c', 'dict_cpt_2.c', @@ -41,6 +96,7 @@ sources = HuffmanFiles + [str(Path('uefi_support', 'HuffmanCompressor.c'))], include_dirs = [ str(HuffmanPath) ]) +# libmspack sources (vendored under libmspack/) mspackPath = Path("libmspack/libmspack/mspack") mspackFiles = [str(Path(mspackPath, x)) for x in ['system.c', 'cabd.c', 'lzxd.c', 'mszipd.c', 'qtmd.c']] @@ -49,12 +105,14 @@ "uefi_support.Cab", sources = mspackFiles + [str(Path('uefi_support', 'Cab.c'))], include_dirs = [ str(mspackPath) ], - extra_compile_args = []) - -setup(name="uefi_support", - version="1.0", - ext_modules=[edk2Module, lzmaModule, huffmanModule, cabModule], - package_dir={'uefi_support': 'uefi_support'}, - packages=['uefi_support'], - package_data={'uefi_support': ['huff11.bin']} - ) +) + +setup( + use_scm_version=True, + package_dir={'uefi_support': 'uefi_support'}, + packages=['uefi_support'], + package_data={'uefi_support': ['huff11.bin']}, + ext_modules=[edk2Module, lzmaModule, huffmanModule, cabModule], + cmdclass={"build_ext": FriendlyBuildExt}, +) + diff --git a/uefi_support/Cab.pyi b/uefi_support/Cab.pyi index 5754b08..b31d1be 100644 --- a/uefi_support/Cab.pyi +++ b/uefi_support/Cab.pyi @@ -3,7 +3,7 @@ import io from typing import Optional class File(object): - def next(self) -> Optional[File]: ... + def next(self) -> Optional['File']: ... def data(self, buffer: io.BytesIO) -> io.BytesIO: ... info: tuple[str, int, int, tuple[int, int, int, int, int, int]] diff --git a/uefi_support/EfiCompressor.c b/uefi_support/EfiCompressor.c index adf9d5a..4d42cec 100644 --- a/uefi_support/EfiCompressor.c +++ b/uefi_support/EfiCompressor.c @@ -28,7 +28,7 @@ UefiDecompress( ) { PyObject *Retval; - UINT32 SrcDataSize; + Py_ssize_t SrcDataSize; UINT32 DstDataSize; UINTN Status; UINT8 *SrcBuf; @@ -70,7 +70,7 @@ FrameworkDecompress( ) { PyObject *Retval; - UINT32 SrcDataSize; + Py_ssize_t SrcDataSize; UINT32 DstDataSize; UINTN Status; UINT8 *SrcBuf; diff --git a/uefi_support/EfiCompressor.pyi b/uefi_support/EfiCompressor.pyi index 17967df..2c5423e 100644 --- a/uefi_support/EfiCompressor.pyi +++ b/uefi_support/EfiCompressor.pyi @@ -5,4 +5,3 @@ def FrameworkDecompress(data: bytes) -> bytes: ... class EfiException(Exception): pass - diff --git a/uefi_support/__init__.py b/uefi_support/__init__.py index 7b5e6db..1a68924 100644 --- a/uefi_support/__init__.py +++ b/uefi_support/__init__.py @@ -5,11 +5,10 @@ import os import io from enum import Enum -from collections import namedtuple from dataclasses import dataclass from types import TracebackType from collections.abc import Callable -from typing import Optional, Union, Iterator, Any, Type, TypeVar, ParamSpec, BinaryIO +from typing import Optional, Union, Iterator, Any, Type, TypeVar, ParamSpec from .EfiCompressor import ( UefiDecompress as _uefiDecompress, @@ -18,6 +17,11 @@ from .HuffmanCompressor import HuffmanDecompress as _huffmanDecompress, HuffmanException from .huffman import HuffDecoder, Error as HuffError from .Cab import Decompressor as CabDecompressor +try: + from ._version import version as __version__ # type: ignore +except Exception: + __version__ = "0.0.0" + __all__ = ['UefiDecompress', 'FrameworkDecompress', 'LzmaDecompress', 'HuffmanDecompress', 'HuffmanFlags', @@ -113,7 +117,6 @@ def huff11len(args: list[Any], kwds: Any) -> int: @wrap_errors(HuffError, "Huffman11", huff11len) def Huffman11Decompress(data: bytes, length: Optional[int] = None) -> bytes: """Decompress Intel ME 11 compressed data""" - global decoder if length is None: length = len(data) result = decoder.decompress(data, length) @@ -150,7 +153,7 @@ def infolist_gen(self) -> Iterator[CabInfo]: def infolist(self) -> list[CabInfo]: return list(self.infolist_gen()) - def open(self, filename: Union[str, CabInfo]) -> Optional[io.BytesIO]: + def open(self, filename: Union[str, CabInfo]) -> Optional[io.BytesIO]: # noqa [A003] if isinstance(filename, CabInfo): filename = CabInfo.filename x = self.cab.first_file() diff --git a/uefi_support/huffman.py b/uefi_support/huffman.py index 7995ea2..f8220cd 100644 --- a/uefi_support/huffman.py +++ b/uefi_support/huffman.py @@ -1,6 +1,4 @@ #!/bin/env python3 - -import sys import os import struct import zlib @@ -9,11 +7,11 @@ class Error(Exception): pass -def cwDec(w: int) -> str: # Convert 16-bit value to string codeword +def cwDec(w: int) -> str: # Convert 16-bit value to string codeword return bin(0x10000 | w).rstrip('0')[3:-1] -def cwEnc(cw: str) -> int: # Convert string codeword to 16-bit value - return int((cw+'1').ljust(16, '0'), 2) +def cwEnc(cw: str) -> int: # Convert string codeword to 16-bit value + return int((cw + '1').ljust(16, '0'), 2) #*************************************************************************** #*************************************************************************** @@ -25,10 +23,10 @@ def HuffTabReader_bin(ab: bytes) -> Iterator[tuple[str, int, bytes]]: while o < len(ab): w, cb = fmtRec.unpack_from(ab, o) o += fmtRec.size - v = ab[o:o+cb] + v = ab[o:o + cb] assert len(v) == cb o += cb - yield(cwDec(w), cb, v) + yield (cwDec(w), cb, v) #*************************************************************************** #*************************************************************************** @@ -36,14 +34,14 @@ def HuffTabReader_bin(ab: bytes) -> Iterator[tuple[str, int, bytes]]: class HuffNode(object): def __init__(self, cw: str, hd: Optional['HuffDecoder']): - self.cw = cw # String codeword value - self.w = cwEnc(cw) # Encoded codeword value + self.cw = cw # String codeword value + self.w = cwEnc(cw) # Encoded codeword value if hd: - self.nBits: Optional[int] = len(cw) # Length of codeword in bits + self.nBits: Optional[int] = len(cw) # Length of codeword in bits self.cb = hd.dLen.get(cw, None) self.av: list[Any] = [d.get(cw, None) for d in hd.adTab] else: - self.nBits = None # Actual length of codeword is unknown + self.nBits = None # Actual length of codeword is unknown #*************************************************************************** #*************************************************************************** @@ -56,15 +54,15 @@ class HuffDecoder(object): DUMP_ALL = 2 fmtInt = struct.Struct(" None: with open(os.path.join(self.baseDir, "huff11.bin"), "rb") as fi: - self.unpackTables(zlib.decompress(fi.read(), -15)) # Load from compressed version + self.unpackTables(zlib.decompress(fi.read(), -15)) # Load from compressed version self.prepareMap() def loadTable(self, items: Iterator[tuple[str, int, bytes]]) -> None: - sv = set() # Set for values + sv = set() # Set for values d = {} for cw, cb, v in items: if cw in d: @@ -81,7 +79,7 @@ def loadTable(self, items: Iterator[tuple[str, int, bytes]]) -> None: if v is None: continue assert len(v) == cb - d[cw] = v # Remember value + d[cw] = v # Remember value sv.add(v) self.adTab.append(d) @@ -94,35 +92,35 @@ def unpackTables(self, ab: bytes) -> None: for i in range(n): cb, = self.fmtInt.unpack_from(ab, o) o += self.fmtInt.size - data = ab[o:o+cb] + data = ab[o:o + cb] assert len(data) == cb o += cb self.loadTable(HuffTabReader_bin(data)) def propagateMap(self, node: HuffNode) -> None: cw = node.cw - for idx in range(int(cw[::-1], 2), len(self.aMap), 1< None: aCW = sorted(self.dLen.keys())[::-1] minBits, maxBits = len(aCW[0]), len(aCW[-1]) - self.aMap: list[Optional[Any]] = [None]*(1< nBits # Length must increase - s = int(aCW[o-1], 2) # Start value for current length - for i in range(s, e+1): + continue # Run until length change + assert nextBits > nBits # Length must increase + s = int(aCW[o - 1], 2) # Start value for current length + for i in range(s, e + 1): cw = bin(i)[2:].zfill(nBits) self.propagateMap(HuffNode(cw, self)) - e = int(aCW[o], 2)|1 # End value for next length - for i in range(int(e/2) + 1, s): # Handle values with unknown codeword length + e = int(aCW[o], 2) | 1 # End value for next length + for i in range(int(e / 2) + 1, s): # Handle values with unknown codeword length cw = bin(i)[2:].zfill(nBits) self.propagateMap(HuffNode(cw, None)) nBits = nextBits @@ -130,9 +128,9 @@ def prepareMap(self) -> None: assert v is not None def enumCW(self, ab: bytes) -> Iterator[HuffNode]: - v = int(bin(int("01"+ab.hex(), 16))[3:][::-1], 2) # Reversed bits + v = int(bin(int("01" + ab.hex(), 16))[3:][::-1], 2) # Reversed bits cb = 0 - while cb < self.BLOCK_SIZE: # Block length + while cb < self.BLOCK_SIZE: # Block length node = self.aMap[v & 0x7FFF] assert node is not None if node.nBits is None: @@ -159,17 +157,18 @@ def decompress(self, ab: bytes, length: int) -> bytes: nChunks, left = divmod(length, self.BLOCK_SIZE) assert 0 == left aOfs = list(struct.unpack_from("<%dL" % nChunks, ab)) - aOpt = [0]*nChunks + aOpt = [0] * nChunks for i in range(nChunks): aOpt[i], aOfs[i] = divmod(aOfs[i], 0x40000000) - base = nChunks*4 + base = nChunks * 4 aOfs.append(len(ab) - base) r = [] for i, opt in enumerate(aOpt): iTab, bCompr = divmod(opt, 2) assert 1 == bCompr - unpacked = self.decompressChunk(ab[base + aOfs[i]: base + aOfs[i+1]], iTab) + unpacked = self.decompressChunk( + ab[base + aOfs[i]: base + aOfs[i + 1]], iTab) assert len(unpacked) == self.BLOCK_SIZE r.append(unpacked) return b''.join(r) @@ -188,7 +187,7 @@ def get_chunks_offsets(chunk_count: int, data_start: int, data_size: int, for i in range(0, chunk_count): #chunk = llut[0x40 + i * 4:0x44 + i * 4] - chunk = lut_data[(i*4):(i+1)*4] + chunk = lut_data[(i * 4):(i + 1) * 4] offset = 0 if chunk[3] != 0x80: