106 lines
4 KiB
Python
Executable file
106 lines
4 KiB
Python
Executable file
#!/usr/bin/env python3
|
|
"""Walk linux-firmware git history for MediaTek Connac2 WiFi blobs.
|
|
|
|
For every revision of every mt79*_wm/wa/rom_patch/wo file, extract
|
|
version-recovering metadata (kernel trailer date, hidden build string,
|
|
region table) and emit JSONL. Metadata only — no blob bytes are stored.
|
|
|
|
Usage: python3 tools/fw_history.py [linux-firmware-repo] -o dataset/history.jsonl
|
|
"""
|
|
import argparse
|
|
import hashlib
|
|
import json
|
|
import re
|
|
import subprocess
|
|
import struct
|
|
import sys
|
|
from pathlib import Path
|
|
|
|
sys.path.insert(0, str(Path(__file__).resolve().parent))
|
|
from mtk_fw_extract import PATCH_HDR, REGION, TRAILER # noqa: E402
|
|
|
|
TARGET_RE = re.compile(r'^mediatek/mt79(15|16|81|86)_(wm|wa|rom_patch|wo(_[01])?)\.bin$')
|
|
|
|
|
|
def git(repo, *args, binary=False):
|
|
r = subprocess.run(['git', '-C', str(repo), *args], capture_output=True)
|
|
if r.returncode:
|
|
raise RuntimeError(r.stderr.decode())
|
|
return r.stdout if binary else r.stdout.decode()
|
|
|
|
|
|
|
|
def peek_ram(d: bytes) -> dict:
|
|
t = TRAILER.unpack_from(d, len(d) - TRAILER.size)
|
|
chip, eco, n = t[0], t[1], t[2]
|
|
bdate = t[7].decode(errors='replace')
|
|
table = len(d) - TRAILER.size - n * REGION.size
|
|
off, addrs, feats = 0, [], []
|
|
for i in range(n):
|
|
r = REGION.unpack_from(d, table + i * REGION.size)
|
|
addrs.append(f'0x{r[4]:08x}')
|
|
feats.append(f'0x{r[6]:02x}')
|
|
off += r[5]
|
|
hidden = None
|
|
gap = table - off
|
|
if gap > 0:
|
|
runs = [x.strip(b'#') for x in re.findall(rb'[\x20-\x7e]{8,}', d[off:table])]
|
|
runs = [x for x in runs if len(x) >= 8]
|
|
hidden = runs[0].decode() if runs else None
|
|
return {'chip_id': chip, 'eco': eco, 'n_region': n,
|
|
'trailer_fw_ver': t[6].decode(errors='replace'),
|
|
'trailer_date': bdate, 'hidden': hidden,
|
|
'region_addrs': addrs, 'region_feats': feats, 'gap': gap}
|
|
|
|
|
|
def peek_patch(d: bytes) -> dict:
|
|
h = PATCH_HDR.unpack_from(d, 0)
|
|
n = h[9]
|
|
return {'platform': h[1].decode(errors='replace'),
|
|
'patch_date': h[0].decode(errors='replace').strip('\x00'),
|
|
'hw_sw_ver': f'0x{h[2]:08x}', 'n_section': n}
|
|
|
|
|
|
def main():
|
|
ap = argparse.ArgumentParser()
|
|
ap.add_argument('repo', nargs='?', default='firmware/linux-firmware')
|
|
ap.add_argument('-o', '--out', default='dataset/history.jsonl')
|
|
args = ap.parse_args()
|
|
repo = Path(args.repo)
|
|
head = git(repo, 'rev-parse', 'HEAD').strip()
|
|
files = [f for f in git(repo, 'ls-tree', '-r', '--name-only', 'HEAD',
|
|
'mediatek').splitlines()
|
|
if TARGET_RE.match(f)]
|
|
out = Path(args.out)
|
|
out.parent.mkdir(parents=True, exist_ok=True)
|
|
n_rev = 0
|
|
with out.open('w') as fh:
|
|
for f in sorted(files):
|
|
log = git(repo, 'log', '--format=%H%x00%cs%x00%s', '--', f)
|
|
revs = [l.split('\x00') for l in log.strip().splitlines()]
|
|
for commit, date, subject in reversed(revs): # oldest first
|
|
d = git(repo, 'cat-file', 'blob', f'{commit}:{f}', binary=True)
|
|
rec = {'file': f, 'commit': commit[:12], 'date': date,
|
|
'subject': subject, 'repo_head': head[:12],
|
|
'sha256': hashlib.sha256(d).hexdigest()[:16],
|
|
'size': len(d)}
|
|
try:
|
|
if '_rom_patch' in f:
|
|
rec.update(peek_patch(d))
|
|
elif '_wo' in f:
|
|
rec['format'] = 'wed-wo (unsupported parser)'
|
|
else:
|
|
rec.update(peek_ram(d))
|
|
except Exception as e: # noqa: BLE001
|
|
rec['parse_error'] = str(e)
|
|
fh.write(json.dumps(rec) + '\n')
|
|
n_rev += 1
|
|
hv = rec.get('hidden') or rec.get('patch_date') or ''
|
|
print(f"{f.split('/')[-1]:26s} {date} {commit[:8]} "
|
|
f"{rec.get('trailer_date', rec.get('patch_date', '?'))} "
|
|
f"{hv[:60]}")
|
|
print(f'--- {n_rev} revisions -> {out}')
|
|
|
|
|
|
if __name__ == '__main__':
|
|
main()
|