8d7721c775
B7A: QMI DMS time vs VNIIFTRI NTP (DMS = UTC - 0.28 s, sigma 1 ms; NITZ = DMS truncated to 1 s). B7B: a 1970 -> 2026 date -s step is harmless for the modem stack. B7C: aurora-modem probes DMS/NITZ along the start flow and steps CLOCK_REALTIME once at the first valid sample (REG registered/attached), never RTC; background SNTP check (query only) after START OK. 3/3 class-A cold boots PASS (residual +0.04/+0.36/+0.84 s), cache 14fe453a written (B7C_WRITE_VERIFIED) and verified by a normal power-on (V1, +0.26 s). Also publishes the prerequisite R2 (boot-hang / lk eMMC investigation, T4 telnet baseline that B7C builds on) and R3 (autonomous cold boot, NCM loss) material, and extends tools/publish-sanitize.py to r2/, r3/, b7/. Binary images, initramfs, busybox and raw logs stay out (see b7/*/SHA256SUMS).
121 lines
6.3 KiB
Python
Executable file
121 lines
6.3 KiB
Python
Executable file
#!/usr/bin/env python3
|
|
"""Build the sanitized publication tree of Project Aurora.
|
|
|
|
usage: publish-sanitize.py SRC_LOCAL SRC_480S_OVERLAY OUT_DIR IDS_FILE REPORT_DIR
|
|
|
|
- SRC_LOCAL: project copy (mirror); SRC_480S_OVERLAY: files taken from the 480s copy instead
|
|
(longer UART logs, 480s-only text); OUT_DIR must not exist.
|
|
- IDS_FILE: private "KIND<TAB>VALUE" lines (IMEI, EMMC_CID, EMMC_SERIAL, SAHARA_SERIAL, PUBLIC_IP, ...) - never published.
|
|
- Only whitelisted text files are copied; binaries/dumps/NV/firmware are never copied.
|
|
- Originals are never modified.
|
|
"""
|
|
import os, re, sys, shutil, json
|
|
|
|
SRC, OVL, OUT, IDS, REP = sys.argv[1:6]
|
|
|
|
INCLUDE = [r'^(README\.md|NOTES\.md|MANIFEST\.md|manifest\.tsv|firmware-fat-compare\.tsv)$',
|
|
r'^tools/[^/]+\.(py|sh)$',
|
|
r'^android/(?!logcat\.txt$)[^/]+\.(txt|prop)$',
|
|
r'^b5a/([^/]+\.(sh|h|diff|log)|SHA256SUMS)$',
|
|
r'^b5b/([^/]+\.sh|SHA256SUMS|extlinux\.conf|components/cmdline\.txt)$',
|
|
r'^b5b/emu/([^/]+\.(out|txt|py|sh|c)|srclist|shim/.*\.h)$',
|
|
r'^b6/(aurora-modem[^/]*|b6-run\.sh|ro-check\.sh|SHA256SUMS)$',
|
|
r'^b6j/([^/]+\.(sh|conf|md)|aurora-modem[^/]*|SHA256SUMS)$',
|
|
r'^b6p/([^/]+\.(md|sh|py|diff|conf|sha256)|SHA256SUMS)$',
|
|
r'^b6p/rootfs/(init|usr/sbin/aurora-modem|etc/init\.d/aurora-modem|etc/aurora/modem\.conf)$',
|
|
r'^b6p/out/(SHA256SUMS|manifest-diff\.txt|overlay\.list)$',
|
|
r'^b6p/test/([^/]+\.(sh|txt)|p-datatest|SHA256SUMS)$',
|
|
r'^b6p/emu/[^/]+\.(out|txt|sh)$',
|
|
r'^r2/([^/]+\.(md|sh|py|txt)|SHA256SUMS)$',
|
|
r'^r2/t4/([^/]+\.(sh|conf)|SHA256SUMS|rootfs/init|out/(SHA256SUMS|overlay-t4\.list|t4-unpack/info\.txt))$',
|
|
r'^r2/t4/emu/[^/]+\.(out|txt|sh)$',
|
|
r'^r3/[^/]+/[^/]+\.sh$',
|
|
r'^b7/b7[abc]/([^/]+\.(md|sh|py|diff)|SHA256SUMS)$',
|
|
r'^b7/b7c/rootfs/(usr/sbin/aurora-modem|etc/aurora/build-epoch)$',
|
|
r'^b7/b7c/out/(SHA256SUMS|overlay-b7c\.list)$',
|
|
r'^b7/b7c/cache/([^/]+\.(md|sh|conf)|SHA256SUMS)$',
|
|
r'^b7/b7c/cache/emu/[^/]+\.(out|txt|sh)$',
|
|
r'^bootchain-migration/([^/]+\.(md|py)|SHA256SUMS)$',
|
|
r'^bootchain/(elfinfo\.txt|(boot|recovery)/(info\.txt|ramdisk\.list|ramdisk\.type))$',
|
|
r'^(firehose|gpt)/SHA256SUMS$', r'^partitions/(SHA256SUMS|rawprogram0\.xml)$',
|
|
r'^lk2nd-build/(BUILD\.md|WRITE-PLAN\.md|build\.log|build-lk1st\.log)$',
|
|
r'^linux/([^/]+\.(config|diff|sh|log)|[^/]+-apk-closure[^/]*\.txt)$',
|
|
r'^linux/patches/[^/]+\.patch$', r'^linux/dts/msm8916-jz08-aurora[^/.]*\.dts$',
|
|
r'^linux/b4[^/]*/[^/]+\.sh$', r'^linux/b4b2/tcpdump/APK-SHA256SUMS$', r'^linux/beeline/curl/SHA256SUMS$',
|
|
r'^linux/initramfs[^/]*/(init|initramfs\.list|dl/APK-SHA256SUMS|src/[^/]+\.c|root/etc/(passwd|group))$',
|
|
r'^linux/initramfs[^/]*/root/bin/(?!udevadm$)[^/]+$',
|
|
r'^linux/(artifacts/[^/]+|ramboot[^/]*)/(SHA256SUMS|kernel\.config|cmdline\.txt|[^/.]+\.sh|b4b2-mm[^/]*)$',
|
|
r'^logs/[^/]+\.(log|txt|json)$',
|
|
r'^logs/(?!uart/|b6/)[^/]+/.*\.(txt|log|sh|out|tsv|json|md)$',
|
|
r'^logs/(?!uart/|b6/)[^/]+/.*/[^/.]+$',
|
|
r'^logs/uart/[^/]+\.(log|txt)$',
|
|
r'^logs/b6/all/(?!.*\.tar$)(?!b6/mm-|b6-try\d/(amd/)?mm-).*$',
|
|
]
|
|
NEVER = r'^logs/b6[jp]/.*(^|/)mm-[^/]*\.log$|^logs/b7/b7c/.*/uart[^/]*\.log$|(^|/)(dumps|partitions/.*\.bin|firmware-fat/)|\.(bin|img|mbn|pad\d*[KM]?|tar|tgz|gz|pcap|raw|apk|so|dtb|elf|mdt|b\d\d|dis)$|modemst|(^|/)(fsg|fsc|persist|userdata)\b'
|
|
inc = [re.compile(p) for p in INCLUDE]; nev = re.compile(NEVER)
|
|
|
|
ids = []
|
|
for line in open(IDS):
|
|
k, v = line.rstrip('\n').split('\t', 1)
|
|
ids.append((k, v))
|
|
|
|
def is_text(b):
|
|
return b'\0' not in b[:65536]
|
|
|
|
HEXB = r'(?:[0-9A-Fa-f]{2}:){5,}[0-9A-Fa-f]{2}'
|
|
RULES = [
|
|
# QMI RAW/TLV hex payloads carry IMEI/IMSI/ICCID bytes (ModemManager only hides the translated text)
|
|
(re.compile(r'((?:data|value)\s*=\s*)' + HEXB + r'(?:\.\.\.)?'), r'\1<qmi-hex-redacted>'),
|
|
(re.compile(r'(?<![0-9A-Fa-f:])(?:[0-9A-Fa-f]{2}:){5}[0-9A-Fa-f]{2}(?![0-9A-Fa-f:])'), '<MAC>'),
|
|
(re.compile(r'4G-MIFI-[0-9A-Za-z_]+'), '4G-MIFI-<SSID-SUFFIX>'),
|
|
(re.compile(r'\+7\d{7,}|\+\*{4,}\d{3,}'), '<PHONE>'),
|
|
(re.compile(r'\b\d{4,8}\*{4,}\d{3,6}\b|\*{6,}\d{3,6}\b'), '<REDACTED>'),
|
|
(re.compile(r'(?i)\b(Global Cell ID|Cell ID|Tracking Area Code|Location Area Code|ECI|GCI|TAC|LAC)(\b[\'": =]{0,4})(0x[0-9a-f]+|\d{3,})'),
|
|
r'\1\2<CELL>'),
|
|
# HTTP test responses: tracking cookies, captcha/redirect tokens, request ids
|
|
(re.compile(r'(?im)^(set-cookie:\s*).*$'), r'\1<redacted>'),
|
|
(re.compile(r'(?im)^(location:\s*[^?\s]*\?).*$'), r'\1<redacted>'),
|
|
(re.compile(r'(?im)^(x-yandex-req-id:\s*).*$'), r'\1<redacted>'),
|
|
]
|
|
|
|
def sanitize(t, counts):
|
|
for k, v in ids:
|
|
for vv in {v, v.upper(), v.lower()}:
|
|
if vv in t:
|
|
counts[k] = counts.get(k, 0) + t.count(vv); t = t.replace(vv, f'<{k}>')
|
|
for rx, rep in RULES:
|
|
t, n = rx.subn(rep, t)
|
|
if n: counts[rx.pattern[:40]] = counts.get(rx.pattern[:40], 0) + n
|
|
return t
|
|
|
|
def walk(root):
|
|
for d, _, fs in os.walk(root):
|
|
for f in fs:
|
|
p = os.path.relpath(os.path.join(d, f), root)
|
|
if p.startswith(('linux/src/', 'linux/out/', 'lk2nd-build/src/')): continue
|
|
yield p
|
|
|
|
src = {p: os.path.join(SRC, p) for p in walk(SRC)}
|
|
for p in walk(OVL): src[p] = os.path.join(OVL, p) # 480s overlay wins (longer/only-480s)
|
|
os.makedirs(OUT); os.makedirs(REP, exist_ok=True)
|
|
taken, skipped, total = [], [], {}
|
|
for p in sorted(src):
|
|
if not any(r.search(p) for r in inc) or nev.search(p):
|
|
skipped.append(p); continue
|
|
b = open(src[p], 'rb').read()
|
|
uart = p.startswith('logs/uart/')
|
|
if not uart and not is_text(b):
|
|
skipped.append(p + ' [binary]'); continue
|
|
t = b.decode('utf-8', 'replace'); c = {}
|
|
if uart: # serial line noise: NULs, stray control bytes, ANSI escapes
|
|
t = re.sub(r'\x1b\[[0-9;?]*[A-Za-z]', '', t)
|
|
t = re.sub(r'[\x00-\x08\x0b\x0c\x0e-\x1f\x7f]', '', t)
|
|
t = sanitize(t, c)
|
|
o = os.path.join(OUT, p); os.makedirs(os.path.dirname(o), exist_ok=True)
|
|
open(o, 'w').write(t); shutil.copymode(src[p], o)
|
|
taken.append((p, len(t.encode()), c))
|
|
for k, n in c.items(): total[k] = total.get(k, 0) + n
|
|
with open(os.path.join(REP, 'taken.tsv'), 'w') as f:
|
|
for p, n, c in taken: f.write(f'{p}\t{n}\t{json.dumps(c)}\n')
|
|
open(os.path.join(REP, 'skipped.txt'), 'w').write('\n'.join(skipped) + '\n')
|
|
print(f'taken {len(taken)} files, skipped {len(skipped)}'); print(json.dumps(total, indent=1))
|