4983fe2f76
- final persistent validation (cold2): autonomous class-A boot from cache c3732515, live cache hash re-read,
CHARGING/FAST, USB remove/insert recovery, LTE/Wi-Fi/NCM/display PASS, 15-min stability PASS -> B10 CLOSED
- README: B7/B8/B9/B10 CLOSED, production profile (4.20 V / 450 mA / HOLD 4.05 V / full-min 4.15 V / CV 60 min /
timer 512 min / FAULT safe clamp 4.00 V) and the four production limitations; NOTES session 38
- current tree cleaned of raw B10 run logs (monitor/dmesg/snapshot/UART excerpt/action logs/register dumps/emulator
outputs); history (58dcb3d) not rewritten
- publication rules: tools/publish-sanitize.py and .gitignore keep B10+ logs to reports (*.md) and *-diff.txt tables
144 lines
8.6 KiB
Python
Executable file
144 lines
8.6 KiB
Python
Executable file
#!/usr/bin/env python3
|
|
"""Build the sanitized publication tree of Project Aurora.
|
|
|
|
usage: publish-sanitize.py SRC_LOCAL SRC_480S_OVERLAY OUT_DIR IDS_FILE REPORT_DIR
|
|
|
|
- SRC_LOCAL: project copy (mirror); SRC_480S_OVERLAY: files taken from the 480s copy instead
|
|
(longer UART logs, 480s-only text); OUT_DIR must not exist.
|
|
- IDS_FILE: private "KIND<TAB>VALUE" lines (IMEI, EMMC_CID, EMMC_SERIAL, SAHARA_SERIAL, PUBLIC_IP, ...) - never published.
|
|
- Only whitelisted text files are copied; binaries/dumps/NV/firmware are never copied.
|
|
- Originals are never modified.
|
|
- Stages B10+: logs/b1x/ publish ONLY reports (*.md) and compact register-diff tables (*-diff.txt); raw monitor/dmesg/
|
|
snapshot/action/UART/register-dump logs and emulator outputs are never published (also enforced by .gitignore).
|
|
"""
|
|
import os, re, sys, shutil, json
|
|
|
|
SRC, OVL, OUT, IDS, REP = sys.argv[1:6]
|
|
|
|
INCLUDE = [r'^(README\.md|NOTES\.md|MANIFEST\.md|manifest\.tsv|firmware-fat-compare\.tsv)$',
|
|
r'^tools/[^/]+\.(py|sh)$',
|
|
r'^android/(?!logcat\.txt$)[^/]+\.(txt|prop)$',
|
|
r'^b5a/([^/]+\.(sh|h|diff|log)|SHA256SUMS)$',
|
|
r'^b5b/([^/]+\.sh|SHA256SUMS|extlinux\.conf|components/cmdline\.txt)$',
|
|
r'^b5b/emu/([^/]+\.(out|txt|py|sh|c)|srclist|shim/.*\.h)$',
|
|
r'^b6/(aurora-modem[^/]*|b6-run\.sh|ro-check\.sh|SHA256SUMS)$',
|
|
r'^b6j/([^/]+\.(sh|conf|md)|aurora-modem[^/]*|SHA256SUMS)$',
|
|
r'^b6p/([^/]+\.(md|sh|py|diff|conf|sha256)|SHA256SUMS)$',
|
|
r'^b6p/rootfs/(init|usr/sbin/aurora-modem|etc/init\.d/aurora-modem|etc/aurora/modem\.conf)$',
|
|
r'^b6p/out/(SHA256SUMS|manifest-diff\.txt|overlay\.list)$',
|
|
r'^b6p/test/([^/]+\.(sh|txt)|p-datatest|SHA256SUMS)$',
|
|
r'^b6p/emu/[^/]+\.(out|txt|sh)$',
|
|
r'^r2/([^/]+\.(md|sh|py|txt)|SHA256SUMS)$',
|
|
r'^r2/t4/([^/]+\.(sh|conf)|SHA256SUMS|rootfs/init|out/(SHA256SUMS|overlay-t4\.list|t4-unpack/info\.txt))$',
|
|
r'^r2/t4/emu/[^/]+\.(out|txt|sh)$',
|
|
r'^r3/[^/]+/[^/]+\.sh$',
|
|
r'^b7/b7[abc]/([^/]+\.(md|sh|py|diff)|SHA256SUMS)$',
|
|
r'^b7/b7c/rootfs/(usr/sbin/aurora-modem|etc/aurora/build-epoch)$',
|
|
r'^b7/b7c/out/(SHA256SUMS|overlay-b7c\.list)$',
|
|
r'^b7/b7c/cache/([^/]+\.(md|sh|conf)|SHA256SUMS)$',
|
|
r'^b7/b7c/cache/emu/[^/]+\.(out|txt|sh)$',
|
|
r'^b8/b8[a-f]/([^/]+\.(md|sh|py|diff|txt)|SHA256SUMS)$',
|
|
r'^b8/b8[b-f]/out/(SHA256SUMS|overlay-b8[a-f]\.list|init\.diff)$',
|
|
r'^b8/b8[de]/dl/APK-SHA256SUMS$',
|
|
r'^b8/b8e/bundle/(hostapd-b8e\.conf|[^/]+\.sh|SHA256SUMS)$',
|
|
r'^b8/b8f/rootfs/(init|usr/sbin/aurora-wifi|etc/init\.d/aurora-wifi|etc/aurora/wifi\.conf\.in)$',
|
|
r'^b8/b8f/cache/([^/]+\.(md|sh|conf|txt)|SHA256SUMS)$',
|
|
r'^b8/b8f/cache/emu/[^/]+\.(out|txt|sh)$',
|
|
r'^bootchain-migration/([^/]+\.(md|py)|SHA256SUMS)$',
|
|
r'^b9/b9a/([^/]+\.(py|sh)|SHA256SUMS)$',
|
|
r'^b9/b9b/([^/]+\.(sh|py)|SHA256SUMS|rootfs/usr/libexec/aurora/b9b/b9b-test\.sh)$',
|
|
r'^b9/b9b/out/(SHA256SUMS|panel-fw\.txt|overlay-b9b\.list)$',
|
|
r'^b9/b9c/([^/]+\.sh|cmdline\.txt|SHA256SUMS|rootfs/(init|usr/sbin/aurora-display|etc/init\.d/aurora-display))$',
|
|
r'^b9/b9c/out/(SHA256SUMS|init\.diff|overlay-b9c\.list|cmdline\.txt)$',
|
|
r'^b9/b9c/cache/([^/]+\.(md|sh|conf|txt)|SHA256SUMS)$',
|
|
r'^b9/b9c/cache/emu/[^/]+\.(out|txt|sh)$',
|
|
r'^b9/b9l/([^/]+\.(sh|md)|SHA256SUMS|lk2nd-23\.1-aurora-b9l\.patch|splash/mk-splash\.py)$',
|
|
r'^b9/b9l/persist/(B9L-write\.sh|B9L-rollback\.sh|b9l-boot\.sh|B9L-PERSIST-PLAN\.md|config-vs-current-lk1st\.diff|structure-vs-current\.txt|lk2nd-23\.1-aurora-b9l-lk1st\.patch|SHA256SUMS)$',
|
|
r'^bootchain/(elfinfo\.txt|(boot|recovery)/(info\.txt|ramdisk\.list|ramdisk\.type))$',
|
|
r'^b10/b10[a-z0-9]*/([^/]+\.(md|sh|py|diff|patch|txt|conf|c)|SHA256SUMS)$',
|
|
r'^b10/b10[a-z0-9]*/(out[^/]*)/(SHA256SUMS|cmdline\.txt)$',
|
|
r'^b10/b10b3/(cache|cache-v4)/([^/]+\.(md|sh|conf|txt)|SHA256SUMS)$',
|
|
r'^b10/b10b3/(cache|cache-v4)/emu/[^/]+\.(out|txt|sh)$',
|
|
r'^b10/b10b3/rbl/(reboot-bootloader\.c|rbl-test\.sh)$',
|
|
r'^(firehose|gpt)/SHA256SUMS$', r'^partitions/(SHA256SUMS|rawprogram0\.xml)$',
|
|
r'^lk2nd-build/(BUILD\.md|WRITE-PLAN\.md|build\.log|build-lk1st\.log)$',
|
|
r'^linux/([^/]+\.(config|diff|sh|log)|[^/]+-apk-closure[^/]*\.txt)$',
|
|
r'^linux/patches/[^/]+\.patch$', r'^linux/dts/msm8916-jz08-aurora[^/.]*\.dts$',
|
|
r'^linux/b4[^/]*/[^/]+\.sh$', r'^linux/b4b2/tcpdump/APK-SHA256SUMS$', r'^linux/beeline/curl/SHA256SUMS$',
|
|
r'^linux/initramfs[^/]*/(init|initramfs\.list|dl/APK-SHA256SUMS|src/[^/]+\.c|root/etc/(passwd|group))$',
|
|
r'^linux/initramfs[^/]*/root/bin/(?!udevadm$)[^/]+$',
|
|
r'^linux/(artifacts/[^/]+|ramboot[^/]*)/(SHA256SUMS|kernel\.config|cmdline\.txt|[^/.]+\.sh|b4b2-mm[^/]*)$',
|
|
r'^logs/[^/]+\.(log|txt|json)$',
|
|
r'^logs/(?!uart/|b6/)[^/]+/.*\.(txt|log|sh|out|tsv|json|md)$',
|
|
r'^logs/(?!uart/|b6/)[^/]+/.*/[^/.]+$',
|
|
r'^logs/uart/[^/]+\.(log|txt)$',
|
|
r'^logs/b6/all/(?!.*\.tar$)(?!b6/mm-|b6-try\d/(amd/)?mm-).*$',
|
|
]
|
|
NEVER = r'^logs/b1\d[^/]*/(?!.*(\.md$|-diff\.txt$)|b10b3/downstream-7f1748ce/)|^b1\d/.*/emu/[^/]+\.(out|txt)$|^b10/b10a/[^/]+\.txt$|^logs/b10/.*(^|/)uart[^/]*\.log$|^logs/b10/.*/board-logs\.tar$|^b10/.*/out[^/]*/(?!SHA256SUMS$|cmdline\.txt$)|^b10/.*\.(img|bin)$|^logs/b6[jp]/.*(^|/)mm-[^/]*\.log$|^logs/b7/b7c/.*/uart[^/]*\.log$|^logs/b8/.*/uart[^/]*\.log$|^logs/b9/.*/uart[^/]*\.log$|^logs/b9/b9l/W/|^logs/b9/b9l/W-console\.txt$|^logs/b9/.*\.png$|^b9/b9[bc]/out/(?!SHA256SUMS$|overlay-b9[bc]\.list$|init\.diff$|panel-fw\.txt$|cmdline\.txt$)|(^|/)private/|^b8/b8f/out/(?!SHA256SUMS$|overlay-b8f\.list$)|(^|/)(dumps|partitions/.*\.bin|firmware-fat/)|\.(bin|img|mbn|pad\d*[KM]?|tar|tgz|gz|pcap|raw|apk|so|dtb|elf|mdt|b\d\d|dis)$|modemst|(^|/)(fsg|fsc|persist|userdata)\b(?!/)'
|
|
inc = [re.compile(p) for p in INCLUDE]; nev = re.compile(NEVER)
|
|
|
|
ids = []
|
|
for line in open(IDS):
|
|
k, v = line.rstrip('\n').split('\t', 1)
|
|
ids.append((k, v))
|
|
|
|
def is_text(b):
|
|
return b'\0' not in b[:65536]
|
|
|
|
HEXB = r'(?:[0-9A-Fa-f]{2}:){5,}[0-9A-Fa-f]{2}'
|
|
RULES = [
|
|
# QMI RAW/TLV hex payloads carry IMEI/IMSI/ICCID bytes (ModemManager only hides the translated text)
|
|
(re.compile(r'((?:data|value)\s*=\s*)' + HEXB + r'(?:\.\.\.)?'), r'\1<qmi-hex-redacted>'),
|
|
(re.compile(r'(?<![0-9A-Fa-f:])(?:[0-9A-Fa-f]{2}:){5}[0-9A-Fa-f]{2}(?![0-9A-Fa-f:])'), '<MAC>'),
|
|
(re.compile(r'4G-MIFI-[0-9A-Za-z_]+'), '4G-MIFI-<SSID-SUFFIX>'),
|
|
(re.compile(r'\+7\d{7,}|\+\*{4,}\d{3,}'), '<PHONE>'),
|
|
(re.compile(r'\b\d{4,8}\*{4,}\d{3,6}\b|\*{6,}\d{3,6}\b'), '<REDACTED>'),
|
|
(re.compile(r'(?i)\b(Global Cell ID|Cell ID|Tracking Area Code|Location Area Code|ECI|GCI|TAC|LAC)(\b[\'": =]{0,4})(0x[0-9a-f]+|\d{3,})'),
|
|
r'\1\2<CELL>'),
|
|
# HTTP test responses: tracking cookies, captcha/redirect tokens, request ids
|
|
(re.compile(r'(?im)^(set-cookie:\s*).*$'), r'\1<redacted>'),
|
|
(re.compile(r'(?im)^(location:\s*[^?\s]*\?).*$'), r'\1<redacted>'),
|
|
(re.compile(r'(?im)^(x-yandex-req-id:\s*).*$'), r'\1<redacted>'),
|
|
]
|
|
|
|
def sanitize(t, counts):
|
|
for k, v in ids:
|
|
for vv in {v, v.upper(), v.lower()}:
|
|
if vv in t:
|
|
counts[k] = counts.get(k, 0) + t.count(vv); t = t.replace(vv, f'<{k}>')
|
|
for rx, rep in RULES:
|
|
t, n = rx.subn(rep, t)
|
|
if n: counts[rx.pattern[:40]] = counts.get(rx.pattern[:40], 0) + n
|
|
return t
|
|
|
|
def walk(root):
|
|
for d, _, fs in os.walk(root):
|
|
for f in fs:
|
|
p = os.path.relpath(os.path.join(d, f), root)
|
|
if p.startswith(('linux/src/', 'linux/out/', 'lk2nd-build/src/')): continue
|
|
yield p
|
|
|
|
src = {p: os.path.join(SRC, p) for p in walk(SRC)}
|
|
for p in walk(OVL): src[p] = os.path.join(OVL, p) # 480s overlay wins (longer/only-480s)
|
|
os.makedirs(OUT); os.makedirs(REP, exist_ok=True)
|
|
taken, skipped, total = [], [], {}
|
|
for p in sorted(src):
|
|
if not any(r.search(p) for r in inc) or nev.search(p):
|
|
skipped.append(p); continue
|
|
b = open(src[p], 'rb').read()
|
|
uart = p.startswith('logs/uart/')
|
|
if not uart and not is_text(b):
|
|
skipped.append(p + ' [binary]'); continue
|
|
t = b.decode('utf-8', 'replace'); c = {}
|
|
if uart: # serial line noise: NULs, stray control bytes, ANSI escapes
|
|
t = re.sub(r'\x1b\[[0-9;?]*[A-Za-z]', '', t)
|
|
t = re.sub(r'[\x00-\x08\x0b\x0c\x0e-\x1f\x7f]', '', t)
|
|
t = sanitize(t, c)
|
|
o = os.path.join(OUT, p); os.makedirs(os.path.dirname(o), exist_ok=True)
|
|
open(o, 'w').write(t); shutil.copymode(src[p], o)
|
|
taken.append((p, len(t.encode()), c))
|
|
for k, n in c.items(): total[k] = total.get(k, 0) + n
|
|
with open(os.path.join(REP, 'taken.tsv'), 'w') as f:
|
|
for p, n, c in taken: f.write(f'{p}\t{n}\t{json.dumps(c)}\n')
|
|
open(os.path.join(REP, 'skipped.txt'), 'w').write('\n'.join(skipped) + '\n')
|
|
print(f'taken {len(taken)} files, skipped {len(skipped)}'); print(json.dumps(total, indent=1))
|