#!/usr/bin/env python3
"""Build the sanitized publication tree of Project Aurora.

usage: publish-sanitize.py SRC_LOCAL SRC_480S_OVERLAY OUT_DIR IDS_FILE REPORT_DIR

- SRC_LOCAL: project copy (mirror); SRC_480S_OVERLAY: files taken from the 480s copy instead
  (longer UART logs, 480s-only text); OUT_DIR must not exist.
- IDS_FILE: private "KIND<TAB>VALUE" lines (IMEI, EMMC_CID, EMMC_SERIAL, SAHARA_SERIAL, PUBLIC_IP, ...) - never published.
- Only whitelisted text files are copied; binaries/dumps/NV/firmware are never copied.
- Originals are never modified.
"""
import os, re, sys, shutil, json

SRC, OVL, OUT, IDS, REP = sys.argv[1:6]

INCLUDE = [r'^(README\.md|NOTES\.md|MANIFEST\.md|manifest\.tsv|firmware-fat-compare\.tsv)$',
    r'^tools/[^/]+\.(py|sh)$',
    r'^android/(?!logcat\.txt$)[^/]+\.(txt|prop)$',
    r'^b5a/([^/]+\.(sh|h|diff|log)|SHA256SUMS)$',
    r'^b5b/([^/]+\.sh|SHA256SUMS|extlinux\.conf|components/cmdline\.txt)$',
    r'^b5b/emu/([^/]+\.(out|txt|py|sh|c)|srclist|shim/.*\.h)$',
    r'^b6/(aurora-modem[^/]*|b6-run\.sh|ro-check\.sh|SHA256SUMS)$',
    r'^b6j/([^/]+\.(sh|conf|md)|aurora-modem[^/]*|SHA256SUMS)$',
    r'^b6p/([^/]+\.(md|sh|py|diff|conf|sha256)|SHA256SUMS)$',
    r'^b6p/rootfs/(init|usr/sbin/aurora-modem|etc/init\.d/aurora-modem|etc/aurora/modem\.conf)$',
    r'^b6p/out/(SHA256SUMS|manifest-diff\.txt|overlay\.list)$',
    r'^b6p/test/([^/]+\.(sh|txt)|p-datatest|SHA256SUMS)$',
    r'^b6p/emu/[^/]+\.(out|txt|sh)$',
    r'^r2/([^/]+\.(md|sh|py|txt)|SHA256SUMS)$',
    r'^r2/t4/([^/]+\.(sh|conf)|SHA256SUMS|rootfs/init|out/(SHA256SUMS|overlay-t4\.list|t4-unpack/info\.txt))$',
    r'^r2/t4/emu/[^/]+\.(out|txt|sh)$',
    r'^r3/[^/]+/[^/]+\.sh$',
    r'^b7/b7[abc]/([^/]+\.(md|sh|py|diff)|SHA256SUMS)$',
    r'^b7/b7c/rootfs/(usr/sbin/aurora-modem|etc/aurora/build-epoch)$',
    r'^b7/b7c/out/(SHA256SUMS|overlay-b7c\.list)$',
    r'^b7/b7c/cache/([^/]+\.(md|sh|conf)|SHA256SUMS)$',
    r'^b7/b7c/cache/emu/[^/]+\.(out|txt|sh)$',
    r'^b8/b8[a-f]/([^/]+\.(md|sh|py|diff|txt)|SHA256SUMS)$',
    r'^b8/b8[b-f]/out/(SHA256SUMS|overlay-b8[a-f]\.list|init\.diff)$',
    r'^b8/b8[de]/dl/APK-SHA256SUMS$',
    r'^b8/b8e/bundle/(hostapd-b8e\.conf|[^/]+\.sh|SHA256SUMS)$',
    r'^b8/b8f/rootfs/(init|usr/sbin/aurora-wifi|etc/init\.d/aurora-wifi|etc/aurora/wifi\.conf\.in)$',
    r'^b8/b8f/cache/([^/]+\.(md|sh|conf|txt)|SHA256SUMS)$',
    r'^b8/b8f/cache/emu/[^/]+\.(out|txt|sh)$',
    r'^bootchain-migration/([^/]+\.(md|py)|SHA256SUMS)$',
    r'^b9/b9a/([^/]+\.(py|sh)|SHA256SUMS)$',
    r'^b9/b9b/([^/]+\.(sh|py)|SHA256SUMS|rootfs/usr/libexec/aurora/b9b/b9b-test\.sh)$',
    r'^b9/b9b/out/(SHA256SUMS|panel-fw\.txt|overlay-b9b\.list)$',
    r'^b9/b9c/([^/]+\.sh|cmdline\.txt|SHA256SUMS|rootfs/(init|usr/sbin/aurora-display|etc/init\.d/aurora-display))$',
    r'^b9/b9c/out/(SHA256SUMS|init\.diff|overlay-b9c\.list|cmdline\.txt)$',
    r'^b9/b9c/cache/([^/]+\.(md|sh|conf|txt)|SHA256SUMS)$',
    r'^b9/b9c/cache/emu/[^/]+\.(out|txt|sh)$',
    r'^b9/b9l/([^/]+\.(sh|md)|SHA256SUMS|lk2nd-23\.1-aurora-b9l\.patch|splash/mk-splash\.py)$',
    r'^b9/b9l/persist/(B9L-write\.sh|B9L-rollback\.sh|b9l-boot\.sh|B9L-PERSIST-PLAN\.md|config-vs-current-lk1st\.diff|structure-vs-current\.txt|lk2nd-23\.1-aurora-b9l-lk1st\.patch|SHA256SUMS)$',
    r'^bootchain/(elfinfo\.txt|(boot|recovery)/(info\.txt|ramdisk\.list|ramdisk\.type))$',
    r'^b10/b10[a-z0-9]*/([^/]+\.(md|sh|py|diff|patch|txt|conf|c)|SHA256SUMS)$',
    r'^b10/b10[a-z0-9]*/(out[^/]*)/(SHA256SUMS|cmdline\.txt)$',
    r'^b10/b10b3/(cache|cache-v4)/([^/]+\.(md|sh|conf|txt)|SHA256SUMS)$',
    r'^b10/b10b3/(cache|cache-v4)/emu/[^/]+\.(out|txt|sh)$',
    r'^b10/b10b3/rbl/(reboot-bootloader\.c|rbl-test\.sh)$',
    r'^(firehose|gpt)/SHA256SUMS$', r'^partitions/(SHA256SUMS|rawprogram0\.xml)$',
    r'^lk2nd-build/(BUILD\.md|WRITE-PLAN\.md|build\.log|build-lk1st\.log)$',
    r'^linux/([^/]+\.(config|diff|sh|log)|[^/]+-apk-closure[^/]*\.txt)$',
    r'^linux/patches/[^/]+\.patch$', r'^linux/dts/msm8916-jz08-aurora[^/.]*\.dts$',
    r'^linux/b4[^/]*/[^/]+\.sh$', r'^linux/b4b2/tcpdump/APK-SHA256SUMS$', r'^linux/beeline/curl/SHA256SUMS$',
    r'^linux/initramfs[^/]*/(init|initramfs\.list|dl/APK-SHA256SUMS|src/[^/]+\.c|root/etc/(passwd|group))$',
    r'^linux/initramfs[^/]*/root/bin/(?!udevadm$)[^/]+$',
    r'^linux/(artifacts/[^/]+|ramboot[^/]*)/(SHA256SUMS|kernel\.config|cmdline\.txt|[^/.]+\.sh|b4b2-mm[^/]*)$',
    r'^logs/[^/]+\.(log|txt|json)$',
    r'^logs/(?!uart/|b6/)[^/]+/.*\.(txt|log|sh|out|tsv|json|md)$',
    r'^logs/(?!uart/|b6/)[^/]+/.*/[^/.]+$',
    r'^logs/uart/[^/]+\.(log|txt)$',
    r'^logs/b6/all/(?!.*\.tar$)(?!b6/mm-|b6-try\d/(amd/)?mm-).*$',
]
NEVER = r'^logs/b10/.*(^|/)uart[^/]*\.log$|^logs/b10/.*/board-logs\.tar$|^b10/.*/out[^/]*/(?!SHA256SUMS$|cmdline\.txt$)|^b10/.*\.(img|bin)$|^logs/b6[jp]/.*(^|/)mm-[^/]*\.log$|^logs/b7/b7c/.*/uart[^/]*\.log$|^logs/b8/.*/uart[^/]*\.log$|^logs/b9/.*/uart[^/]*\.log$|^logs/b9/b9l/W/|^logs/b9/b9l/W-console\.txt$|^logs/b9/.*\.png$|^b9/b9[bc]/out/(?!SHA256SUMS$|overlay-b9[bc]\.list$|init\.diff$|panel-fw\.txt$|cmdline\.txt$)|(^|/)private/|^b8/b8f/out/(?!SHA256SUMS$|overlay-b8f\.list$)|(^|/)(dumps|partitions/.*\.bin|firmware-fat/)|\.(bin|img|mbn|pad\d*[KM]?|tar|tgz|gz|pcap|raw|apk|so|dtb|elf|mdt|b\d\d|dis)$|modemst|(^|/)(fsg|fsc|persist|userdata)\b(?!/)'
inc = [re.compile(p) for p in INCLUDE]; nev = re.compile(NEVER)

ids = []
for line in open(IDS):
    k, v = line.rstrip('\n').split('\t', 1)
    ids.append((k, v))

def is_text(b):
    return b'\0' not in b[:65536]

HEXB = r'(?:[0-9A-Fa-f]{2}:){5,}[0-9A-Fa-f]{2}'
RULES = [
    # QMI RAW/TLV hex payloads carry IMEI/IMSI/ICCID bytes (ModemManager only hides the translated text)
    (re.compile(r'((?:data|value)\s*=\s*)' + HEXB + r'(?:\.\.\.)?'), r'\1<qmi-hex-redacted>'),
    (re.compile(r'(?<![0-9A-Fa-f:])(?:[0-9A-Fa-f]{2}:){5}[0-9A-Fa-f]{2}(?![0-9A-Fa-f:])'), '<MAC>'),
    (re.compile(r'4G-MIFI-[0-9A-Za-z_]+'), '4G-MIFI-<SSID-SUFFIX>'),
    (re.compile(r'\+7\d{7,}|\+\*{4,}\d{3,}'), '<PHONE>'),
    (re.compile(r'\b\d{4,8}\*{4,}\d{3,6}\b|\*{6,}\d{3,6}\b'), '<REDACTED>'),
    (re.compile(r'(?i)\b(Global Cell ID|Cell ID|Tracking Area Code|Location Area Code|ECI|GCI|TAC|LAC)(\b[\'": =]{0,4})(0x[0-9a-f]+|\d{3,})'),
     r'\1\2<CELL>'),
    # HTTP test responses: tracking cookies, captcha/redirect tokens, request ids
    (re.compile(r'(?im)^(set-cookie:\s*).*$'), r'\1<redacted>'),
    (re.compile(r'(?im)^(location:\s*[^?\s]*\?).*$'), r'\1<redacted>'),
    (re.compile(r'(?im)^(x-yandex-req-id:\s*).*$'), r'\1<redacted>'),
]

def sanitize(t, counts):
    for k, v in ids:
        for vv in {v, v.upper(), v.lower()}:
            if vv in t:
                counts[k] = counts.get(k, 0) + t.count(vv); t = t.replace(vv, f'<{k}>')
    for rx, rep in RULES:
        t, n = rx.subn(rep, t)
        if n: counts[rx.pattern[:40]] = counts.get(rx.pattern[:40], 0) + n
    return t

def walk(root):
    for d, _, fs in os.walk(root):
        for f in fs:
            p = os.path.relpath(os.path.join(d, f), root)
            if p.startswith(('linux/src/', 'linux/out/', 'lk2nd-build/src/')): continue
            yield p

src = {p: os.path.join(SRC, p) for p in walk(SRC)}
for p in walk(OVL): src[p] = os.path.join(OVL, p)          # 480s overlay wins (longer/only-480s)
os.makedirs(OUT); os.makedirs(REP, exist_ok=True)
taken, skipped, total = [], [], {}
for p in sorted(src):
    if not any(r.search(p) for r in inc) or nev.search(p):
        skipped.append(p); continue
    b = open(src[p], 'rb').read()
    uart = p.startswith('logs/uart/')
    if not uart and not is_text(b):
        skipped.append(p + '  [binary]'); continue
    t = b.decode('utf-8', 'replace'); c = {}
    if uart:   # serial line noise: NULs, stray control bytes, ANSI escapes
        t = re.sub(r'\x1b\[[0-9;?]*[A-Za-z]', '', t)
        t = re.sub(r'[\x00-\x08\x0b\x0c\x0e-\x1f\x7f]', '', t)
    t = sanitize(t, c)
    o = os.path.join(OUT, p); os.makedirs(os.path.dirname(o), exist_ok=True)
    open(o, 'w').write(t); shutil.copymode(src[p], o)
    taken.append((p, len(t.encode()), c))
    for k, n in c.items(): total[k] = total.get(k, 0) + n
with open(os.path.join(REP, 'taken.tsv'), 'w') as f:
    for p, n, c in taken: f.write(f'{p}\t{n}\t{json.dumps(c)}\n')
open(os.path.join(REP, 'skipped.txt'), 'w').write('\n'.join(skipped) + '\n')
print(f'taken {len(taken)} files, skipped {len(skipped)}'); print(json.dumps(total, indent=1))