DSCF

.co.uk

PC32

A ColorForth compiler and editor for x86-64
git clone https://dscf.co.uk/clone/PC32.git
Log | Files | Refs | README

mk32root.py (7702B)


      1 #!/usr/bin/env python3
      2 """Build a local 32-bit (i386) sysroot without root and without multilib.
      3 
      4 Downloads the needed i386 .debs from the Debian archive, resolves their
      5 dependencies, and unpacks them in pure Python (no dpkg/root/distro tools)
      6 into a directory. Works on any Linux with python3.
      7 Also fetches a native nasm if the host has none.
      8 
      9 usage: mk32root.py SYSROOT_DIR
     10 """
     11 import os, re, sys, io, lzma, gzip, tarfile, urllib.request
     12 
     13 MIRROR = "http://archive.debian.org/debian"
     14 SUITE = "buster"
     15 SEEDS = ["libc6-dev", "libgcc-8-dev", "libglfw3-dev", "libglfw3", "libmikmod-dev",
     16          "libgl1-mesa-dev", "libglu1-mesa-dev", "libstdc++-8-dev", "libx11-dev", "libxrandr-dev", "libxinerama-dev", "libxcursor-dev", "libxi-dev", "libxxf86vm-dev", "libxext-dev", "libgl1-mesa-dri", "libgl1", "libglx-mesa0"]
     17 # deps we never need to pull in (not libraries, or provided by the host)
     18 SKIP = {"debconf", "dpkg", "perl-base", "lsb-base", "multiarch-support"}
     19 
     20 def load(arch, dest):
     21     url = f"{MIRROR}/dists/{SUITE}/main/binary-{arch}/Packages.xz"
     22     print("fetching index", url, flush=True)
     23     raw = lzma.decompress(urllib.request.urlopen(url).read()).decode()
     24     pk, prov = {}, {}
     25     for blk in raw.split("\n\n"):
     26         f = dict(re.findall(r"^([A-Za-z-]+): (.*)$", blk, re.M))
     27         if "Package" not in f: continue
     28         pk[f["Package"]] = f
     29         for p in f.get("Provides", "").split(","):
     30             p = p.strip().split(" ")[0]
     31             if p: prov.setdefault(p, f["Package"])
     32     return pk, prov
     33 
     34 def deps(f):
     35     out = []
     36     for fld in ("Depends", "Pre-Depends"):
     37         for alt in f.get(fld, "").split(","):
     38             alt = alt.strip()
     39             if alt: out.append([re.sub(r"[:\s(].*", "", a.strip()) for a in alt.split("|")])
     40     return out
     41 
     42 def closure(pk, prov, seeds):
     43     seen, todo = [], list(seeds)
     44     while todo:
     45         n = todo.pop()
     46         n = n if n in pk else prov.get(n)
     47         if not n or n in seen or n in SKIP: continue
     48         seen.append(n)
     49         for alts in deps(pk[n]):
     50             # take the first alternative that exists (prefer already-chosen)
     51             pick = next((a for a in alts if a in seen), None) or \
     52                    next((a for a in alts if a in pk or a in prov), None)
     53             if pick: todo.append(pick)
     54     return seen
     55 
     56 CAN_SYMLINK = True   # set in main(); False on e.g. VirtualBox shared folders
     57 LINKS = []           # symlinks to materialise as copies when CAN_SYMLINK is False
     58 
     59 def probe_symlinks(root):
     60     t = os.path.join(root, ".symtest")
     61     try:
     62         if os.path.lexists(t): os.unlink(t)
     63         os.symlink("x", t); os.unlink(t); return True
     64     except OSError:
     65         return False
     66 
     67 def materialize(root):
     68     """no symlink support: replace each link by a real copy of its target"""
     69     import shutil
     70     pending = list(LINKS)
     71     while pending:
     72         left = []
     73         for path, tgt, hard in pending:
     74             dst = os.path.join(root, path)
     75             if hard: src = os.path.join(root, tgt)
     76             elif tgt.startswith("/"): src = root + tgt
     77             else: src = os.path.normpath(os.path.join(os.path.dirname(dst), tgt))
     78             if not os.path.exists(src): left.append((path, tgt, hard)); continue
     79             os.makedirs(os.path.dirname(dst), exist_ok=True)
     80             if os.path.isdir(src) and (os.path.abspath(dst) + os.sep).startswith(os.path.abspath(src) + os.sep):
     81                 print("  skip self-referencing dir link", path, "->", tgt); continue   # e.g. x -> . / .. (would recurse forever)
     82             if os.path.isdir(src): shutil.copytree(src, dst, dirs_exist_ok=True) if sys.version_info >= (3, 8) else _cp(src, dst)
     83             else: shutil.copy2(src, dst)
     84         if len(left) == len(pending): break
     85         pending = left
     86     print(len(LINKS) - len(pending), "links copied;", len(pending), "dangling skipped")
     87 
     88 def _cp(src, dst):
     89     import shutil
     90     os.makedirs(dst, exist_ok=True)
     91     for n in os.listdir(src):
     92         a, b = os.path.join(src, n), os.path.join(dst, n)
     93         _cp(a, b) if os.path.isdir(a) else shutil.copy2(a, b)
     94 
     95 def deb_extract(fn, root):
     96     """pure-python 'dpkg-deb -x': an .deb is an ar archive holding data.tar.*"""
     97     b = open(fn, "rb").read()
     98     assert b[:8] == b"!<arch>\n"
     99     i = 8
    100     while i < len(b):
    101         name = b[i:i+16].decode().strip().rstrip("/")
    102         size = int(b[i+48:i+58]); i += 60
    103         if name.startswith("data.tar"):
    104             data = b[i:i+size]
    105             if name.endswith(".xz"): data = lzma.decompress(data)
    106             elif name.endswith(".gz"): data = gzip.decompress(data)
    107             elif not name.endswith(".tar"): raise SystemExit("unsupported " + name)
    108             with tarfile.open(fileobj=io.BytesIO(data)) as t:
    109                 for m in t:
    110                     if (m.issym() or m.islnk()) and not CAN_SYMLINK:
    111                         LINKS.append((m.name.lstrip("./"), m.linkname, m.islnk()))
    112                         continue
    113                     # 'tar' filter: still blocks escaping root, but keeps absolute symlinks
    114                     # (fixlinks() rewrites them to relative later); py3.14's default 'data'
    115                     # filter raises AbsoluteLinkError on them.
    116                     if hasattr(tarfile, "tar_filter"): t.extract(m, root, filter="tar")
    117                     else: t.extract(m, root)
    118             return
    119         i += size + (size & 1)
    120     raise SystemExit("no data.tar in " + fn)
    121 
    122 def sha_ok(fn, want):
    123     import hashlib
    124     if not want: return True
    125     h = hashlib.sha256()
    126     with open(fn, "rb") as f:
    127         for blk in iter(lambda: f.read(1 << 20), b""): h.update(blk)
    128     return h.hexdigest() == want
    129 
    130 def fetch_unpack(pk, names, root, cache):
    131     for n in names:
    132         fn = os.path.join(cache, os.path.basename(pk[n]["Filename"]))
    133         want = pk[n].get("SHA256")
    134         if not (os.path.exists(fn) and sha_ok(fn, want)):
    135             print("  get", n, flush=True)
    136             tmp = fn + ".part%d" % os.getpid()      # never leave a half-written file under the final name
    137             urllib.request.urlretrieve(f"{MIRROR}/{pk[n]['Filename']}", tmp)
    138             if not sha_ok(tmp, want): raise SystemExit("checksum mismatch for " + n)
    139             os.replace(tmp, fn)
    140         os.makedirs(root, exist_ok=True); deb_extract(fn, root)
    141 
    142 def fixlinks(root):
    143     """absolute symlinks (e.g. /usr/lib/i386-linux-gnu/x -> /lib/...) -> relative"""
    144     for d, _, fs in os.walk(root):
    145         for f in fs:
    146             p = os.path.join(d, f)
    147             if os.path.islink(p):
    148                 t = os.readlink(p)
    149                 if t.startswith("/"):
    150                     new = os.path.relpath(root + t, d)
    151                     os.unlink(p); os.symlink(new, p)
    152 
    153 def main():
    154     root = os.path.abspath(sys.argv[1])
    155     cache = os.path.join(root, ".debs"); os.makedirs(cache, exist_ok=True)
    156     global CAN_SYMLINK
    157     CAN_SYMLINK = probe_symlinks(root)
    158     print("symlinks", "supported" if CAN_SYMLINK else "NOT supported -> copying files instead")
    159     pk, prov = load("i386", root)
    160     names = closure(pk, prov, SEEDS)
    161     print(len(names), "i386 packages")
    162     fetch_unpack(pk, names, root, cache)
    163     if CAN_SYMLINK: fixlinks(root)
    164     else: materialize(root); LINKS.clear()
    165     if not any(os.access(os.path.join(d, "nasm"), os.X_OK)
    166                for d in os.environ.get("PATH", "").split(":")):
    167         pk, prov = load("amd64", root)
    168         hroot = os.path.join(root, "host")
    169         fetch_unpack(pk, ["nasm"], hroot, cache)
    170         if not CAN_SYMLINK: materialize(hroot)
    171     import shutil; shutil.rmtree(cache, ignore_errors=True)   # keep the folder lean
    172     open(os.path.join(root, ".done"), "w").write("ok\n")
    173     print("sysroot ready:", root)
    174 
    175 if __name__ == "__main__":
    176     main()