mk32root.py (7702B)
1 #!/usr/bin/env python3 2 """Build a local 32-bit (i386) sysroot without root and without multilib. 3 4 Downloads the needed i386 .debs from the Debian archive, resolves their 5 dependencies, and unpacks them in pure Python (no dpkg/root/distro tools) 6 into a directory. Works on any Linux with python3. 7 Also fetches a native nasm if the host has none. 8 9 usage: mk32root.py SYSROOT_DIR 10 """ 11 import os, re, sys, io, lzma, gzip, tarfile, urllib.request 12 13 MIRROR = "http://archive.debian.org/debian" 14 SUITE = "buster" 15 SEEDS = ["libc6-dev", "libgcc-8-dev", "libglfw3-dev", "libglfw3", "libmikmod-dev", 16 "libgl1-mesa-dev", "libglu1-mesa-dev", "libstdc++-8-dev", "libx11-dev", "libxrandr-dev", "libxinerama-dev", "libxcursor-dev", "libxi-dev", "libxxf86vm-dev", "libxext-dev", "libgl1-mesa-dri", "libgl1", "libglx-mesa0"] 17 # deps we never need to pull in (not libraries, or provided by the host) 18 SKIP = {"debconf", "dpkg", "perl-base", "lsb-base", "multiarch-support"} 19 20 def load(arch, dest): 21 url = f"{MIRROR}/dists/{SUITE}/main/binary-{arch}/Packages.xz" 22 print("fetching index", url, flush=True) 23 raw = lzma.decompress(urllib.request.urlopen(url).read()).decode() 24 pk, prov = {}, {} 25 for blk in raw.split("\n\n"): 26 f = dict(re.findall(r"^([A-Za-z-]+): (.*)$", blk, re.M)) 27 if "Package" not in f: continue 28 pk[f["Package"]] = f 29 for p in f.get("Provides", "").split(","): 30 p = p.strip().split(" ")[0] 31 if p: prov.setdefault(p, f["Package"]) 32 return pk, prov 33 34 def deps(f): 35 out = [] 36 for fld in ("Depends", "Pre-Depends"): 37 for alt in f.get(fld, "").split(","): 38 alt = alt.strip() 39 if alt: out.append([re.sub(r"[:\s(].*", "", a.strip()) for a in alt.split("|")]) 40 return out 41 42 def closure(pk, prov, seeds): 43 seen, todo = [], list(seeds) 44 while todo: 45 n = todo.pop() 46 n = n if n in pk else prov.get(n) 47 if not n or n in seen or n in SKIP: continue 48 seen.append(n) 49 for alts in deps(pk[n]): 50 # take the first alternative that exists (prefer already-chosen) 51 pick = next((a for a in alts if a in seen), None) or \ 52 next((a for a in alts if a in pk or a in prov), None) 53 if pick: todo.append(pick) 54 return seen 55 56 CAN_SYMLINK = True # set in main(); False on e.g. VirtualBox shared folders 57 LINKS = [] # symlinks to materialise as copies when CAN_SYMLINK is False 58 59 def probe_symlinks(root): 60 t = os.path.join(root, ".symtest") 61 try: 62 if os.path.lexists(t): os.unlink(t) 63 os.symlink("x", t); os.unlink(t); return True 64 except OSError: 65 return False 66 67 def materialize(root): 68 """no symlink support: replace each link by a real copy of its target""" 69 import shutil 70 pending = list(LINKS) 71 while pending: 72 left = [] 73 for path, tgt, hard in pending: 74 dst = os.path.join(root, path) 75 if hard: src = os.path.join(root, tgt) 76 elif tgt.startswith("/"): src = root + tgt 77 else: src = os.path.normpath(os.path.join(os.path.dirname(dst), tgt)) 78 if not os.path.exists(src): left.append((path, tgt, hard)); continue 79 os.makedirs(os.path.dirname(dst), exist_ok=True) 80 if os.path.isdir(src) and (os.path.abspath(dst) + os.sep).startswith(os.path.abspath(src) + os.sep): 81 print(" skip self-referencing dir link", path, "->", tgt); continue # e.g. x -> . / .. (would recurse forever) 82 if os.path.isdir(src): shutil.copytree(src, dst, dirs_exist_ok=True) if sys.version_info >= (3, 8) else _cp(src, dst) 83 else: shutil.copy2(src, dst) 84 if len(left) == len(pending): break 85 pending = left 86 print(len(LINKS) - len(pending), "links copied;", len(pending), "dangling skipped") 87 88 def _cp(src, dst): 89 import shutil 90 os.makedirs(dst, exist_ok=True) 91 for n in os.listdir(src): 92 a, b = os.path.join(src, n), os.path.join(dst, n) 93 _cp(a, b) if os.path.isdir(a) else shutil.copy2(a, b) 94 95 def deb_extract(fn, root): 96 """pure-python 'dpkg-deb -x': an .deb is an ar archive holding data.tar.*""" 97 b = open(fn, "rb").read() 98 assert b[:8] == b"!<arch>\n" 99 i = 8 100 while i < len(b): 101 name = b[i:i+16].decode().strip().rstrip("/") 102 size = int(b[i+48:i+58]); i += 60 103 if name.startswith("data.tar"): 104 data = b[i:i+size] 105 if name.endswith(".xz"): data = lzma.decompress(data) 106 elif name.endswith(".gz"): data = gzip.decompress(data) 107 elif not name.endswith(".tar"): raise SystemExit("unsupported " + name) 108 with tarfile.open(fileobj=io.BytesIO(data)) as t: 109 for m in t: 110 if (m.issym() or m.islnk()) and not CAN_SYMLINK: 111 LINKS.append((m.name.lstrip("./"), m.linkname, m.islnk())) 112 continue 113 # 'tar' filter: still blocks escaping root, but keeps absolute symlinks 114 # (fixlinks() rewrites them to relative later); py3.14's default 'data' 115 # filter raises AbsoluteLinkError on them. 116 if hasattr(tarfile, "tar_filter"): t.extract(m, root, filter="tar") 117 else: t.extract(m, root) 118 return 119 i += size + (size & 1) 120 raise SystemExit("no data.tar in " + fn) 121 122 def sha_ok(fn, want): 123 import hashlib 124 if not want: return True 125 h = hashlib.sha256() 126 with open(fn, "rb") as f: 127 for blk in iter(lambda: f.read(1 << 20), b""): h.update(blk) 128 return h.hexdigest() == want 129 130 def fetch_unpack(pk, names, root, cache): 131 for n in names: 132 fn = os.path.join(cache, os.path.basename(pk[n]["Filename"])) 133 want = pk[n].get("SHA256") 134 if not (os.path.exists(fn) and sha_ok(fn, want)): 135 print(" get", n, flush=True) 136 tmp = fn + ".part%d" % os.getpid() # never leave a half-written file under the final name 137 urllib.request.urlretrieve(f"{MIRROR}/{pk[n]['Filename']}", tmp) 138 if not sha_ok(tmp, want): raise SystemExit("checksum mismatch for " + n) 139 os.replace(tmp, fn) 140 os.makedirs(root, exist_ok=True); deb_extract(fn, root) 141 142 def fixlinks(root): 143 """absolute symlinks (e.g. /usr/lib/i386-linux-gnu/x -> /lib/...) -> relative""" 144 for d, _, fs in os.walk(root): 145 for f in fs: 146 p = os.path.join(d, f) 147 if os.path.islink(p): 148 t = os.readlink(p) 149 if t.startswith("/"): 150 new = os.path.relpath(root + t, d) 151 os.unlink(p); os.symlink(new, p) 152 153 def main(): 154 root = os.path.abspath(sys.argv[1]) 155 cache = os.path.join(root, ".debs"); os.makedirs(cache, exist_ok=True) 156 global CAN_SYMLINK 157 CAN_SYMLINK = probe_symlinks(root) 158 print("symlinks", "supported" if CAN_SYMLINK else "NOT supported -> copying files instead") 159 pk, prov = load("i386", root) 160 names = closure(pk, prov, SEEDS) 161 print(len(names), "i386 packages") 162 fetch_unpack(pk, names, root, cache) 163 if CAN_SYMLINK: fixlinks(root) 164 else: materialize(root); LINKS.clear() 165 if not any(os.access(os.path.join(d, "nasm"), os.X_OK) 166 for d in os.environ.get("PATH", "").split(":")): 167 pk, prov = load("amd64", root) 168 hroot = os.path.join(root, "host") 169 fetch_unpack(pk, ["nasm"], hroot, cache) 170 if not CAN_SYMLINK: materialize(hroot) 171 import shutil; shutil.rmtree(cache, ignore_errors=True) # keep the folder lean 172 open(os.path.join(root, ".done"), "w").write("ok\n") 173 print("sysroot ready:", root) 174 175 if __name__ == "__main__": 176 main()