#!/usr/bin/env bash # Fetch the exact "corresponding source" of the GPL/LGPL programs that QuantaPrism Code sends to # browsers (Tiny Linux: kernel, busybox, tcc, cmatrix; v86's SeaBIOS/VGA BIOS) plus the MPL-2.0 # Pyodide source, and our own Tiny Linux build scripts. The sources are big (about 160 MB), so # they are kept OUT of git; the "source" site (golive/learnsites, NodePort 30104) serves them. # # CodeSite/tools/tinylinux/fetch-sources.sh [output folder] (default /disk2/work/tinylinux-source) # # For every Alpine package: the aports folder (APKBUILD, patches, kernel configs) at the exact # aports commit that built the binary package (the "c:" field of Alpine 3.22's x86 APKINDEX, # i.e. the commit inside the .apk), and every upstream file the APKBUILD names, checked against # the APKBUILD's sha512sums. Upstream first, Alpine's distfiles mirror as a fallback. # Downloads run one after another (be gentle with the servers); files already there and correct # are not downloaded again. After a Tiny Linux rebuild with newer package versions, update the # PKGS table below from the new vendor/tinylinux/packages.txt and the APKINDEX commits. set -euo pipefail HERE=$(cd "$(dirname "$0")" && pwd) OUT=${1:-/disk2/work/tinylinux-source} DISTFILES=https://distfiles.alpinelinux.org/distfiles/v3.22 APORTS=https://gitlab.alpinelinux.org/alpine/aports V86=https://raw.githubusercontent.com/copy/v86/cc37f35c43bf14865d74ad76879ecf9410f69eb1 mkdir -p "$OUT" TMP=$(mktemp -d) trap 'rm -rf "$TMP"' EXIT say() { printf '\n== %s\n' "$*"; } # fetch URL FILE: download to FILE (via FILE.part), a few retries, a short pause after each file fetch() { echo " get $1" curl -fL --retry 3 --retry-delay 10 --connect-timeout 30 -sS -o "$2.part" "$1" && mv "$2.part" "$2" local rc=$?; sleep 2; return $rc } # info DIR key value...: SOURCE-INFO.txt, read by make-source-index.py info() { local d=$1; shift; : > "$d/SOURCE-INFO.txt"; while [ $# -gt 1 ]; do printf '%s=%s\n' "$1" "$2" >> "$d/SOURCE-INFO.txt"; shift 2; done; } # Alpine package table: dir | aports path | aports commit | binary packages | licence | upstream URLs (file=url) PKGS=( "linux-lts-6.12.117-r0|main/linux-lts|e30fb966aa1d9cec17dac8283b37f4e779575cc2|linux-virt 6.12.117-r0 (bzImage) and linux-virt-dev (to build hello.ko)|GPL-2.0-only|linux-6.12.tar.xz=https://cdn.kernel.org/pub/linux/kernel/v6.x/linux-6.12.tar.xz patch-6.12.117.patch.xz=https://cdn.kernel.org/pub/linux/kernel/v6.x/patch-6.12.117.xz" "busybox-1.37.0-r20|main/busybox|2e97d754a30d558f15524b8e422303c8a96832df|busybox-static 1.37.0-r20 (/bin/busybox.static: the shell and commands)|GPL-2.0-only|busybox-1.37.0.tar.bz2=https://busybox.net/downloads/busybox-1.37.0.tar.bz2" "tcc-0.9.27_git20250106-r0|community/tcc|0f89d47b17bbafb8f5f9060fb08d1b5d05b6a715|tcc, tcc-libs, tcc-libs-static, tcc-dev 0.9.27_git20250106-r0 (the C compiler, libtcc1.a)|LGPL-2.1-or-later|tcc-f6385c05308f715bdd2c06336801193a21d69b50.tar.gz=https://repo.or.cz/tinycc.git/snapshot/f6385c05308f715bdd2c06336801193a21d69b50.tar.gz" "cmatrix-2.0-r2|community/cmatrix|dcf926d126ee921df71282780cae6c160fe30783|cmatrix 2.0-r2|GPL-3.0-or-later|cmatrix-2.0.tar.gz=https://github.com/abishekvashok/cmatrix/archive/v2.0.tar.gz" ) # Evidence of the aports commits: Alpine 3.22's x86 package index (kept from the first run; the # "c:" field of each package is the aports commit that built it) E=$OUT/alpine-3.22-x86-apkindex; mkdir -p "$E" for r in main community; do [ -f "$E/APKINDEX-$r.tar.gz" ] || fetch "https://dl-cdn.alpinelinux.org/alpine/v3.22/$r/x86/APKINDEX.tar.gz" "$E/APKINDEX-$r.tar.gz"; done for r in main community; do tar -xzOf "$E/APKINDEX-$r.tar.gz" APKINDEX | awk -v r=$r 'BEGIN{RS=""; FS="\n"} {p=v=c=o=""; for(i=1;i<=NF;i++){k=substr($i,1,2); x=substr($i,3); if(k=="P:")p=x; if(k=="V:")v=x; if(k=="c:")c=x; if(k=="o:")o=x} print r, p "-" v, "origin=" o, "aports-commit=" c}'; done \ | grep -E " (busybox-static|cmatrix|figlet|fortune|libbsd|libmd|libncursesw|musl|musl-dev|ncurses-terminfo-base|pkgconf|sl|tcc|tcc-dev|tcc-libs|tcc-libs-static|linux-virt|linux-virt-dev)-[0-9]" > "$E/our-packages.txt" info "$E" name alpine-apkindex version 3.22-x86 licence "-" binaries "-" recipe "https://dl-cdn.alpinelinux.org/alpine/v3.22/" upstream "" \ checked "our-packages.txt: version and aports commit of every package in Tiny Linux, from these indexes" # sha512 lines of an APKBUILD ("hash name"), single- or multi-line sha512sums="..." sums() { awk '/^sha512sums="/ { f=1; sub(/^sha512sums="/, "") } f { done = index($0, "\""); s = $0; gsub(/"/, "", s); if (split(s, a, " ") >= 2) print a[1], a[2]; if (done) f = 0 }' "$1"; } ok512() { (cd "$1" && echo "$2 $3" | sha512sum -c --quiet - >/dev/null 2>&1); } for row in "${PKGS[@]}"; do IFS='|' read -r dir path commit bins lic urls <<< "$row" D=$OUT/$dir; mkdir -p "$D" say "$dir ($path @ ${commit:0:12})" # 1. the aports folder at that commit, copied flat next to the upstream files (like abuild's startdir) if [ ! -f "$D/APKBUILD" ]; then X=$TMP/$dir; rm -rf "$X"; mkdir -p "$X" fetch "$APORTS/-/archive/$commit/aports-$commit.tar.gz?path=$path" "$X/aports.tar.gz" tar -xzf "$X/aports.tar.gz" -C "$X" --no-same-owner --no-same-permissions src=$(echo "$X"/aports-*/"$path") [ -f "$src/APKBUILD" ] || { echo "no APKBUILD in the aports archive" >&2; exit 1; } find "$src" -maxdepth 1 -type f -exec cp {} "$D/" \; fi # 2. every file in sha512sums: local files must match, upstream files are downloaded and checked n=0 while read -r hash name; do n=$((n + 1)) if [ -f "$D/$name" ] && ok512 "$D" "$hash" "$name"; then continue; fi url=""; for u in $urls; do [ "${u%%=*}" = "$name" ] && url=${u#*=}; done [ -n "$url" ] || { echo " $name: missing, and no upstream URL in the table" >&2; exit 1; } rm -f "$D/$name" if fetch "$url" "$D/$name" && ok512 "$D" "$hash" "$name"; then echo " $name: sha512 OK (upstream)"; continue; fi echo " $name: upstream copy missing or different, trying Alpine distfiles" rm -f "$D/$name" fetch "$DISTFILES/$name" "$D/$name" && ok512 "$D" "$hash" "$name" || { echo " $name: sha512 FAILED" >&2; exit 1; } echo " $name: sha512 OK (Alpine distfiles)" done < <(sums "$D/APKBUILD") (cd "$D" && sums APKBUILD | awk '{print $1 " " $2}' | sha512sum -c --quiet -) && echo " all $n files match the APKBUILD's sha512sums" info "$D" name "${dir%%-[0-9]*}" version "${dir#${dir%%-[0-9]*}-}" licence "$lic" binaries "$bins" \ recipe "$APORTS/-/tree/$commit/$path" upstream "$urls" checked "sha512 of every file matches the APKBUILD ($n files)" done # SeaBIOS + VGA BIOS as built by the v86 project (vendor/v86-0.5.473/seabios.bin, vgabios.bin): # v86's bios/fetch-and-build-seabios.sh builds SeaBIOS tag rel-1.16.2 with bios/seabios.config. D=$OUT/seabios-1.16.2; mkdir -p "$D"; say "seabios-1.16.2 (v86 bios)" [ -f "$D/seabios-1.16.2.tar.gz" ] || fetch https://www.seabios.org/downloads/seabios-1.16.2.tar.gz "$D/seabios-1.16.2.tar.gz" for f in fetch-and-build-seabios.sh seabios.config COPYING.LESSER; do [ -f "$D/v86-bios-$f" ] || fetch "$V86/bios/$f" "$D/v86-bios-$f" done tar -tzf "$D/seabios-1.16.2.tar.gz" >/dev/null info "$D" name seabios version 1.16.2 licence "LGPL-3.0-or-later" \ binaries "seabios.bin and vgabios.bin in vendor/v86-0.5.473 (built by the v86 project, commit cc37f35c43bf)" \ recipe "https://github.com/copy/v86/tree/cc37f35c43bf14865d74ad76879ecf9410f69eb1/bios" \ upstream "seabios-1.16.2.tar.gz=https://www.seabios.org/downloads/seabios-1.16.2.tar.gz" \ checked "the binaries in vendor/ equal v86's bios/*.bin at that commit (sha256); the build script and config are v86's" # Pyodide 0.29.5 (MPL-2.0): the source of pyodide.js / pyodide.asm.* at the release tag D=$OUT/pyodide-0.29.5; mkdir -p "$D"; say "pyodide-0.29.5" [ -f "$D/pyodide-0.29.5.tar.gz" ] || fetch https://github.com/pyodide/pyodide/archive/refs/tags/0.29.5.tar.gz "$D/pyodide-0.29.5.tar.gz" tar -tzf "$D/pyodide-0.29.5.tar.gz" >/dev/null info "$D" name pyodide version 0.29.5 licence "MPL-2.0 (CPython inside: PSF-2.0)" \ binaries "vendor/pyodide-0.29.5 (pyodide.js, pyodide.asm.js, pyodide.asm.wasm, python_stdlib.zip)" \ recipe "https://github.com/pyodide/pyodide/tree/0.29.5" \ upstream "pyodide-0.29.5.tar.gz=https://github.com/pyodide/pyodide/archive/refs/tags/0.29.5.tar.gz" \ checked "release tag archive from GitHub (Pyodide's own code and build recipes)" # Our own part: the scripts that control compilation and installation, hello.ko's source, # the files we add to the root file system, and the exact package list of the shipped build. REV=$(git -C "$HERE" log -1 --format=%h -- . ../../vendor/tinylinux 2>/dev/null || echo local) D=$OUT/quantaprism-tinylinux-$REV; say "quantaprism-tinylinux-$REV" rm -rf "$D" && mkdir -p "$D" cp -r "$HERE/build.sh" "$HERE/build-in-container.sh" "$HERE/fetch-sources.sh" "$HERE/make-source-index.py" "$HERE/hello" "$HERE/rootfs" "$D/" cp "$HERE/../../vendor/tinylinux/packages.txt" "$HERE/../../vendor/tinylinux/kernel-version.txt" "$D/" (cd "$HERE/../../vendor/tinylinux" && sha256sum bzImage initrd.gz) > "$D/shipped-files.sha256" tar -C "$OUT" -czf "$TMP/q.tgz" "quantaprism-tinylinux-$REV" && mv "$TMP/q.tgz" "$D/quantaprism-tinylinux-$REV.tar.gz" info "$D" name quantaprism-tinylinux version "$REV" licence "GPL-2.0-only (hello.c); our scripts: same terms" \ binaries "vendor/tinylinux/bzImage and initrd.gz (packages.txt lists every package inside)" \ recipe "CodeSite/tools/tinylinux in the QuantaPrism repository (git $REV)" upstream "" \ checked "shipped-files.sha256: sha256 of the bzImage and initrd.gz this source belongs to" for old in "$OUT"/quantaprism-tinylinux-*; do [ "$old" = "$D" ] || rm -rf "$old"; done # checksums, manifest and the index page say "SHA256SUMS, MANIFEST.md, index.html" for d in "$OUT"/*/; do [ -f "$d/SOURCE-INFO.txt" ] || continue (cd "$d" && find . -type f ! -name SHA256SUMS | sed 's|^\./||' | LC_ALL=C sort | xargs -d '\n' sha256sum > SHA256SUMS) done python3 -I "$HERE/make-source-index.py" "$OUT" (cd "$OUT" && find . -mindepth 2 -type f ! -path './_*' | sed 's|^\./||' | LC_ALL=C sort | xargs -d '\n' sha256sum > SHA256SUMS) du -sh "$OUT"