diff --git a/common/build/build-debian-iso.sh b/common/build/build-debian-iso.sh new file mode 100644 index 0000000..5565ff8 --- /dev/null +++ b/common/build/build-debian-iso.sh @@ -0,0 +1,326 @@ +#!/usr/bin/env bash +# build-debian-iso.sh — Build all Debian amd64 BD ISOs from jigdo files +# Source: https://cdimage.debian.org/debian-cd/current/amd64/jigdo-bd/ +# +# Requirements: +# sudo apt install jigdo-file wget + +set -euo pipefail + +JIGDO_URL="https://cdimage.debian.org/debian-cd/current/amd64/jigdo-bd" +DEFAULT_MIRROR="https://deb.debian.org/debian" +OUTPUT_DIR="${OUTPUT_DIR:-./debian-isos}" +MIRROR="$DEFAULT_MIRROR" +SCAN_DIR="" +CACHE_NAME="jigdo-file-cache.db" +FETCH_BATCH_SIZE="${FETCH_BATCH_SIZE:-30}" +WGET_ARGS=( + --continue + --tries=5 + --retry-connrefused + --waitretry=1 + --timeout=30 +) + +die() { echo "ERROR: $*" >&2; exit 1; } + +usage() { + cat </dev/null || missing+=("jigdo-file (sudo apt install jigdo-file)") + command -v wget &>/dev/null || missing+=("wget (sudo apt install wget)") + command -v gzip &>/dev/null || missing+=("gzip") + command -v grep &>/dev/null || missing+=("grep") + command -v sed &>/dev/null || missing+=("sed") + command -v awk &>/dev/null || missing+=("awk") + (( ${#missing[@]} == 0 )) || { printf 'Missing: %s\n' "${missing[@]}" >&2; exit 1; } +} + +normalize_mirror() { + local url="$1" + printf '%s/\n' "${url%/}" +} + +fetch_metadata() { + local base="$1" + local jigdo="${OUTPUT_DIR}/${base}.jigdo" + local jigdo_unpacked="${OUTPUT_DIR}/${base}.jigdo.unpacked" + local template="${OUTPUT_DIR}/${base}.template" + + [[ -s "$jigdo" ]] || wget -qO "$jigdo" "${JIGDO_URL}/${base}.jigdo" + [[ -s "$template" ]] || wget -qO "$template" "${JIGDO_URL}/${base}.template" + + if [[ ! -s "$jigdo_unpacked" || "$jigdo_unpacked" -ot "$jigdo" ]]; then + if gzip -cd "$jigdo" >"${jigdo_unpacked}.tmp" 2>/dev/null; then + mv "${jigdo_unpacked}.tmp" "$jigdo_unpacked" + else + cp "$jigdo" "$jigdo_unpacked" + fi + fi +} + +run_make_image() { + local iso="$1" + local jigdo="$2" + local template="$3" + local input_path="$4" + local rc=0 + + jigdo-file make-image \ + --image="$iso" \ + --jigdo="$jigdo" \ + --template="$template" \ + --cache="${OUTPUT_DIR}/${CACHE_NAME}" \ + "$input_path" || rc=$? + + case "$rc" in + 0|1) return "$rc" ;; + 2) die "jigdo-file reported a recoverable error while processing '$input_path'" ;; + *) die "jigdo-file failed while processing '$input_path' (exit $rc)" ;; + esac +} + +write_missing_list() { + local iso="$1" + local jigdo="$2" + local jigdo_unpacked="$3" + local template="$4" + local list_file="$5" + local checksums_file="${list_file}.checksums" + local template_source="$template" + + [[ -f "${iso}.tmp" ]] && template_source="${iso}.tmp" + + jigdo-file list-template --template="$template_source" \ + | awk '$1 ~ /^need-file-/ { print $4 }' >"$checksums_file" + + awk -v mirror="$(normalize_mirror "$MIRROR")" ' + function trim(s) { + sub(/^[[:space:]]+/, "", s) + sub(/[[:space:]]+$/, "", s) + return s + } + + NR == FNR { + if ($1 != "") { + wanted[$1] = 1 + wanted_count++ + } + next + } + + /^\[/ { + section = $0 + next + } + + section != "[Parts]" { + next + } + + /^[[:space:]]*#/ { + next + } + + { + pos = index($0, "=") + if (!pos) { + next + } + + key = trim(substr($0, 1, pos - 1)) + if (!(key in wanted)) { + next + } + + value = trim(substr($0, pos + 1)) + + if (value ~ /^[A-Za-z][A-Za-z0-9+.-]*:\/\// || value ~ /^file:/) { + print value + found[key] = 1 + found_count++ + next + } + + split(value, parts, ":") + label = parts[1] + path = substr(value, length(label) + 2) + + if (label == "Debian" || label == "Non-US") { + print mirror path + found[key] = 1 + found_count++ + } + } + + END { + if (wanted_count != found_count) { + for (key in wanted) { + if (!(key in found)) { + printf "Unresolved jigdo part checksum: %s\n", key > "/dev/stderr" + } + } + exit 2 + } + } + ' "$checksums_file" "$jigdo_unpacked" >"$list_file" + + rm -f "$checksums_file" +} + +download_batch() { + local list_file="$1" + local batch_dir="$2" + + rm -rf "$batch_dir" + mkdir -p "$batch_dir" + + wget "${WGET_ARGS[@]}" \ + --input-file="$list_file" \ + --force-directories \ + --directory-prefix="$batch_dir" +} + +build_image() { + local base="$1" + local iso="${OUTPUT_DIR}/${base}.iso" + local jigdo="${OUTPUT_DIR}/${base}.jigdo" + local jigdo_unpacked="${OUTPUT_DIR}/${base}.jigdo.unpacked" + local template="${OUTPUT_DIR}/${base}.template" + local batch_dir="${OUTPUT_DIR}/${base}.download" + local list_file="${OUTPUT_DIR}/${base}.missing" + local batch_file="${OUTPUT_DIR}/${base}.batch" + local fetched_any=0 + + echo "" + echo "--- $base ---" + + if [[ -f "$iso" ]]; then + echo "Already exists, skipping. Delete to rebuild: rm '$iso'" + return 0 + fi + + fetch_metadata "$base" + + if [[ -n "$SCAN_DIR" ]]; then + echo "Scanning local packages from: $SCAN_DIR" + if run_make_image "$iso" "$jigdo" "$template" "$SCAN_DIR"; then + : + fi + if [[ -f "$iso" ]]; then + echo "Done: $iso ($(du -sh "$iso" | cut -f1))" + return 0 + fi + fi + + while true; do + write_missing_list "$iso" "$jigdo" "$jigdo_unpacked" "$template" "$list_file" + mapfile -t urls < <(grep -v '^$' "$list_file") + + if (( ${#urls[@]} == 0 )); then + [[ -f "$iso" ]] && break + die "No missing URLs were returned for $base, but the ISO is still incomplete" + fi + + echo "Missing files: ${#urls[@]}" + + local offset=0 + while (( offset < ${#urls[@]} )); do + local count=$FETCH_BATCH_SIZE + (( offset + count > ${#urls[@]} )) && count=$(( ${#urls[@]} - offset )) + + printf '%s\n' "${urls[@]:offset:count}" >"$batch_file" + + echo "Downloading batch: $((offset + 1))-$((offset + count)) / ${#urls[@]}" + download_batch "$batch_file" "$batch_dir" + fetched_any=1 + + if run_make_image "$iso" "$jigdo" "$template" "$batch_dir"; then + : + fi + + rm -rf "$batch_dir" + + if [[ -f "$iso" ]]; then + break 2 + fi + + offset=$((offset + count)) + done + + if (( fetched_any == 0 )); then + die "No files were downloaded for $base" + fi + done + + rm -f "$list_file" "$batch_file" + jigdo-file verify --image="$iso" --template="$template" --report=quiet >/dev/null + echo "Done: $iso ($(du -sh "$iso" | cut -f1))" +} + +main() { + while [[ $# -gt 0 ]]; do + case "$1" in + -h|--help) usage; exit 0 ;; + -o|--output) OUTPUT_DIR="$2"; shift 2 ;; + -m|--mirror) MIRROR="$2"; shift 2 ;; + -s|--scan) SCAN_DIR="$2"; shift 2 ;; + *) die "Unknown option: $1" ;; + esac + done + + check_deps + mkdir -p "$OUTPUT_DIR" + [[ -n "$SCAN_DIR" && ! -d "$SCAN_DIR" ]] && die "Scan directory not found: $SCAN_DIR" + + local -a images=() + while IFS= read -r name; do + images+=("${name%.jigdo}") + done < <( + wget -qO- "${JIGDO_URL}/" \ + | grep -oP 'href="debian-[^"]+BD-[0-9]+\.jigdo"' \ + | grep -oP 'debian-[^"]+\.jigdo' \ + | grep -v 'debian-edu' \ + | sort -uV + ) + + (( ${#images[@]} > 0 )) || die "No BD jigdo files found at $JIGDO_URL" + + echo "Mirror: $(normalize_mirror "$MIRROR")" + echo "Output: $OUTPUT_DIR" + [[ -n "$SCAN_DIR" ]] && echo "Scan : $SCAN_DIR" + echo "Images: ${images[*]}" + + local failed=0 + for base in "${images[@]}"; do + build_image "$base" || (( ++failed )) + done + + echo "" + (( failed == 0 )) || die "$failed image(s) failed." + echo "All done. ISOs in: $OUTPUT_DIR" +} + +main "$@"