Add Debian ISO building script (incomplete)
This commit is contained in:
326
common/build/build-debian-iso.sh
Normal file
326
common/build/build-debian-iso.sh
Normal file
@@ -0,0 +1,326 @@
|
||||
#!/usr/bin/env bash
|
||||
# build-debian-iso.sh — Build all Debian amd64 BD ISOs from jigdo files
|
||||
# Source: https://cdimage.debian.org/debian-cd/current/amd64/jigdo-bd/
|
||||
#
|
||||
# Requirements:
|
||||
# sudo apt install jigdo-file wget
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
JIGDO_URL="https://cdimage.debian.org/debian-cd/current/amd64/jigdo-bd"
|
||||
DEFAULT_MIRROR="https://deb.debian.org/debian"
|
||||
OUTPUT_DIR="${OUTPUT_DIR:-./debian-isos}"
|
||||
MIRROR="$DEFAULT_MIRROR"
|
||||
SCAN_DIR=""
|
||||
CACHE_NAME="jigdo-file-cache.db"
|
||||
FETCH_BATCH_SIZE="${FETCH_BATCH_SIZE:-30}"
|
||||
WGET_ARGS=(
|
||||
--continue
|
||||
--tries=5
|
||||
--retry-connrefused
|
||||
--waitretry=1
|
||||
--timeout=30
|
||||
)
|
||||
|
||||
die() { echo "ERROR: $*" >&2; exit 1; }
|
||||
|
||||
usage() {
|
||||
cat <<EOF
|
||||
Usage: $(basename "$0") [OPTIONS]
|
||||
|
||||
Build all Debian amd64 BD ISOs from jigdo files without using jigdo-lite.
|
||||
|
||||
Options:
|
||||
-o DIR Output directory (default: ./debian-isos, or \$OUTPUT_DIR)
|
||||
-m URL Debian mirror (default: https://deb.debian.org/debian)
|
||||
-s DIR Scan DIR for locally cached .deb packages to reuse
|
||||
-h Show this help
|
||||
|
||||
Environment:
|
||||
FETCH_BATCH_SIZE Number of package URLs fetched per batch (default: 30)
|
||||
|
||||
Examples:
|
||||
$(basename "$0")
|
||||
$(basename "$0") -o /data/isos
|
||||
$(basename "$0") -s /var/cache/apt/archives
|
||||
$(basename "$0") -m http://ftp.de.debian.org/debian
|
||||
EOF
|
||||
}
|
||||
|
||||
check_deps() {
|
||||
local missing=()
|
||||
command -v jigdo-file &>/dev/null || missing+=("jigdo-file (sudo apt install jigdo-file)")
|
||||
command -v wget &>/dev/null || missing+=("wget (sudo apt install wget)")
|
||||
command -v gzip &>/dev/null || missing+=("gzip")
|
||||
command -v grep &>/dev/null || missing+=("grep")
|
||||
command -v sed &>/dev/null || missing+=("sed")
|
||||
command -v awk &>/dev/null || missing+=("awk")
|
||||
(( ${#missing[@]} == 0 )) || { printf 'Missing: %s\n' "${missing[@]}" >&2; exit 1; }
|
||||
}
|
||||
|
||||
normalize_mirror() {
|
||||
local url="$1"
|
||||
printf '%s/\n' "${url%/}"
|
||||
}
|
||||
|
||||
fetch_metadata() {
|
||||
local base="$1"
|
||||
local jigdo="${OUTPUT_DIR}/${base}.jigdo"
|
||||
local jigdo_unpacked="${OUTPUT_DIR}/${base}.jigdo.unpacked"
|
||||
local template="${OUTPUT_DIR}/${base}.template"
|
||||
|
||||
[[ -s "$jigdo" ]] || wget -qO "$jigdo" "${JIGDO_URL}/${base}.jigdo"
|
||||
[[ -s "$template" ]] || wget -qO "$template" "${JIGDO_URL}/${base}.template"
|
||||
|
||||
if [[ ! -s "$jigdo_unpacked" || "$jigdo_unpacked" -ot "$jigdo" ]]; then
|
||||
if gzip -cd "$jigdo" >"${jigdo_unpacked}.tmp" 2>/dev/null; then
|
||||
mv "${jigdo_unpacked}.tmp" "$jigdo_unpacked"
|
||||
else
|
||||
cp "$jigdo" "$jigdo_unpacked"
|
||||
fi
|
||||
fi
|
||||
}
|
||||
|
||||
run_make_image() {
|
||||
local iso="$1"
|
||||
local jigdo="$2"
|
||||
local template="$3"
|
||||
local input_path="$4"
|
||||
local rc=0
|
||||
|
||||
jigdo-file make-image \
|
||||
--image="$iso" \
|
||||
--jigdo="$jigdo" \
|
||||
--template="$template" \
|
||||
--cache="${OUTPUT_DIR}/${CACHE_NAME}" \
|
||||
"$input_path" || rc=$?
|
||||
|
||||
case "$rc" in
|
||||
0|1) return "$rc" ;;
|
||||
2) die "jigdo-file reported a recoverable error while processing '$input_path'" ;;
|
||||
*) die "jigdo-file failed while processing '$input_path' (exit $rc)" ;;
|
||||
esac
|
||||
}
|
||||
|
||||
write_missing_list() {
|
||||
local iso="$1"
|
||||
local jigdo="$2"
|
||||
local jigdo_unpacked="$3"
|
||||
local template="$4"
|
||||
local list_file="$5"
|
||||
local checksums_file="${list_file}.checksums"
|
||||
local template_source="$template"
|
||||
|
||||
[[ -f "${iso}.tmp" ]] && template_source="${iso}.tmp"
|
||||
|
||||
jigdo-file list-template --template="$template_source" \
|
||||
| awk '$1 ~ /^need-file-/ { print $4 }' >"$checksums_file"
|
||||
|
||||
awk -v mirror="$(normalize_mirror "$MIRROR")" '
|
||||
function trim(s) {
|
||||
sub(/^[[:space:]]+/, "", s)
|
||||
sub(/[[:space:]]+$/, "", s)
|
||||
return s
|
||||
}
|
||||
|
||||
NR == FNR {
|
||||
if ($1 != "") {
|
||||
wanted[$1] = 1
|
||||
wanted_count++
|
||||
}
|
||||
next
|
||||
}
|
||||
|
||||
/^\[/ {
|
||||
section = $0
|
||||
next
|
||||
}
|
||||
|
||||
section != "[Parts]" {
|
||||
next
|
||||
}
|
||||
|
||||
/^[[:space:]]*#/ {
|
||||
next
|
||||
}
|
||||
|
||||
{
|
||||
pos = index($0, "=")
|
||||
if (!pos) {
|
||||
next
|
||||
}
|
||||
|
||||
key = trim(substr($0, 1, pos - 1))
|
||||
if (!(key in wanted)) {
|
||||
next
|
||||
}
|
||||
|
||||
value = trim(substr($0, pos + 1))
|
||||
|
||||
if (value ~ /^[A-Za-z][A-Za-z0-9+.-]*:\/\// || value ~ /^file:/) {
|
||||
print value
|
||||
found[key] = 1
|
||||
found_count++
|
||||
next
|
||||
}
|
||||
|
||||
split(value, parts, ":")
|
||||
label = parts[1]
|
||||
path = substr(value, length(label) + 2)
|
||||
|
||||
if (label == "Debian" || label == "Non-US") {
|
||||
print mirror path
|
||||
found[key] = 1
|
||||
found_count++
|
||||
}
|
||||
}
|
||||
|
||||
END {
|
||||
if (wanted_count != found_count) {
|
||||
for (key in wanted) {
|
||||
if (!(key in found)) {
|
||||
printf "Unresolved jigdo part checksum: %s\n", key > "/dev/stderr"
|
||||
}
|
||||
}
|
||||
exit 2
|
||||
}
|
||||
}
|
||||
' "$checksums_file" "$jigdo_unpacked" >"$list_file"
|
||||
|
||||
rm -f "$checksums_file"
|
||||
}
|
||||
|
||||
download_batch() {
|
||||
local list_file="$1"
|
||||
local batch_dir="$2"
|
||||
|
||||
rm -rf "$batch_dir"
|
||||
mkdir -p "$batch_dir"
|
||||
|
||||
wget "${WGET_ARGS[@]}" \
|
||||
--input-file="$list_file" \
|
||||
--force-directories \
|
||||
--directory-prefix="$batch_dir"
|
||||
}
|
||||
|
||||
build_image() {
|
||||
local base="$1"
|
||||
local iso="${OUTPUT_DIR}/${base}.iso"
|
||||
local jigdo="${OUTPUT_DIR}/${base}.jigdo"
|
||||
local jigdo_unpacked="${OUTPUT_DIR}/${base}.jigdo.unpacked"
|
||||
local template="${OUTPUT_DIR}/${base}.template"
|
||||
local batch_dir="${OUTPUT_DIR}/${base}.download"
|
||||
local list_file="${OUTPUT_DIR}/${base}.missing"
|
||||
local batch_file="${OUTPUT_DIR}/${base}.batch"
|
||||
local fetched_any=0
|
||||
|
||||
echo ""
|
||||
echo "--- $base ---"
|
||||
|
||||
if [[ -f "$iso" ]]; then
|
||||
echo "Already exists, skipping. Delete to rebuild: rm '$iso'"
|
||||
return 0
|
||||
fi
|
||||
|
||||
fetch_metadata "$base"
|
||||
|
||||
if [[ -n "$SCAN_DIR" ]]; then
|
||||
echo "Scanning local packages from: $SCAN_DIR"
|
||||
if run_make_image "$iso" "$jigdo" "$template" "$SCAN_DIR"; then
|
||||
:
|
||||
fi
|
||||
if [[ -f "$iso" ]]; then
|
||||
echo "Done: $iso ($(du -sh "$iso" | cut -f1))"
|
||||
return 0
|
||||
fi
|
||||
fi
|
||||
|
||||
while true; do
|
||||
write_missing_list "$iso" "$jigdo" "$jigdo_unpacked" "$template" "$list_file"
|
||||
mapfile -t urls < <(grep -v '^$' "$list_file")
|
||||
|
||||
if (( ${#urls[@]} == 0 )); then
|
||||
[[ -f "$iso" ]] && break
|
||||
die "No missing URLs were returned for $base, but the ISO is still incomplete"
|
||||
fi
|
||||
|
||||
echo "Missing files: ${#urls[@]}"
|
||||
|
||||
local offset=0
|
||||
while (( offset < ${#urls[@]} )); do
|
||||
local count=$FETCH_BATCH_SIZE
|
||||
(( offset + count > ${#urls[@]} )) && count=$(( ${#urls[@]} - offset ))
|
||||
|
||||
printf '%s\n' "${urls[@]:offset:count}" >"$batch_file"
|
||||
|
||||
echo "Downloading batch: $((offset + 1))-$((offset + count)) / ${#urls[@]}"
|
||||
download_batch "$batch_file" "$batch_dir"
|
||||
fetched_any=1
|
||||
|
||||
if run_make_image "$iso" "$jigdo" "$template" "$batch_dir"; then
|
||||
:
|
||||
fi
|
||||
|
||||
rm -rf "$batch_dir"
|
||||
|
||||
if [[ -f "$iso" ]]; then
|
||||
break 2
|
||||
fi
|
||||
|
||||
offset=$((offset + count))
|
||||
done
|
||||
|
||||
if (( fetched_any == 0 )); then
|
||||
die "No files were downloaded for $base"
|
||||
fi
|
||||
done
|
||||
|
||||
rm -f "$list_file" "$batch_file"
|
||||
jigdo-file verify --image="$iso" --template="$template" --report=quiet >/dev/null
|
||||
echo "Done: $iso ($(du -sh "$iso" | cut -f1))"
|
||||
}
|
||||
|
||||
main() {
|
||||
while [[ $# -gt 0 ]]; do
|
||||
case "$1" in
|
||||
-h|--help) usage; exit 0 ;;
|
||||
-o|--output) OUTPUT_DIR="$2"; shift 2 ;;
|
||||
-m|--mirror) MIRROR="$2"; shift 2 ;;
|
||||
-s|--scan) SCAN_DIR="$2"; shift 2 ;;
|
||||
*) die "Unknown option: $1" ;;
|
||||
esac
|
||||
done
|
||||
|
||||
check_deps
|
||||
mkdir -p "$OUTPUT_DIR"
|
||||
[[ -n "$SCAN_DIR" && ! -d "$SCAN_DIR" ]] && die "Scan directory not found: $SCAN_DIR"
|
||||
|
||||
local -a images=()
|
||||
while IFS= read -r name; do
|
||||
images+=("${name%.jigdo}")
|
||||
done < <(
|
||||
wget -qO- "${JIGDO_URL}/" \
|
||||
| grep -oP 'href="debian-[^"]+BD-[0-9]+\.jigdo"' \
|
||||
| grep -oP 'debian-[^"]+\.jigdo' \
|
||||
| grep -v 'debian-edu' \
|
||||
| sort -uV
|
||||
)
|
||||
|
||||
(( ${#images[@]} > 0 )) || die "No BD jigdo files found at $JIGDO_URL"
|
||||
|
||||
echo "Mirror: $(normalize_mirror "$MIRROR")"
|
||||
echo "Output: $OUTPUT_DIR"
|
||||
[[ -n "$SCAN_DIR" ]] && echo "Scan : $SCAN_DIR"
|
||||
echo "Images: ${images[*]}"
|
||||
|
||||
local failed=0
|
||||
for base in "${images[@]}"; do
|
||||
build_image "$base" || (( ++failed ))
|
||||
done
|
||||
|
||||
echo ""
|
||||
(( failed == 0 )) || die "$failed image(s) failed."
|
||||
echo "All done. ISOs in: $OUTPUT_DIR"
|
||||
}
|
||||
|
||||
main "$@"
|
||||
Reference in New Issue
Block a user