327 lines
8.8 KiB
Bash
327 lines
8.8 KiB
Bash
#!/usr/bin/env bash
|
|
# build-debian-iso.sh — Build all Debian amd64 BD ISOs from jigdo files
|
|
# Source: https://cdimage.debian.org/debian-cd/current/amd64/jigdo-bd/
|
|
#
|
|
# Requirements:
|
|
# sudo apt install jigdo-file wget
|
|
|
|
set -euo pipefail
|
|
|
|
JIGDO_URL="https://cdimage.debian.org/debian-cd/current/amd64/jigdo-bd"
|
|
DEFAULT_MIRROR="https://deb.debian.org/debian"
|
|
OUTPUT_DIR="${OUTPUT_DIR:-./debian-isos}"
|
|
MIRROR="$DEFAULT_MIRROR"
|
|
SCAN_DIR=""
|
|
CACHE_NAME="jigdo-file-cache.db"
|
|
FETCH_BATCH_SIZE="${FETCH_BATCH_SIZE:-30}"
|
|
WGET_ARGS=(
|
|
--continue
|
|
--tries=5
|
|
--retry-connrefused
|
|
--waitretry=1
|
|
--timeout=30
|
|
)
|
|
|
|
die() { echo "ERROR: $*" >&2; exit 1; }
|
|
|
|
usage() {
|
|
cat <<EOF
|
|
Usage: $(basename "$0") [OPTIONS]
|
|
|
|
Build all Debian amd64 BD ISOs from jigdo files without using jigdo-lite.
|
|
|
|
Options:
|
|
-o DIR Output directory (default: ./debian-isos, or \$OUTPUT_DIR)
|
|
-m URL Debian mirror (default: https://deb.debian.org/debian)
|
|
-s DIR Scan DIR for locally cached .deb packages to reuse
|
|
-h Show this help
|
|
|
|
Environment:
|
|
FETCH_BATCH_SIZE Number of package URLs fetched per batch (default: 30)
|
|
|
|
Examples:
|
|
$(basename "$0")
|
|
$(basename "$0") -o /data/isos
|
|
$(basename "$0") -s /var/cache/apt/archives
|
|
$(basename "$0") -m http://ftp.de.debian.org/debian
|
|
EOF
|
|
}
|
|
|
|
check_deps() {
|
|
local missing=()
|
|
command -v jigdo-file &>/dev/null || missing+=("jigdo-file (sudo apt install jigdo-file)")
|
|
command -v wget &>/dev/null || missing+=("wget (sudo apt install wget)")
|
|
command -v gzip &>/dev/null || missing+=("gzip")
|
|
command -v grep &>/dev/null || missing+=("grep")
|
|
command -v sed &>/dev/null || missing+=("sed")
|
|
command -v awk &>/dev/null || missing+=("awk")
|
|
(( ${#missing[@]} == 0 )) || { printf 'Missing: %s\n' "${missing[@]}" >&2; exit 1; }
|
|
}
|
|
|
|
normalize_mirror() {
|
|
local url="$1"
|
|
printf '%s/\n' "${url%/}"
|
|
}
|
|
|
|
fetch_metadata() {
|
|
local base="$1"
|
|
local jigdo="${OUTPUT_DIR}/${base}.jigdo"
|
|
local jigdo_unpacked="${OUTPUT_DIR}/${base}.jigdo.unpacked"
|
|
local template="${OUTPUT_DIR}/${base}.template"
|
|
|
|
[[ -s "$jigdo" ]] || wget -qO "$jigdo" "${JIGDO_URL}/${base}.jigdo"
|
|
[[ -s "$template" ]] || wget -qO "$template" "${JIGDO_URL}/${base}.template"
|
|
|
|
if [[ ! -s "$jigdo_unpacked" || "$jigdo_unpacked" -ot "$jigdo" ]]; then
|
|
if gzip -cd "$jigdo" >"${jigdo_unpacked}.tmp" 2>/dev/null; then
|
|
mv "${jigdo_unpacked}.tmp" "$jigdo_unpacked"
|
|
else
|
|
cp "$jigdo" "$jigdo_unpacked"
|
|
fi
|
|
fi
|
|
}
|
|
|
|
run_make_image() {
|
|
local iso="$1"
|
|
local jigdo="$2"
|
|
local template="$3"
|
|
local input_path="$4"
|
|
local rc=0
|
|
|
|
jigdo-file make-image \
|
|
--image="$iso" \
|
|
--jigdo="$jigdo" \
|
|
--template="$template" \
|
|
--cache="${OUTPUT_DIR}/${CACHE_NAME}" \
|
|
"$input_path" || rc=$?
|
|
|
|
case "$rc" in
|
|
0|1) return "$rc" ;;
|
|
2) die "jigdo-file reported a recoverable error while processing '$input_path'" ;;
|
|
*) die "jigdo-file failed while processing '$input_path' (exit $rc)" ;;
|
|
esac
|
|
}
|
|
|
|
write_missing_list() {
|
|
local iso="$1"
|
|
local jigdo="$2"
|
|
local jigdo_unpacked="$3"
|
|
local template="$4"
|
|
local list_file="$5"
|
|
local checksums_file="${list_file}.checksums"
|
|
local template_source="$template"
|
|
|
|
[[ -f "${iso}.tmp" ]] && template_source="${iso}.tmp"
|
|
|
|
jigdo-file list-template --template="$template_source" \
|
|
| awk '$1 ~ /^need-file-/ { print $4 }' >"$checksums_file"
|
|
|
|
awk -v mirror="$(normalize_mirror "$MIRROR")" '
|
|
function trim(s) {
|
|
sub(/^[[:space:]]+/, "", s)
|
|
sub(/[[:space:]]+$/, "", s)
|
|
return s
|
|
}
|
|
|
|
NR == FNR {
|
|
if ($1 != "") {
|
|
wanted[$1] = 1
|
|
wanted_count++
|
|
}
|
|
next
|
|
}
|
|
|
|
/^\[/ {
|
|
section = $0
|
|
next
|
|
}
|
|
|
|
section != "[Parts]" {
|
|
next
|
|
}
|
|
|
|
/^[[:space:]]*#/ {
|
|
next
|
|
}
|
|
|
|
{
|
|
pos = index($0, "=")
|
|
if (!pos) {
|
|
next
|
|
}
|
|
|
|
key = trim(substr($0, 1, pos - 1))
|
|
if (!(key in wanted)) {
|
|
next
|
|
}
|
|
|
|
value = trim(substr($0, pos + 1))
|
|
|
|
if (value ~ /^[A-Za-z][A-Za-z0-9+.-]*:\/\// || value ~ /^file:/) {
|
|
print value
|
|
found[key] = 1
|
|
found_count++
|
|
next
|
|
}
|
|
|
|
split(value, parts, ":")
|
|
label = parts[1]
|
|
path = substr(value, length(label) + 2)
|
|
|
|
if (label == "Debian" || label == "Non-US") {
|
|
print mirror path
|
|
found[key] = 1
|
|
found_count++
|
|
}
|
|
}
|
|
|
|
END {
|
|
if (wanted_count != found_count) {
|
|
for (key in wanted) {
|
|
if (!(key in found)) {
|
|
printf "Unresolved jigdo part checksum: %s\n", key > "/dev/stderr"
|
|
}
|
|
}
|
|
exit 2
|
|
}
|
|
}
|
|
' "$checksums_file" "$jigdo_unpacked" >"$list_file"
|
|
|
|
rm -f "$checksums_file"
|
|
}
|
|
|
|
download_batch() {
|
|
local list_file="$1"
|
|
local batch_dir="$2"
|
|
|
|
rm -rf "$batch_dir"
|
|
mkdir -p "$batch_dir"
|
|
|
|
wget "${WGET_ARGS[@]}" \
|
|
--input-file="$list_file" \
|
|
--force-directories \
|
|
--directory-prefix="$batch_dir"
|
|
}
|
|
|
|
build_image() {
|
|
local base="$1"
|
|
local iso="${OUTPUT_DIR}/${base}.iso"
|
|
local jigdo="${OUTPUT_DIR}/${base}.jigdo"
|
|
local jigdo_unpacked="${OUTPUT_DIR}/${base}.jigdo.unpacked"
|
|
local template="${OUTPUT_DIR}/${base}.template"
|
|
local batch_dir="${OUTPUT_DIR}/${base}.download"
|
|
local list_file="${OUTPUT_DIR}/${base}.missing"
|
|
local batch_file="${OUTPUT_DIR}/${base}.batch"
|
|
local fetched_any=0
|
|
|
|
echo ""
|
|
echo "--- $base ---"
|
|
|
|
if [[ -f "$iso" ]]; then
|
|
echo "Already exists, skipping. Delete to rebuild: rm '$iso'"
|
|
return 0
|
|
fi
|
|
|
|
fetch_metadata "$base"
|
|
|
|
if [[ -n "$SCAN_DIR" ]]; then
|
|
echo "Scanning local packages from: $SCAN_DIR"
|
|
if run_make_image "$iso" "$jigdo" "$template" "$SCAN_DIR"; then
|
|
:
|
|
fi
|
|
if [[ -f "$iso" ]]; then
|
|
echo "Done: $iso ($(du -sh "$iso" | cut -f1))"
|
|
return 0
|
|
fi
|
|
fi
|
|
|
|
while true; do
|
|
write_missing_list "$iso" "$jigdo" "$jigdo_unpacked" "$template" "$list_file"
|
|
mapfile -t urls < <(grep -v '^$' "$list_file")
|
|
|
|
if (( ${#urls[@]} == 0 )); then
|
|
[[ -f "$iso" ]] && break
|
|
die "No missing URLs were returned for $base, but the ISO is still incomplete"
|
|
fi
|
|
|
|
echo "Missing files: ${#urls[@]}"
|
|
|
|
local offset=0
|
|
while (( offset < ${#urls[@]} )); do
|
|
local count=$FETCH_BATCH_SIZE
|
|
(( offset + count > ${#urls[@]} )) && count=$(( ${#urls[@]} - offset ))
|
|
|
|
printf '%s\n' "${urls[@]:offset:count}" >"$batch_file"
|
|
|
|
echo "Downloading batch: $((offset + 1))-$((offset + count)) / ${#urls[@]}"
|
|
download_batch "$batch_file" "$batch_dir"
|
|
fetched_any=1
|
|
|
|
if run_make_image "$iso" "$jigdo" "$template" "$batch_dir"; then
|
|
:
|
|
fi
|
|
|
|
rm -rf "$batch_dir"
|
|
|
|
if [[ -f "$iso" ]]; then
|
|
break 2
|
|
fi
|
|
|
|
offset=$((offset + count))
|
|
done
|
|
|
|
if (( fetched_any == 0 )); then
|
|
die "No files were downloaded for $base"
|
|
fi
|
|
done
|
|
|
|
rm -f "$list_file" "$batch_file"
|
|
jigdo-file verify --image="$iso" --template="$template" --report=quiet >/dev/null
|
|
echo "Done: $iso ($(du -sh "$iso" | cut -f1))"
|
|
}
|
|
|
|
main() {
|
|
while [[ $# -gt 0 ]]; do
|
|
case "$1" in
|
|
-h|--help) usage; exit 0 ;;
|
|
-o|--output) OUTPUT_DIR="$2"; shift 2 ;;
|
|
-m|--mirror) MIRROR="$2"; shift 2 ;;
|
|
-s|--scan) SCAN_DIR="$2"; shift 2 ;;
|
|
*) die "Unknown option: $1" ;;
|
|
esac
|
|
done
|
|
|
|
check_deps
|
|
mkdir -p "$OUTPUT_DIR"
|
|
[[ -n "$SCAN_DIR" && ! -d "$SCAN_DIR" ]] && die "Scan directory not found: $SCAN_DIR"
|
|
|
|
local -a images=()
|
|
while IFS= read -r name; do
|
|
images+=("${name%.jigdo}")
|
|
done < <(
|
|
wget -qO- "${JIGDO_URL}/" \
|
|
| grep -oP 'href="debian-[^"]+BD-[0-9]+\.jigdo"' \
|
|
| grep -oP 'debian-[^"]+\.jigdo' \
|
|
| grep -v 'debian-edu' \
|
|
| sort -uV
|
|
)
|
|
|
|
(( ${#images[@]} > 0 )) || die "No BD jigdo files found at $JIGDO_URL"
|
|
|
|
echo "Mirror: $(normalize_mirror "$MIRROR")"
|
|
echo "Output: $OUTPUT_DIR"
|
|
[[ -n "$SCAN_DIR" ]] && echo "Scan : $SCAN_DIR"
|
|
echo "Images: ${images[*]}"
|
|
|
|
local failed=0
|
|
for base in "${images[@]}"; do
|
|
build_image "$base" || (( ++failed ))
|
|
done
|
|
|
|
echo ""
|
|
(( failed == 0 )) || die "$failed image(s) failed."
|
|
echo "All done. ISOs in: $OUTPUT_DIR"
|
|
}
|
|
|
|
main "$@"
|