#!/usr/bin/env bash
# splunk_uf_downloads.sh
# ------------------------------------------------------------
# Splunk Universal Forwarder bulk downloader (Linux Bash)
#
# Features:
# - Fetches the last 10 UF versions from Splunk "Previous Releases" page
# - Lets you pick a version (default = latest in that list)
# - Extracts direct download URLs for that version (no directory browsing)
#   (important: older versions often do NOT allow directory listing on download.splunk.com)
# - Derives available OS folders from the URLs found
# - Defaults OS selection to: linux + windows
# - Optionally add additional OS folders that exist for that version, or choose from scratch / all
# - Downloads everything into: ./splunk_uf_downloads/<version>/<os>/...
#
# Requirements:
# - bash, curl
# - python3 (used for robust HTML unescape + parsing). If you truly cannot use python3,
#   tell me and I’ll give a pure bash/sed/grep version.
# ------------------------------------------------------------

set -euo pipefail

UF_PREV_PAGE="https://www.splunk.com/en_us/download/previous-releases-universal-forwarder.html"
UF_LATEST_PAGE="https://www.splunk.com/en_us/download/universal-forwarder.html"
BASE_DL="https://download.splunk.com/products/universalforwarder/releases"
OUT_ROOT="$(pwd)/splunk_uf_downloads"

UA="Mozilla/5.0"

tmpdir="$(mktemp -d)"
trap 'rm -rf "$tmpdir"' EXIT

prev_html="$tmpdir/prev.html"
latest_html="$tmpdir/latest.html"
versions_txt="$tmpdir/versions.txt"
urls_all_txt="$tmpdir/urls_all.txt"
os_all_txt="$tmpdir/os_all.txt"
os_selected_txt="$tmpdir/os_selected.txt"
urls_selected_txt="$tmpdir/urls_selected.txt"

echo "Fetching Previous Releases page..."
curl -fsSL -A "$UA" "$UF_PREV_PAGE" -o "$prev_html"

# Extract versions + last10 (semantic sort)
python3 - <<'PY' "$prev_html" "$BASE_DL" > "$versions_txt"
import re, sys, html
path, base = sys.argv[1], sys.argv[2]
s = html.unescape(open(path, "r", encoding="utf-8", errors="ignore").read())
pat = re.escape(base) + r"/(\d+\.\d+\.\d+)/"
vers = sorted(set(re.findall(pat, s)), key=lambda v: tuple(map(int, v.split("."))))
# print last 10 (or fewer)
for v in vers[-10:]:
    print(v)
PY

if [[ ! -s "$versions_txt" ]]; then
  echo "ERROR: Could not extract versions from Previous Releases page." >&2
  exit 1
fi

mapfile -t VERSIONS < "$versions_txt"
LATEST="${VERSIONS[-1]}"

echo
echo "Available UF versions (last ${#VERSIONS[@]}, oldest -> newest):"
i=1
for v in "${VERSIONS[@]}"; do
  printf " %2d. %s\n" "$i" "$v"
  ((i++))
done
echo

read -r -p "Select a version by number (default latest = ${LATEST}): " pick
CHOSEN="$LATEST"
if [[ -n "${pick// }" ]]; then
  if [[ "$pick" =~ ^[0-9]+$ ]] && (( pick >= 1 && pick <= ${#VERSIONS[@]} )); then
    CHOSEN="${VERSIONS[$((pick-1))]}"
  else
    echo "Invalid selection. Defaulting to latest: $LATEST"
    CHOSEN="$LATEST"
  fi
fi

echo
echo "Selected UF version: $CHOSEN"
echo

# Extract direct download URLs for the chosen version from prev page.
# If not found (rare), try latest UF page as fallback.
python3 - <<'PY' "$prev_html" "$BASE_DL" "$CHOSEN" > "$urls_all_txt" || true
import re, sys, html
path, base, ver = sys.argv[1], sys.argv[2], sys.argv[3]
s = html.unescape(open(path, "r", encoding="utf-8", errors="ignore").read())
# capture direct file URLs: base/ver/os/anything_not_space_quote
pat = re.escape(base) + r"/" + re.escape(ver) + r"/[^/\"'<>\s]+/[^\"'<>\s]+"
urls = sorted(set(re.findall(pat, s)))
for u in urls:
    print(u)
PY

if [[ ! -s "$urls_all_txt" ]]; then
  echo "No URLs for $CHOSEN found on Previous Releases page; trying latest UF page as fallback..."
  curl -fsSL -A "$UA" "$UF_LATEST_PAGE" -o "$latest_html"
  python3 - <<'PY' "$latest_html" "$BASE_DL" "$CHOSEN" > "$urls_all_txt"
import re, sys, html
path, base, ver = sys.argv[1], sys.argv[2], sys.argv[3]
s = html.unescape(open(path, "r", encoding="utf-8", errors="ignore").read())
pat = re.escape(base) + r"/" + re.escape(ver) + r"/[^/\"'<>\s]+/[^\"'<>\s]+"
urls = sorted(set(re.findall(pat, s)))
for u in urls:
    print(u)
PY
fi

if [[ ! -s "$urls_all_txt" ]]; then
  echo "ERROR: No direct download URLs were found for UF $CHOSEN." >&2
  echo "Splunk may have removed/hid links for that version on public pages." >&2
  exit 1
fi

# Derive OS folders from URLs
python3 - <<'PY' "$urls_all_txt" "$BASE_DL" "$CHOSEN" > "$os_all_txt"
import sys
path, base, ver = sys.argv[1], sys.argv[2], sys.argv[3]
prefix = base.rstrip("/") + "/" + ver + "/"
os_set = set()
with open(path, "r", encoding="utf-8", errors="ignore") as f:
    for line in f:
        u = line.strip()
        if not u.startswith(prefix):
            continue
        rest = u[len(prefix):]
        parts = rest.split("/")
        if len(parts) >= 2 and parts[0]:
            os_set.add(parts[0].lower())
for osn in sorted(os_set):
    print(osn)
PY

echo "OS folders found for UF $CHOSEN:"
while IFS= read -r os; do
  [[ -n "$os" ]] && echo "  - $os"
done < "$os_all_txt"
echo

# ------------------------------------------------------------
# OS selection (defaults: linux + windows if present)
# ------------------------------------------------------------
# Build defaults
declare -a DEFAULTS=()
if grep -qx "linux" "$os_all_txt"; then DEFAULTS+=("linux"); fi
if grep -qx "windows" "$os_all_txt"; then DEFAULTS+=("windows"); fi

if (( ${#DEFAULTS[@]} == 0 )); then
  echo "NOTE: linux/windows not present for this version; defaulting to ALL OS folders found."
  cp "$os_all_txt" "$os_selected_txt"
else
  echo "Default selection:"
  for d in "${DEFAULTS[@]}"; do echo "  - $d (ALL)"; done
  echo
  read -r -p "Are these defaults OK? [Y/n]: " ok
  ok="${ok:-Y}"

  : > "$os_selected_txt"
  if [[ "$ok" =~ ^([nN]|no|NO)$ ]]; then
    echo
    echo "Select which OS folders to download."
    echo "Type a comma-separated list (example: linux,windows,osx) or type ALL"
    echo "Available: $(paste -sd',' "$os_all_txt")"
    echo
    read -r -p "Your choice: " choice
    choice="${choice:-}"
    choice_norm="$(echo "$choice" | tr '[:upper:]' '[:lower:]' | tr -d '[:space:]')"

    if [[ -z "$choice_norm" ]]; then
      echo "ERROR: No selection provided." >&2
      exit 1
    fi

    if [[ "$choice_norm" == "all" ]]; then
      cp "$os_all_txt" "$os_selected_txt"
    else
      # Split comma list into lines
      python3 - <<'PY' "$choice_norm" > "$os_selected_txt"
import sys
c = sys.argv[1]
parts = [p.strip() for p in c.split(",") if p.strip()]
seen = set()
for p in parts:
    if p not in seen:
        print(p)
        seen.add(p)
PY
    fi
  else
    # accept defaults + optional extras
    for d in "${DEFAULTS[@]}"; do echo "$d" >> "$os_selected_txt"; done

    extras="$(grep -v -E '^(linux|windows)$' "$os_all_txt" || true)"
    if [[ -n "$extras" ]]; then
      echo
      echo "Optional additional OS folders detected:"
      echo "$extras" | sed 's/^/  - /'
      echo
      read -r -p "Add any of these? (comma-separated like osx,aix) or press Enter for none: " add
      add="${add:-}"
      add_norm="$(echo "$add" | tr '[:upper:]' '[:lower:]' | tr -d '[:space:]')"
      if [[ -n "$add_norm" ]]; then
        python3 - <<'PY' "$add_norm" >> "$os_selected_txt"
import sys
c = sys.argv[1]
parts = [p.strip() for p in c.split(",") if p.strip()]
seen = set()
for p in parts:
    if p not in seen:
        print(p)
        seen.add(p)
PY
      fi
    fi
  fi

  # Dedup + keep only OS that exist for this version
  python3 - <<'PY' "$os_selected_txt" "$os_all_txt" > "$tmpdir/os_selected_clean.txt"
import sys
sel_path, avail_path = sys.argv[1], sys.argv[2]
avail = set([l.strip().lower() for l in open(avail_path, "r", encoding="utf-8", errors="ignore") if l.strip()])
seen = set()
out = []
for l in open(sel_path, "r", encoding="utf-8", errors="ignore"):
    v = l.strip().lower()
    if not v or v in seen:
        continue
    seen.add(v)
    if v in avail:
        out.append(v)
for v in out:
    print(v)
PY
  mv "$tmpdir/os_selected_clean.txt" "$os_selected_txt"

  if [[ ! -s "$os_selected_txt" ]]; then
    echo "ERROR: None of your selected OS folders exist for UF $CHOSEN." >&2
    echo "Available: $(paste -sd',' "$os_all_txt")" >&2
    exit 1
  fi
fi

echo
echo "Final OS selection:"
cat "$os_selected_txt" | sed 's/^/  - /'
echo

# Filter URLs to selected OS folders
python3 - <<'PY' "$urls_all_txt" "$BASE_DL" "$CHOSEN" "$os_selected_txt" > "$urls_selected_txt"
import sys
urls_path, base, ver, sel_path = sys.argv[1], sys.argv[2], sys.argv[3], sys.argv[4]
prefix = base.rstrip("/") + "/" + ver + "/"
sel = set([l.strip().lower() for l in open(sel_path, "r", encoding="utf-8", errors="ignore") if l.strip()])
out = []
for u in open(urls_path, "r", encoding="utf-8", errors="ignore"):
    u = u.strip()
    if not u.startswith(prefix):
        continue
    rest = u[len(prefix):]
    parts = rest.split("/")
    if len(parts) < 2:
        continue
    osn = parts[0].lower()
    if osn in sel:
        out.append(u)
# unique preserve order
seen = set()
for u in out:
    if u not in seen:
        print(u)
        seen.add(u)
PY

COUNT="$(wc -l < "$urls_selected_txt" | tr -d ' ')"
OUTDIR="${OUT_ROOT}/${CHOSEN}"
mkdir -p "$OUTDIR"

echo "Will download ${COUNT} files for UF ${CHOSEN}."
echo "Output directory: ${OUTDIR}"
echo

# Download preserving directory structure after /releases/<ver>/
FAILS=0
while IFS= read -r url; do
  [[ -z "$url" ]] && continue

  rel="${url#${BASE_DL}/${CHOSEN}/}"          # os/path/file
  dest="${OUTDIR}/${rel}"
  mkdir -p "$(dirname "$dest")"

  if [[ -f "$dest" ]]; then
    echo "SKIP (exists): $rel"
    continue
  fi

  echo "DOWNLOADING: $rel"
  if ! curl -fL --retry 5 --retry-delay 2 -A "$UA" -o "$dest" "$url"; then
    echo "FAILED: $url" >&2
    FAILS=$((FAILS+1))
  fi
done < "$urls_selected_txt"

cp -f "$urls_selected_txt" "${OUTDIR}/_download_urls_selected.txt" 2>/dev/null || true

echo
echo "Done. Version: ${CHOSEN}"
echo "Output: ${OUTDIR}"
if (( FAILS > 0 )); then
  echo "WARNING: ${FAILS} downloads failed. Re-run the script to retry." >&2
fi