#!/usr/bin/env bash
# Run the per-country feeders for every configured country.
#
# The two Sifa feeders already take --country, so widening to Europe needs a loop, not a rewrite.
# The two expensive feeders are deliberately NOT in here: the Jetstream collector and the PLC
# export scan are single global passes that bucket every country at once, so running them per
# country would multiply their cost for no gain. Run those separately, once.
#
# Countries come from scripts/country-tlds.json (the same map the handle scan uses) unless
# COUNTRIES is set, so there is one list to maintain rather than two that drift.
#
# Usage:
#   bash scripts/collect-countries.sh                    # every country in the map
#   COUNTRIES="nl be de" bash scripts/collect-countries.sh
#   DIR=/path/to/nl-estimate bash scripts/collect-countries.sh
#
# Output per country, in $DIR:
#   sifa-<cc>.dids          accounts with a Sifa location in that country
#   sifa-company-<cc>.dids  company accounts for that country
#
# Failures never abort the run: one country's API hiccup must not cost the other 44. Failed
# countries are retried once at the end -- with 45 countries x 2 endpoints, the usual cause is
# Sifa's 60/min rate limit rather than anything being wrong, and the same country succeeds when
# run on its own. PAUSE spaces the countries out to stay under that limit in the first place.
set -uo pipefail

HERE="$(cd "$(dirname "$0")" && pwd)"
DIR="${DIR:-$HERE}"
SLEEP="${SLEEP:-0.3}"
PAUSE="${PAUSE:-1.5}"   # seconds between countries; Sifa rate-limits at 60/min
MAP="${MAP:-$HERE/country-tlds.json}"

# Country codes Sifa's API rejects. Empty since singi-labs/sifa-api#1280 landed, which added
# fo (Faroe Islands) and gl (Greenland) -- autonomous Danish territories with their own ISO
# 3166-1 codes that the validation list had omitted. Kept as a lever for the next such gap.
SIFA_UNSUPPORTED="${SIFA_UNSUPPORTED:-}"

if [ -z "${COUNTRIES:-}" ]; then
  COUNTRIES="$(python3 -c "
import json,sys
print(' '.join(sorted(set(json.load(open(sys.argv[1]))['tlds'].values()))))
" "$MAP")"
fi

echo "[collect] $(date -u +%FT%TZ) countries: $COUNTRIES"
failed=""
ok=0

run_country() {
  local cc="$1"
  case " $SIFA_UNSUPPORTED " in
    *" $cc "*) echo "[collect] $cc skipped (Sifa API does not accept this country code)"; return 0 ;;
  esac
  CC_UPPER="$(printf '%s' "$cc" | tr '[:lower:]' '[:upper:]')"

  # Sifa location. --country takes the uppercase alpha-2 the API expects.
  if python3 "$HERE/nl-sifa-location.py" --country "$CC_UPPER" \
      --out-prefix "$DIR/sifa-$cc" --sleep "$SLEEP" >/dev/null 2>&1; then
    n=$(wc -l < "$DIR/sifa-$cc.dids" 2>/dev/null || echo 0)
    [ "${n:-0}" -gt 0 ] && echo "[collect] $cc sifa-location: $n"
  else
    failed="$failed sifa-location:$cc"
  fi

  # Sifa companies.
  if python3 "$HERE/nl-sifa-company.py" --country "$CC_UPPER" \
      --out-prefix "$DIR/sifa-company-$cc" --dir "$DIR" >/dev/null 2>&1; then
    n=$(wc -l < "$DIR/sifa-company-$cc.dids" 2>/dev/null || echo 0)
    [ "${n:-0}" -gt 0 ] && echo "[collect] $cc sifa-company: $n"
    ok=$((ok + 1))
  else
    failed="$failed sifa-company:$cc"
    return 1
  fi
  return 0
}

retry=""
for cc in $COUNTRIES; do
  run_country "$cc" || retry="$retry $cc"
  sleep "$PAUSE"
done

# One retry pass. Rate-limit rejections are the common case and they clear on their own.
if [ -n "$retry" ]; then
  echo "[collect] retrying:$retry"
  failed=""
  for cc in $retry; do
    sleep 5
    run_country "$cc" || true
  done
fi

echo "[collect] done; $ok country-runs completed"
if [ -n "$failed" ]; then
  # Not fatal: these feeders are backfillable, so a partial run is worth keeping.
  echo "[collect] still failing after retry:$failed" >&2
fi