Something went wrong. Try again.
Central platform for European atproto.<cc> country community websites
Something went wrong. Try again.
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899#!/usr/bin/env bash# Run the per-country feeders for every configured country.## The two Sifa feeders already take --country, so widening to Europe needs a loop, not a rewrite.# The two expensive feeders are deliberately NOT in here: the Jetstream collector and the PLC# export scan are single global passes that bucket every country at once, so running them per# country would multiply their cost for no gain. Run those separately, once.## Countries come from scripts/country-tlds.json (the same map the handle scan uses) unless# COUNTRIES is set, so there is one list to maintain rather than two that drift.## Usage:# bash scripts/collect-countries.sh # every country in the map# COUNTRIES="nl be de" bash scripts/collect-countries.sh# DIR=/path/to/nl-estimate bash scripts/collect-countries.sh## Output per country, in $DIR:# sifa-<cc>.dids accounts with a Sifa location in that country# sifa-company-<cc>.dids company accounts for that country## Failures never abort the run: one country's API hiccup must not cost the other 44. Failed# countries are retried once at the end -- with 45 countries x 2 endpoints, the usual cause is# Sifa's 60/min rate limit rather than anything being wrong, and the same country succeeds when# run on its own. PAUSE spaces the countries out to stay under that limit in the first place.set -uo pipefail
HERE="$(cd "$(dirname "$0")" && pwd)"DIR="${DIR:-$HERE}"SLEEP="${SLEEP:-0.3}"PAUSE="${PAUSE:-1.5}" # seconds between countries; Sifa rate-limits at 60/minMAP="${MAP:-$HERE/country-tlds.json}"
# Country codes Sifa's API rejects. Empty since singi-labs/sifa-api#1280 landed, which added# fo (Faroe Islands) and gl (Greenland) -- autonomous Danish territories with their own ISO# 3166-1 codes that the validation list had omitted. Kept as a lever for the next such gap.SIFA_UNSUPPORTED="${SIFA_UNSUPPORTED:-}"
if [ -z "${COUNTRIES:-}" ]; then COUNTRIES="$(python3 -c "import json,sysprint(' '.join(sorted(set(json.load(open(sys.argv[1]))['tlds'].values()))))" "$MAP")"fi
echo "[collect] $(date -u +%FT%TZ) countries: $COUNTRIES"failed=""ok=0
run_country() { local cc="$1" case " $SIFA_UNSUPPORTED " in *" $cc "*) echo "[collect] $cc skipped (Sifa API does not accept this country code)"; return 0 ;; esac CC_UPPER="$(printf '%s' "$cc" | tr '[:lower:]' '[:upper:]')"
# Sifa location. --country takes the uppercase alpha-2 the API expects. if python3 "$HERE/nl-sifa-location.py" --country "$CC_UPPER" \ --out-prefix "$DIR/sifa-$cc" --sleep "$SLEEP" >/dev/null 2>&1; then n=$(wc -l < "$DIR/sifa-$cc.dids" 2>/dev/null || echo 0) [ "${n:-0}" -gt 0 ] && echo "[collect] $cc sifa-location: $n" else failed="$failed sifa-location:$cc" fi
# Sifa companies. if python3 "$HERE/nl-sifa-company.py" --country "$CC_UPPER" \ --out-prefix "$DIR/sifa-company-$cc" --dir "$DIR" >/dev/null 2>&1; then n=$(wc -l < "$DIR/sifa-company-$cc.dids" 2>/dev/null || echo 0) [ "${n:-0}" -gt 0 ] && echo "[collect] $cc sifa-company: $n" ok=$((ok + 1)) else failed="$failed sifa-company:$cc" return 1 fi return 0}
retry=""for cc in $COUNTRIES; do run_country "$cc" || retry="$retry $cc" sleep "$PAUSE"done
# One retry pass. Rate-limit rejections are the common case and they clear on their own.if [ -n "$retry" ]; then echo "[collect] retrying:$retry" failed="" for cc in $retry; do sleep 5 run_country "$cc" || true donefi
echo "[collect] done; $ok country-runs completed"if [ -n "$failed" ]; then # Not fatal: these feeders are backfillable, so a partial run is worth keeping. echo "[collect] still failing after retry:$failed" >&2fi