Updated docs and bug fixes and updated redeploy script

2026-03-17 05:13:01 +00:00 · 2026-02-09 15:23:02 +02:00 · 2026-02-09 15:23:02 +02:00 · a297a14b44
commit a297a14b44
parent e2b38c409a
12 changed files with 524 additions and 70 deletions
--- a/.gitignore
+++ b/.gitignore
@ -99,3 +99,5 @@ keys_backup/
 vps.txt

 bin-linux/
+
+website/
--- a/docs/DEVNET_INSTALL.md
+++ b/docs/DEVNET_INSTALL.md
@ -68,7 +68,8 @@ sudo orama install --no-pull --pre-built \
  --anyone-nickname <relay-name> \
  --anyone-wallet <wallet-address> \
  --anyone-contact "<contact-info>" \
-  --anyone-family "<fingerprint1>,<fingerprint2>,..."
+  --anyone-family "<fingerprint1>,<fingerprint2>,..." \
+  --anyone-bandwidth 30
 ```

 ## ns3 - Nameserver + Relay
@ -86,7 +87,8 @@ sudo orama install --no-pull --pre-built \
  --anyone-nickname <relay-name> \
  --anyone-wallet <wallet-address> \
  --anyone-contact "<contact-info>" \
-  --anyone-family "<fingerprint1>,<fingerprint2>,..."
+  --anyone-family "<fingerprint1>,<fingerprint2>,..." \
+  --anyone-bandwidth 30
 ```

 ## node4 - Non-Nameserver + Relay
@ -104,7 +106,8 @@ sudo orama install --no-pull --pre-built \
  --anyone-nickname <relay-name> \
  --anyone-wallet <wallet-address> \
  --anyone-contact "<contact-info>" \
-  --anyone-family "<fingerprint1>,<fingerprint2>,..."
+  --anyone-family "<fingerprint1>,<fingerprint2>,..." \
+  --anyone-bandwidth 30
 ```

 ## node5 - Non-Nameserver + Relay
@ -122,7 +125,8 @@ sudo orama install --no-pull --pre-built \
  --anyone-nickname <relay-name> \
  --anyone-wallet <wallet-address> \
  --anyone-contact "<contact-info>" \
-  --anyone-family "<fingerprint1>,<fingerprint2>,..."
+  --anyone-family "<fingerprint1>,<fingerprint2>,..." \
+  --anyone-bandwidth 30
 ```

 ## node6 - Non-Nameserver (No Anyone Relay)
--- a/docs/DEV_DEPLOY.md
+++ b/docs/DEV_DEPLOY.md
@ -228,6 +228,8 @@ To deploy to all nodes, repeat steps 3-5 (dev) or 3-4 (production) for each VPS
 | `--anyone-family <fps>` | Comma-separated fingerprints of related relays (MyFamily) |
 | `--anyone-orport <port>` | ORPort for relay (default: 9001) |
 | `--anyone-exit` | Configure as an exit relay (default: non-exit) |
+| `--anyone-bandwidth <pct>` | Limit relay to N% of VPS bandwidth (default: 30, 0=unlimited). Runs a speedtest during install to measure available bandwidth |
+| `--anyone-accounting <GB>` | Monthly data cap for relay in GB (0=unlimited) |

 #### `orama invite`

@ -249,6 +251,9 @@ To deploy to all nodes, repeat steps 3-5 (dev) or 3-4 (production) for each VPS
 | `--no-pull` | Skip git pull, use existing source |
 | `--pre-built` | Skip all Go compilation, use pre-built binaries already on disk |
 | `--restart` | Restart all services after upgrade |
+| `--anyone-relay` | Enable Anyone relay (same flags as install) |
+| `--anyone-bandwidth <pct>` | Limit relay to N% of VPS bandwidth (default: 30, 0=unlimited) |
+| `--anyone-accounting <GB>` | Monthly data cap for relay in GB (0=unlimited) |

 #### `orama prod` (Service Management)

--- a/pkg/cli/production/install/flags.go
+++ b/pkg/cli/production/install/flags.go
@ -42,6 +42,8 @@ type Flags struct {
 	AnyoneWallet     string // Ethereum wallet for rewards
 	AnyoneORPort     int    // ORPort for relay (default 9001)
 	AnyoneFamily     string // Comma-separated fingerprints of other relays you operate
+	AnyoneBandwidth  int    // Percentage of VPS bandwidth for relay (default: 30, 0=unlimited)
+	AnyoneAccounting int    // Monthly data cap for relay in GB (0=unlimited)
 }

 // ParseFlags parses install command flags
@ -87,6 +89,8 @@ func ParseFlags(args []string) (*Flags, error) {
 	fs.StringVar(&flags.AnyoneWallet, "anyone-wallet", "", "Ethereum wallet address for rewards")
 	fs.IntVar(&flags.AnyoneORPort, "anyone-orport", 9001, "ORPort for relay (default 9001)")
 	fs.StringVar(&flags.AnyoneFamily, "anyone-family", "", "Comma-separated fingerprints of other relays you operate")
+	fs.IntVar(&flags.AnyoneBandwidth, "anyone-bandwidth", 30, "Limit relay to N% of VPS bandwidth (0=unlimited, runs speedtest)")
+	fs.IntVar(&flags.AnyoneAccounting, "anyone-accounting", 0, "Monthly data cap for relay in GB (0=unlimited)")

 	if err := fs.Parse(args); err != nil {
 		if err == flag.ErrHelp {
--- a/pkg/cli/production/install/orchestrator.go
+++ b/pkg/cli/production/install/orchestrator.go
@ -58,6 +58,8 @@ func NewOrchestrator(flags *Flags) (*Orchestrator, error) {
 			Wallet:        flags.AnyoneWallet,
 			ORPort:        flags.AnyoneORPort,
 			MyFamily:      flags.AnyoneFamily,
+			BandwidthPct:  flags.AnyoneBandwidth,
+			AccountingMax: flags.AnyoneAccounting,
 		})
 	}

--- a/pkg/cli/production/install/validator.go
+++ b/pkg/cli/production/install/validator.go
@ -194,6 +194,16 @@ func (v *Validator) ValidateAnyoneRelayFlags() error {
 		return fmt.Errorf("--anyone-orport must be between 1 and 65535")
 	}

+	// Validate bandwidth percentage
+	if v.flags.AnyoneBandwidth < 0 || v.flags.AnyoneBandwidth > 100 {
+		return fmt.Errorf("--anyone-bandwidth must be between 0 and 100")
+	}
+
+	// Validate accounting
+	if v.flags.AnyoneAccounting < 0 {
+		return fmt.Errorf("--anyone-accounting must be >= 0")
+	}
+
 	// Display configuration summary
 	fmt.Printf("  Nickname:  %s\n", v.flags.AnyoneNickname)
 	fmt.Printf("  Contact:   %s\n", v.flags.AnyoneContact)
@ -204,6 +214,14 @@ func (v *Validator) ValidateAnyoneRelayFlags() error {
 	} else {
 		fmt.Printf("  Mode:      Non-exit Relay\n")
 	}
+	if v.flags.AnyoneBandwidth > 0 {
+		fmt.Printf("  Bandwidth: %d%% of VPS speed (speedtest will run during install)\n", v.flags.AnyoneBandwidth)
+	} else {
+		fmt.Printf("  Bandwidth: Unlimited\n")
+	}
+	if v.flags.AnyoneAccounting > 0 {
+		fmt.Printf("  Data cap:  %d GB/month\n", v.flags.AnyoneAccounting)
+	}

 	// Warning about token requirement
 	fmt.Printf("\n  ⚠️  IMPORTANT: Relay operators must hold 100 $ANYONE tokens\n")
--- a/pkg/cli/production/upgrade/flags.go
+++ b/pkg/cli/production/upgrade/flags.go
@ -25,6 +25,8 @@ type Flags struct {
 	AnyoneWallet     string
 	AnyoneORPort     int
 	AnyoneFamily     string
+	AnyoneBandwidth  int // Percentage of VPS bandwidth for relay (default: 30, 0=unlimited)
+	AnyoneAccounting int // Monthly data cap for relay in GB (0=unlimited)
 }

 // ParseFlags parses upgrade command flags
@ -53,6 +55,8 @@ func ParseFlags(args []string) (*Flags, error) {
 	fs.StringVar(&flags.AnyoneWallet, "anyone-wallet", "", "Ethereum wallet address for rewards")
 	fs.IntVar(&flags.AnyoneORPort, "anyone-orport", 9001, "ORPort for relay (default 9001)")
 	fs.StringVar(&flags.AnyoneFamily, "anyone-family", "", "Comma-separated fingerprints of other relays you operate")
+	fs.IntVar(&flags.AnyoneBandwidth, "anyone-bandwidth", 30, "Limit relay to N% of VPS bandwidth (0=unlimited, runs speedtest)")
+	fs.IntVar(&flags.AnyoneAccounting, "anyone-accounting", 0, "Monthly data cap for relay in GB (0=unlimited)")

 	// Support legacy flags for backwards compatibility
 	nightly := fs.Bool("nightly", false, "Use nightly branch (deprecated, use --branch nightly)")
--- a/pkg/cli/production/upgrade/orchestrator.go
+++ b/pkg/cli/production/upgrade/orchestrator.go
@ -58,6 +58,8 @@ func NewOrchestrator(flags *Flags) *Orchestrator {
 			Wallet:       flags.AnyoneWallet,
 			ORPort:       flags.AnyoneORPort,
 			MyFamily:     flags.AnyoneFamily,
+			BandwidthPct: flags.AnyoneBandwidth,
+			AccountingMax: flags.AnyoneAccounting,
 		})
 	}

--- a/pkg/environments/production/installers/anyone_relay.go
+++ b/pkg/environments/production/installers/anyone_relay.go
@ -9,6 +9,7 @@ import (
 	"path/filepath"
 	"regexp"
 	"strings"
+	"time"
 )

 // AnyoneRelayConfig holds configuration for the Anyone relay
@ -20,6 +21,9 @@ type AnyoneRelayConfig struct {
 	ExitRelay      bool   // Whether to run as exit relay
 	Migrate        bool   // Whether to migrate existing installation
 	MyFamily       string // Comma-separated list of family fingerprints (for multi-relay operators)
+	BandwidthRate  int    // RelayBandwidthRate in KBytes/s (0 = unlimited)
+	BandwidthBurst int    // RelayBandwidthBurst in KBytes/s (0 = unlimited)
+	AccountingMax  int    // Monthly data cap in GB (0 = unlimited)
 }

 // ExistingAnyoneInfo contains information about an existing Anyone installation
@ -336,6 +340,26 @@ func (ari *AnyoneRelayInstaller) generateAnonrc() string {
 	// Control port for monitoring
 	sb.WriteString("ControlPort 9051\n")

+	// Bandwidth limiting
+	if ari.config.BandwidthRate > 0 {
+		sb.WriteString("\n")
+		sb.WriteString("# Bandwidth limiting (managed by Orama Network)\n")
+		sb.WriteString(fmt.Sprintf("RelayBandwidthRate %d KBytes\n", ari.config.BandwidthRate))
+		sb.WriteString(fmt.Sprintf("RelayBandwidthBurst %d KBytes\n", ari.config.BandwidthBurst))
+
+		rateMbps := float64(ari.config.BandwidthRate) * 8 / 1024
+		burstMbps := float64(ari.config.BandwidthBurst) * 8 / 1024
+		sb.WriteString(fmt.Sprintf("# Rate: %.1f Mbps, Burst: %.1f Mbps\n", rateMbps, burstMbps))
+	}
+
+	// Monthly data cap
+	if ari.config.AccountingMax > 0 {
+		sb.WriteString("\n")
+		sb.WriteString("# Monthly data cap (managed by Orama Network)\n")
+		sb.WriteString("AccountingStart month 1 00:00\n")
+		sb.WriteString(fmt.Sprintf("AccountingMax %d GBytes\n", ari.config.AccountingMax))
+	}
+
 	// MyFamily for multi-relay operators (preserve from existing config)
 	if ari.config.MyFamily != "" {
 		sb.WriteString("\n")
@ -403,6 +427,62 @@ func (ari *AnyoneRelayInstaller) MigrateExistingInstallation(existing *ExistingA
 	return nil
 }

+// MeasureBandwidth downloads a test file and returns the measured download speed in KBytes/s.
+// Uses wget to download a 10MB file from a public CDN and measures throughput.
+// Returns 0 if the test fails (caller should skip bandwidth limiting).
+func MeasureBandwidth(logWriter io.Writer) (int, error) {
+	fmt.Fprintf(logWriter, "  Running bandwidth test...\n")
+
+	testFile := "/tmp/speedtest-orama.tmp"
+	defer os.Remove(testFile)
+
+	// Use wget with progress output to download a 10MB test file
+	// We time the download ourselves for accuracy
+	start := time.Now()
+	cmd := exec.Command("wget", "-q", "-O", testFile, "http://speedtest.tele2.net/10MB.zip")
+	cmd.Env = append(os.Environ(), "LC_ALL=C")
+
+	if err := cmd.Run(); err != nil {
+		fmt.Fprintf(logWriter, "    ⚠️  Bandwidth test failed: %v\n", err)
+		return 0, fmt.Errorf("bandwidth test download failed: %w", err)
+	}
+
+	elapsed := time.Since(start)
+
+	// Get file size
+	info, err := os.Stat(testFile)
+	if err != nil {
+		return 0, fmt.Errorf("failed to stat test file: %w", err)
+	}
+
+	// Calculate speed in KBytes/s
+	sizeKB := int(info.Size() / 1024)
+	seconds := elapsed.Seconds()
+	if seconds < 0.1 {
+		seconds = 0.1 // avoid division by zero
+	}
+	speedKBs := int(float64(sizeKB) / seconds)
+
+	speedMbps := float64(speedKBs) * 8 / 1024 // Convert KBytes/s to Mbps
+	fmt.Fprintf(logWriter, "    Measured download speed: %d KBytes/s (%.1f Mbps)\n", speedKBs, speedMbps)
+
+	return speedKBs, nil
+}
+
+// CalculateBandwidthLimits computes RelayBandwidthRate and RelayBandwidthBurst
+// from measured speed and a percentage. Returns rate and burst in KBytes/s.
+func CalculateBandwidthLimits(measuredKBs int, percent int) (rate int, burst int) {
+	rate = measuredKBs * percent / 100
+	burst = rate * 3 / 2 // 1.5x rate for burst headroom
+	if rate < 1 {
+		rate = 1
+	}
+	if burst < rate {
+		burst = rate
+	}
+	return rate, burst
+}
+
 // ValidateNickname validates the relay nickname (1-19 alphanumeric chars)
 func ValidateNickname(nickname string) error {
 	if len(nickname) < 1 || len(nickname) > 19 {
--- a/pkg/environments/production/orchestrator.go
+++ b/pkg/environments/production/orchestrator.go
@ -22,6 +22,8 @@ type AnyoneRelayConfig struct {
 	Wallet        string // Ethereum wallet for rewards
 	ORPort        int    // ORPort for relay (default 9001)
 	MyFamily      string // Comma-separated fingerprints of other relays (for multi-relay operators)
+	BandwidthPct  int    // Percentage of VPS bandwidth to allocate to relay (0 = unlimited)
+	AccountingMax int    // Monthly data cap in GB (0 = unlimited)
 }

 // ProductionSetup orchestrates the entire production deployment
@ -400,7 +402,24 @@ func (ps *ProductionSetup) Phase2bInstallBinaries() error {
 			ExitRelay:     ps.anyoneRelayConfig.Exit,
 			Migrate:       ps.anyoneRelayConfig.Migrate,
 			MyFamily:      ps.anyoneRelayConfig.MyFamily,
+			AccountingMax: ps.anyoneRelayConfig.AccountingMax,
 		}
+
+		// Run bandwidth test and calculate limits if percentage is set
+		if ps.anyoneRelayConfig.BandwidthPct > 0 {
+			measuredKBs, err := installers.MeasureBandwidth(ps.logWriter)
+			if err != nil {
+				ps.logf("  ⚠️  Bandwidth test failed, relay will run without bandwidth limits: %v", err)
+			} else if measuredKBs > 0 {
+				rate, burst := installers.CalculateBandwidthLimits(measuredKBs, ps.anyoneRelayConfig.BandwidthPct)
+				relayConfig.BandwidthRate = rate
+				relayConfig.BandwidthBurst = burst
+				rateMbps := float64(rate) * 8 / 1024
+				ps.logf("  ✓ Relay bandwidth limited to %d%% of measured speed (%d KBytes/s = %.1f Mbps)",
+					ps.anyoneRelayConfig.BandwidthPct, rate, rateMbps)
+			}
+		}
+
 		relayInstaller := installers.NewAnyoneRelayInstaller(ps.arch, ps.logWriter, relayConfig)

 		// Check for existing installation if migration is requested
--- a/scripts/redeploy.sh
+++ b/scripts/redeploy.sh
@ -0,0 +1,296 @@
+#!/usr/bin/env bash
+#
+# Redeploy to all nodes in a given environment (devnet or testnet).
+# Reads node credentials from scripts/remote-nodes.conf.
+#
+# Flow (per docs/DEV_DEPLOY.md):
+#  1) make build-linux
+#  2) scripts/generate-source-archive.sh -> /tmp/network-source.tar.gz
+#  3) scp archive + extract-deploy.sh + conf to hub node
+#  4) from hub: sshpass scp to all other nodes + sudo bash /tmp/extract-deploy.sh
+#  5) rolling: upgrade followers one-by-one, leader last
+#
+# Usage:
+#   scripts/redeploy.sh --devnet
+#   scripts/redeploy.sh --testnet
+#   scripts/redeploy.sh --devnet --no-build
+#   scripts/redeploy.sh --testnet --no-build
+#
+set -euo pipefail
+
+# ── Parse flags ──────────────────────────────────────────────────────────────
+ENV=""
+NO_BUILD=0
+
+for arg in "$@"; do
+  case "$arg" in
+    --devnet)   ENV="devnet" ;;
+    --testnet)  ENV="testnet" ;;
+    --no-build) NO_BUILD=1 ;;
+    -h|--help)
+      echo "Usage: scripts/redeploy.sh --devnet|--testnet [--no-build]"
+      exit 0
+      ;;
+    *)
+      echo "Unknown flag: $arg" >&2
+      echo "Usage: scripts/redeploy.sh --devnet|--testnet [--no-build]" >&2
+      exit 1
+      ;;
+  esac
+done
+
+if [[ -z "$ENV" ]]; then
+  echo "ERROR: specify --devnet or --testnet" >&2
+  exit 1
+fi
+
+# ── Paths ────────────────────────────────────────────────────────────────────
+ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
+CONF="$ROOT_DIR/scripts/remote-nodes.conf"
+ARCHIVE="/tmp/network-source.tar.gz"
+EXTRACT_SCRIPT="$ROOT_DIR/scripts/extract-deploy.sh"
+
+die() { echo "ERROR: $*" >&2; exit 1; }
+need_file() { [[ -f "$1" ]] || die "Missing file: $1"; }
+
+need_file "$CONF"
+need_file "$EXTRACT_SCRIPT"
+
+# ── Load nodes from conf ────────────────────────────────────────────────────
+HOSTS=()
+PASSES=()
+ROLES=()
+SSH_KEYS=()
+
+while IFS='|' read -r env host pass role key; do
+  [[ -z "$env" || "$env" == \#* ]] && continue
+  env="${env%%#*}"
+  env="$(echo "$env" | xargs)"
+  [[ "$env" != "$ENV" ]] && continue
+
+  HOSTS+=("$host")
+  PASSES+=("$pass")
+  ROLES+=("${role:-node}")
+  SSH_KEYS+=("${key:-}")
+done < "$CONF"
+
+if [[ ${#HOSTS[@]} -eq 0 ]]; then
+  die "No nodes found for environment '$ENV' in $CONF"
+fi
+
+echo "== redeploy.sh ($ENV) — ${#HOSTS[@]} nodes =="
+for i in "${!HOSTS[@]}"; do
+  echo "  [$i] ${HOSTS[$i]} (${ROLES[$i]})"
+done
+
+# ── Pick hub node ────────────────────────────────────────────────────────────
+# Hub = first node that has an SSH key configured (direct SCP from local).
+# If none have a key, use the first node (via sshpass).
+HUB_IDX=0
+HUB_KEY=""
+for i in "${!HOSTS[@]}"; do
+  if [[ -n "${SSH_KEYS[$i]}" ]]; then
+    expanded_key="${SSH_KEYS[$i]/#\~/$HOME}"
+    if [[ -f "$expanded_key" ]]; then
+      HUB_IDX=$i
+      HUB_KEY="$expanded_key"
+      break
+    fi
+  fi
+done
+
+HUB_HOST="${HOSTS[$HUB_IDX]}"
+HUB_PASS="${PASSES[$HUB_IDX]}"
+
+echo "Hub: $HUB_HOST (idx=$HUB_IDX, key=${HUB_KEY:-none})"
+
+# ── Build ────────────────────────────────────────────────────────────────────
+if [[ "$NO_BUILD" -eq 0 ]]; then
+  echo "== build-linux =="
+  (cd "$ROOT_DIR" && make build-linux) || {
+    echo "WARN: make build-linux failed; continuing if existing bin-linux is acceptable."
+  }
+else
+  echo "== skipping build (--no-build) =="
+fi
+
+# ── Generate source archive ─────────────────────────────────────────────────
+echo "== generate source archive =="
+(cd "$ROOT_DIR" && ./scripts/generate-source-archive.sh)
+need_file "$ARCHIVE"
+
+# ── Helper: SSH/SCP to hub ───────────────────────────────────────────────────
+SSH_OPTS=(-o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null)
+
+hub_scp() {
+  if [[ -n "$HUB_KEY" ]]; then
+    scp -i "$HUB_KEY" "${SSH_OPTS[@]}" "$@"
+  else
+    sshpass -p "$HUB_PASS" scp "${SSH_OPTS[@]}" "$@"
+  fi
+}
+
+hub_ssh() {
+  if [[ -n "$HUB_KEY" ]]; then
+    ssh -i "$HUB_KEY" "${SSH_OPTS[@]}" "$@"
+  else
+    sshpass -p "$HUB_PASS" ssh "${SSH_OPTS[@]}" "$@"
+  fi
+}
+
+# ── Upload to hub ────────────────────────────────────────────────────────────
+echo "== upload archive + extract script + conf to hub ($HUB_HOST) =="
+hub_scp "$ARCHIVE" "$EXTRACT_SCRIPT" "$CONF" "$HUB_HOST":/tmp/
+
+# ── Remote: fan-out + extract + rolling upgrade ─────────────────────────────
+echo "== fan-out + extract + rolling upgrade from hub =="
+
+hub_ssh "$HUB_HOST" "DEPLOY_ENV=$ENV HUB_IDX=$HUB_IDX bash -s" <<'REMOTE'
+set -euo pipefail
+export DEBIAN_FRONTEND=noninteractive
+
+TAR=/tmp/network-source.tar.gz
+EX=/tmp/extract-deploy.sh
+CONF=/tmp/remote-nodes.conf
+
+[[ -f "$TAR" ]] || { echo "Missing $TAR on hub"; exit 2; }
+[[ -f "$EX" ]]  || { echo "Missing $EX on hub"; exit 2; }
+[[ -f "$CONF" ]] || { echo "Missing $CONF on hub"; exit 2; }
+chmod +x "$EX" || true
+
+# Parse conf file on the hub — same format as local
+hosts=()
+passes=()
+idx=0
+hub_host=""
+hub_pass=""
+
+while IFS='|' read -r env host pass role key; do
+  [[ -z "$env" || "$env" == \#* ]] && continue
+  env="${env%%#*}"
+  env="$(echo "$env" | xargs)"
+  [[ "$env" != "$DEPLOY_ENV" ]] && continue
+
+  if [[ $idx -eq $HUB_IDX ]]; then
+    hub_host="$host"
+    hub_pass="$pass"
+  else
+    hosts+=("$host")
+    passes+=("$pass")
+  fi
+  ((idx++)) || true
+done < "$CONF"
+
+echo "Hub: $hub_host (this machine)"
+echo "Fan-out nodes: ${#hosts[@]}"
+
+# Install sshpass on hub if needed
+if [[ ${#hosts[@]} -gt 0 ]] && ! command -v sshpass >/dev/null 2>&1; then
+  echo "Installing sshpass on hub..."
+  printf '%s\n' "$hub_pass" | sudo -S apt-get update -y >/dev/null
+  printf '%s\n' "$hub_pass" | sudo -S apt-get install -y sshpass >/dev/null
+fi
+
+echo "== fan-out: upload to ${#hosts[@]} nodes =="
+for i in "${!hosts[@]}"; do
+  h="${hosts[$i]}"
+  p="${passes[$i]}"
+  echo "  -> $h"
+  sshpass -p "$p" scp -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \
+    "$TAR" "$EX" "$h":/tmp/
+done
+
+echo "== extract on all fan-out nodes =="
+for i in "${!hosts[@]}"; do
+  h="${hosts[$i]}"
+  p="${passes[$i]}"
+  echo "  -> $h"
+  sshpass -p "$p" ssh -n -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \
+    "$h" "printf '%s\n' '$p' | sudo -S bash /tmp/extract-deploy.sh >/tmp/extract.log 2>&1 && echo OK"
+done
+
+echo "== extract on hub =="
+printf '%s\n' "$hub_pass" | sudo -S bash "$EX" >/tmp/extract.log 2>&1
+
+# ── Raft state detection ──
+raft_state() {
+  local h="$1" p="$2"
+  local cmd="curl -s http://localhost:5001/status"
+  local parse_py='import sys,json; j=json.load(sys.stdin); r=j.get("store",{}).get("raft",{}); print((r.get("state") or ""), (r.get("num_peers") or 0), (r.get("voter") is True))'
+  sshpass -p "$p" ssh -n -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \
+    "$h" "$cmd | python3 -c '$parse_py'" 2>/dev/null || true
+}
+
+echo "== detect leader =="
+leader=""
+leader_pass=""
+
+for i in "${!hosts[@]}"; do
+  h="${hosts[$i]}"
+  p="${passes[$i]}"
+  out="$(raft_state "$h" "$p")"
+  echo "  $h -> ${out:-NO_OUTPUT}"
+  if [[ "$out" == Leader* ]]; then
+    leader="$h"
+    leader_pass="$p"
+    break
+  fi
+done
+
+# Check hub itself
+if [[ -z "$leader" ]]; then
+  hub_out="$(curl -s http://localhost:5001/status | python3 -c 'import sys,json; j=json.load(sys.stdin); r=j.get("store",{}).get("raft",{}); print((r.get("state") or ""), (r.get("num_peers") or 0), (r.get("voter") is True))' 2>/dev/null || true)"
+  echo "  hub(localhost) -> ${hub_out:-NO_OUTPUT}"
+  if [[ "$hub_out" == Leader* ]]; then
+    leader="HUB"
+    leader_pass="$hub_pass"
+  fi
+fi
+
+if [[ -z "$leader" ]]; then
+  echo "No leader detected. Aborting before upgrades."
+  exit 3
+fi
+echo "Leader: $leader"
+
+upgrade_one() {
+  local h="$1" p="$2"
+  echo "== upgrade $h =="
+  sshpass -p "$p" ssh -n -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \
+    "$h" "printf '%s\n' '$p' | sudo -S orama prod stop && printf '%s\n' '$p' | sudo -S orama upgrade --no-pull --pre-built --restart"
+}
+
+upgrade_hub() {
+  echo "== upgrade hub (localhost) =="
+  printf '%s\n' "$hub_pass" | sudo -S orama prod stop
+  printf '%s\n' "$hub_pass" | sudo -S orama upgrade --no-pull --pre-built --restart
+}
+
+echo "== rolling upgrade (followers first) =="
+for i in "${!hosts[@]}"; do
+  h="${hosts[$i]}"
+  p="${passes[$i]}"
+  [[ "$h" == "$leader" ]] && continue
+  upgrade_one "$h" "$p"
+done
+
+# Upgrade hub if not the leader
+if [[ "$leader" != "HUB" ]]; then
+  upgrade_hub
+fi
+
+# Upgrade leader last
+echo "== upgrade leader last =="
+if [[ "$leader" == "HUB" ]]; then
+  upgrade_hub
+else
+  upgrade_one "$leader" "$leader_pass"
+fi
+
+# Clean up conf from hub
+rm -f "$CONF"
+
+echo "Done."
+REMOTE
+
+echo "== complete =="
--- a/scripts/remote-nodes.conf.example
+++ b/scripts/remote-nodes.conf.example
@ -1,8 +1,26 @@
 # Remote node configuration
-# Format: node_number|user@host|password
-# Copy this file to remote-nodes.conf and fill in your credentials
+# Format: environment|user@host|password|role|ssh_key (optional)
+# environment: devnet, testnet
+# role: node, nameserver-ns1, nameserver-ns2, nameserver-ns3
+# ssh_key: optional path to SSH key (if node requires key-based auth instead of sshpass)
+#
+# Copy this file to remote-nodes.conf and fill in your credentials.
+# The first node with an SSH key will be used as the hub (fan-out relay).

-1|ubuntu@51.83.128.181|your_password_here
-2|root@194.61.28.7|your_password_here
-3|root@83.171.248.66|your_password_here
-4|root@62.72.44.87|your_password_here
+# --- Devnet nameservers ---
+devnet|root@1.2.3.4|your_password_here|nameserver-ns1
+devnet|ubuntu@1.2.3.5|your_password_here|nameserver-ns2
+devnet|root@1.2.3.6|your_password_here|nameserver-ns3
+
+# --- Devnet nodes ---
+devnet|ubuntu@1.2.3.7|your_password_here|node
+devnet|ubuntu@1.2.3.8|your_password_here|node|~/.ssh/my_key/id_ed25519
+
+# --- Testnet nameservers ---
+testnet|ubuntu@2.3.4.5|your_password_here|nameserver-ns1
+testnet|ubuntu@2.3.4.6|your_password_here|nameserver-ns2
+testnet|ubuntu@2.3.4.7|your_password_here|nameserver-ns3
+
+# --- Testnet nodes ---
+testnet|root@2.3.4.8|your_password_here|node
+testnet|ubuntu@2.3.4.9|your_password_here|node