DeBros/orama
4
0
Fork 0
mirror of https://github.com/DeBrosOfficial/orama.git synced 2026-03-27 13:04:12 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #83 from DeBrosDAO/0.115.0

Browse Source 
0.115.0
This commit is contained in:
anonpenguin 2026-03-20 07:22:43 +02:00 committed by GitHub
commit c4fd1878a7
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
149 changed files with 12122 additions and 2918 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

64
Makefile
View File

@ -61,9 +61,9 @@ test-e2e-quick:
# Network - Distributed P2P Database System
# Makefile for development and build tasks
.PHONY: build clean test deps tidy fmt vet lint install-hooks upload-devnet upload-testnet redeploy-devnet redeploy-testnet release health
.PHONY: build clean test deps tidy fmt vet lint install-hooks push-devnet push-testnet rollout-devnet rollout-testnet release
VERSION := 0.112.8
VERSION := 0.115.0
COMMIT ?= $(shell git rev-parse --short HEAD 2>/dev/null || echo unknown)
DATE ?= $(shell date -u +%Y-%m-%dT%H:%M:%SZ)
LDFLAGS := -X 'main.version=$(VERSION)' -X 'main.commit=$(COMMIT)' -X 'main.date=$(DATE)'
@ -89,9 +89,13 @@ build-linux: deps
GOOS=linux GOARCH=amd64 go build -ldflags "$(LDFLAGS_LINUX)" -trimpath -o bin-linux/orama ./cmd/cli/
@echo "✓ CLI built at bin-linux/orama"
@echo ""
@echo "Next steps:"
@echo " ./scripts/generate-source-archive.sh"
@echo " ./bin/orama install --vps-ip <ip> --nameserver --domain ..."
@echo "Prefer 'make build-archive' for full pre-built binary archive."
# Build pre-compiled binary archive for deployment (all binaries + deps)
build-archive: deps
@echo "Building binary archive (version=$(VERSION))..."
go build -ldflags "$(LDFLAGS)" -o bin/orama ./cmd/cli/
./bin/orama build --output /tmp/orama-$(VERSION)-linux-amd64.tar.gz
# Install git hooks
install-hooks:
@ -105,29 +109,21 @@ clean:
rm -rf data/
@echo "Clean complete!"
# Upload source to devnet using fanout (upload to 1 node, parallel distribute to rest)
upload-devnet:
@bash scripts/upload-source-fanout.sh --env devnet
# Push binary archive to devnet nodes (fanout distribution)
push-devnet:
./bin/orama node push --env devnet
# Upload source to testnet using fanout
upload-testnet:
@bash scripts/upload-source-fanout.sh --env testnet
# Push binary archive to testnet nodes (fanout distribution)
push-testnet:
./bin/orama node push --env testnet
# Deploy to devnet (build + rolling upgrade all nodes)
redeploy-devnet:
@bash scripts/redeploy.sh --devnet
# Full rollout to devnet (build + push + rolling upgrade)
rollout-devnet:
./bin/orama node rollout --env devnet --yes
# Deploy to devnet without rebuilding
redeploy-devnet-quick:
@bash scripts/redeploy.sh --devnet --no-build
# Deploy to testnet (build + rolling upgrade all nodes)
redeploy-testnet:
@bash scripts/redeploy.sh --testnet
# Deploy to testnet without rebuilding
redeploy-testnet-quick:
@bash scripts/redeploy.sh --testnet --no-build
# Full rollout to testnet (build + push + rolling upgrade)
rollout-testnet:
./bin/orama node rollout --env testnet --yes
# Interactive release workflow (tag + push)
release:
@ -140,14 +136,7 @@ health:
echo "Usage: make health ENV=devnet|testnet"; \
exit 1; \
fi
@while IFS='|' read -r env host pass role key; do \
[ -z "$$env" ] && continue; \
case "$$env" in \#*) continue;; esac; \
env="$$(echo "$$env" | xargs)"; \
[ "$$env" != "$(ENV)" ] && continue; \
role="$$(echo "$$role" | xargs)"; \
bash scripts/check-node-health.sh "$$host" "$$pass" "$$host ($$role)"; \
done < scripts/remote-nodes.conf
./bin/orama monitor report --env $(ENV)
# Help
help:
@ -170,10 +159,11 @@ help:
@echo " ORAMA_GATEWAY_URL=https://orama-devnet.network make test-e2e-prod"
@echo ""
@echo "Deployment:"
@echo " make redeploy-devnet - Build + rolling deploy to all devnet nodes"
@echo " make redeploy-devnet-quick - Deploy to devnet without rebuilding"
@echo " make redeploy-testnet - Build + rolling deploy to all testnet nodes"
@echo " make redeploy-testnet-quick- Deploy to testnet without rebuilding"
@echo " make build-archive - Build pre-compiled binary archive for deployment"
@echo " make push-devnet - Push binary archive to devnet nodes"
@echo " make push-testnet - Push binary archive to testnet nodes"
@echo " make rollout-devnet - Full rollout: build + push + rolling upgrade (devnet)"
@echo " make rollout-testnet - Full rollout: build + push + rolling upgrade (testnet)"
@echo " make health ENV=devnet - Check health of all nodes in an environment"
@echo " make release - Interactive release workflow (tag + push)"
@echo ""

14
README.md
View File

@ -349,13 +349,13 @@ All configuration lives in `~/.orama/`:
```bash
# Check status
systemctl status orama-node
sudo orama node status
# View logs
journalctl -u orama-node -f
orama node logs node --follow
# Check log files
tail -f /opt/orama/.orama/logs/node.log
sudo orama node doctor
```
### Port Conflicts
@ -417,9 +417,11 @@ See `openapi/gateway.yaml` for complete API specification.
- **[Deployment Guide](docs/DEPLOYMENT_GUIDE.md)** - Deploy React, Next.js, Go apps and manage databases
- **[Architecture Guide](docs/ARCHITECTURE.md)** - System architecture and design patterns
- **[Client SDK](docs/CLIENT_SDK.md)** - Go SDK documentation and examples
- **[Gateway API](docs/GATEWAY_API.md)** - Complete HTTP API reference
- **[Security Deployment](docs/SECURITY_DEPLOYMENT_GUIDE.md)** - Production security hardening
- **[Testing Plan](docs/TESTING_PLAN.md)** - Comprehensive testing strategy and implementation
- **[Monitoring](docs/MONITORING.md)** - Cluster monitoring and health checks
- **[Inspector](docs/INSPECTOR.md)** - Deep subsystem health inspection
- **[Serverless Functions](docs/SERVERLESS.md)** - WASM serverless with host functions
- **[WebRTC](docs/WEBRTC.md)** - Real-time communication setup
- **[Common Problems](docs/COMMON_PROBLEMS.md)** - Troubleshooting known issues
## Resources

8
cmd/cli/root.go
View File

@ -9,6 +9,7 @@ import (
// Command groups
"github.com/DeBrosOfficial/network/pkg/cli/cmd/app"
"github.com/DeBrosOfficial/network/pkg/cli/cmd/authcmd"
"github.com/DeBrosOfficial/network/pkg/cli/cmd/buildcmd"
"github.com/DeBrosOfficial/network/pkg/cli/cmd/dbcmd"
deploycmd "github.com/DeBrosOfficial/network/pkg/cli/cmd/deploy"
"github.com/DeBrosOfficial/network/pkg/cli/cmd/envcmd"
@ -17,6 +18,7 @@ import (
"github.com/DeBrosOfficial/network/pkg/cli/cmd/monitorcmd"
"github.com/DeBrosOfficial/network/pkg/cli/cmd/namespacecmd"
"github.com/DeBrosOfficial/network/pkg/cli/cmd/node"
"github.com/DeBrosOfficial/network/pkg/cli/cmd/sandboxcmd"
)
// version metadata populated via -ldflags at build time
@ -83,6 +85,12 @@ and interacting with the Orama distributed network.`,
// Serverless function commands
rootCmd.AddCommand(functioncmd.Cmd)
// Build command (cross-compile binary archive)
rootCmd.AddCommand(buildcmd.Cmd)
// Sandbox command (ephemeral Hetzner Cloud clusters)
rootCmd.AddCommand(sandboxcmd.Cmd)
return rootCmd
}

54
docs/ARCHITECTURE.md
View File

@ -357,11 +357,36 @@ Function Invocation:
All inter-node communication is encrypted via a WireGuard VPN mesh:
- **WireGuard IPs:** Each node gets a private IP (10.0.0.x) used for all cluster traffic
- **WireGuard IPs:** Each node gets a private IP (10.0.0.x/24) used for all cluster traffic
- **UFW Firewall:** Only public ports are exposed: 22 (SSH), 53 (DNS, nameservers only), 80/443 (HTTP/HTTPS), 51820 (WireGuard UDP)
- **IPv6 disabled:** System-wide via sysctl to prevent bypass of IPv4 firewall rules
- **Internal services** (RQLite 5001/7001, IPFS 4001/4501, Olric 3320/3322, Gateway 6001) are only accessible via WireGuard or localhost
- **Invite tokens:** Single-use, time-limited tokens for secure node joining. No shared secrets on the CLI
- **Join flow:** New nodes authenticate via HTTPS (443), establish WireGuard tunnel, then join all services over the encrypted mesh
- **Join flow:** New nodes authenticate via HTTPS (443) with TOFU certificate pinning, establish WireGuard tunnel, then join all services over the encrypted mesh
### Service Authentication
- **RQLite:** HTTP basic auth on all queries/executions — credentials generated at genesis, distributed via join response
- **Olric:** Memberlist gossip encrypted with a shared 32-byte key
- **IPFS Cluster:** TrustedPeers restricted to known cluster peer IDs (not `*`)
- **Internal endpoints:** `/v1/internal/wg/peers` and `/v1/internal/wg/peer/remove` require cluster secret
- **Vault:** V1 push/pull endpoints require session token authentication when guardian is configured
- **WebSockets:** Origin header validated against the node's configured domain
### Token & Key Security
- **Refresh tokens:** Stored as SHA-256 hashes (never plaintext)
- **API keys:** Stored as HMAC-SHA256 hashes with a server-side secret
- **TURN secrets:** Encrypted at rest with AES-256-GCM (key derived from cluster secret)
- **Binary signing:** Build archives signed with rootwallet EVM signature, verified on install
### Process Isolation
- **Dedicated user:** All services run as `orama` user (not root)
- **systemd hardening:** `ProtectSystem=strict`, `NoNewPrivileges=yes`, `PrivateDevices=yes`, etc.
- **Capabilities:** Caddy and CoreDNS get `CAP_NET_BIND_SERVICE` for privileged ports
See [SECURITY.md](SECURITY.md) for the full security hardening reference.
### TLS/HTTPS
@ -504,6 +529,31 @@ WebRTC uses a separate port allocation system from core namespace services:
See [docs/WEBRTC.md](WEBRTC.md) for full details including client integration, API reference, and debugging.
## OramaOS
For mainnet, devnet, and testnet environments, nodes run **OramaOS** — a custom minimal Linux image built with Buildroot.
**Key properties:**
- No SSH, no shell — operators cannot access the filesystem
- LUKS full-disk encryption with Shamir key distribution across peers
- Read-only rootfs (SquashFS + dm-verity)
- A/B partition updates with cryptographic signature verification
- Service sandboxing via Linux namespaces + seccomp
- Single root process: the **orama-agent**
**The orama-agent manages:**
- Boot sequence and LUKS key reconstruction
- WireGuard tunnel setup
- Service lifecycle in sandboxed namespaces
- Command reception from Gateway over WireGuard (port 9998)
- OS updates (download, verify, A/B swap, reboot with rollback)
**Node enrollment:** OramaOS nodes join via `orama node enroll` instead of `orama node install`. The enrollment flow uses a registration code + invite token + wallet verification.
See [ORAMAOS_DEPLOYMENT.md](ORAMAOS_DEPLOYMENT.md) for the full deployment guide.
Sandbox clusters remain on Ubuntu for development convenience.
## Future Enhancements
1. **GraphQL Support** - GraphQL gateway alongside REST

2
docs/CLEAN_NODE.md
View File

@ -2,6 +2,8 @@
How to completely remove all Orama Network state from a VPS so it can be reinstalled fresh.
> **OramaOS nodes:** This guide applies to Ubuntu-based nodes only. OramaOS has no SSH or shell access. To remove an OramaOS node: use `POST /v1/node/leave` via the Gateway API for graceful departure, or reflash the OramaOS image via your VPS provider's dashboard for a factory reset. See [ORAMAOS_DEPLOYMENT.md](ORAMAOS_DEPLOYMENT.md) for details.
## Quick Clean (Copy-Paste)
Run this as root or with sudo on the target VPS:

73
docs/COMMON_PROBLEMS.md
View File

@ -32,7 +32,7 @@ wg set wg0 peer <NodeA-pubkey> remove
wg set wg0 peer <NodeA-pubkey> endpoint <NodeA-public-ip>:51820 allowed-ips <NodeA-wg-ip>/32 persistent-keepalive 25
```
Then restart services: `sudo orama prod restart`
Then restart services: `sudo orama node restart`
You can find peer public keys with `wg show wg0`.
@ -46,7 +46,7 @@ cat /opt/orama/.orama/data/namespaces/<name>/configs/olric-*.yaml
If `bindAddr` is `0.0.0.0`, the node will try to bind to IPv6 on dual-stack hosts, breaking memberlist gossip.
**Fix:** Edit the YAML to use the node's WireGuard IP (run `ip addr show wg0` to find it), then restart: `sudo orama prod restart`
**Fix:** Edit the YAML to use the node's WireGuard IP (run `ip addr show wg0` to find it), then restart: `sudo orama node restart`
This was fixed in code (BindAddr validation in `SpawnOlric`), so new namespaces won't have this issue.
@ -82,7 +82,7 @@ olric_servers:
- "10.0.0.Z:10002"
```
Then: `sudo orama prod restart`
Then: `sudo orama node restart`
This was fixed in code, so new namespaces get the correct config.
@ -90,7 +90,7 @@ This was fixed in code, so new namespaces get the correct config.
## 3. Namespace not restoring after restart (missing cluster-state.json)
**Symptom:** After `orama prod restart`, the namespace services don't come back because `RestoreLocalClustersFromDisk` has no state file.
**Symptom:** After `orama node restart`, the namespace services don't come back because `RestoreLocalClustersFromDisk` has no state file.
**Check:**
@ -117,9 +117,9 @@ This was fixed in code — `ProvisionCluster` now saves state to all nodes (incl
## 4. Namespace gateway processes not restarting after upgrade
**Symptom:** After `orama upgrade --restart` or `orama prod restart`, namespace gateway/olric/rqlite services don't start.
**Symptom:** After `orama upgrade --restart` or `orama node restart`, namespace gateway/olric/rqlite services don't start.
**Cause:** `orama prod stop` disables systemd template services (`orama-namespace-gateway@<name>.service`). They have `PartOf=orama-node.service`, but that only propagates restart to **enabled** services.
**Cause:** `orama node stop` disables systemd template services (`orama-namespace-gateway@<name>.service`). They have `PartOf=orama-node.service`, but that only propagates restart to **enabled** services.
**Fix:** Re-enable the services before restarting:
@ -127,7 +127,7 @@ This was fixed in code — `ProvisionCluster` now saves state to all nodes (incl
systemctl enable orama-namespace-rqlite@<name>.service
systemctl enable orama-namespace-olric@<name>.service
systemctl enable orama-namespace-gateway@<name>.service
sudo orama prod restart
sudo orama node restart
```
This was fixed in code — the upgrade orchestrator now re-enables `@` services before restarting.
@ -150,11 +150,68 @@ ssh -n user@host 'command'
---
---
## 6. RQLite returns 401 Unauthorized
**Symptom:** RQLite queries fail with HTTP 401 after security hardening.
**Cause:** RQLite now requires basic auth. The client isn't sending credentials.
**Fix:** Ensure the RQLite client is configured with the credentials from `/opt/orama/.orama/secrets/rqlite-auth.json`. The central RQLite client wrapper (`pkg/rqlite/client.go`) handles this automatically. If using a standalone client (e.g., CoreDNS plugin), ensure it's also configured.
---
## 7. Olric cluster split after upgrade
**Symptom:** Olric nodes can't gossip after enabling memberlist encryption.
**Cause:** Olric memberlist encryption is all-or-nothing. Nodes with encryption can't communicate with nodes without it.
**Fix:** All nodes must be restarted simultaneously when enabling Olric encryption. The cache will be lost (it rebuilds from DB). This is expected — Olric is a cache, not persistent storage.
---
## 8. OramaOS: LUKS unlock fails
**Symptom:** OramaOS node can't reconstruct its LUKS key after reboot.
**Cause:** Not enough peer vault-guardians are online to meet the Shamir threshold (K = max(3, N/3)).
**Fix:** Ensure enough cluster nodes are online and reachable over WireGuard. The agent retries with exponential backoff. For genesis nodes before 5+ peers exist, use:
```bash
orama node unlock --genesis --node-ip <wg-ip>
```
---
## 9. OramaOS: Enrollment timeout
**Symptom:** `orama node enroll` hangs or times out.
**Cause:** The OramaOS node's port 9999 isn't reachable, or the Gateway can't reach the node's WebSocket.
**Fix:** Check that port 9999 is open in your VPS provider's external firewall (Hetzner firewall, AWS security groups, etc.). OramaOS opens it internally, but provider-level firewalls must be configured separately.
---
## 10. Binary signature verification fails
**Symptom:** `orama node install` rejects the binary archive with a signature error.
**Cause:** The archive was tampered with, or the manifest.sig file is missing/corrupted.
**Fix:** Rebuild the archive with `orama build` and re-sign with `make sign` (in the orama-os repo). Ensure you're using the rootwallet that matches the embedded signer address.
---
## General Debugging Tips
- **Always use `sudo orama prod restart`** instead of raw `systemctl` commands
- **Always use `sudo orama node restart`** instead of raw `systemctl` commands
- **Namespace data lives at:** `/opt/orama/.orama/data/namespaces/<name>/`
- **Check service logs:** `journalctl -u orama-namespace-olric@<name>.service --no-pager -n 50`
- **Check WireGuard:** `wg show wg0` — look for recent handshakes and transfer bytes
- **Check gateway health:** `curl http://localhost:<port>/v1/health` from the node itself
- **Node IPs:** Check `scripts/remote-nodes.conf` for credentials, `wg show wg0` for WG IPs
- **OramaOS nodes:** No SSH access — use Gateway API endpoints (`/v1/node/status`, `/v1/node/logs`) for diagnostics

225
docs/DEV_DEPLOY.md
View File

@ -27,87 +27,64 @@ make test
## Deploying to VPS
Source is always deployed via SCP (no git on VPS). The CLI is the only binary cross-compiled locally; everything else is built from source on the VPS.
All binaries are pre-compiled locally and shipped as a binary archive. Zero compilation on the VPS.
### Deploy Workflow
```bash
# 1. Cross-compile the CLI for Linux
make build-linux
# One-command: build + push + rolling upgrade
orama node rollout --env testnet
# 2. Generate a source archive (includes CLI binary + full source)
./scripts/generate-source-archive.sh
# Creates: /tmp/network-source.tar.gz
# Or step by step:
# 3. Install on a new VPS (handles SCP, extract, and remote install automatically)
./bin/orama node install --vps-ip <ip> --nameserver --domain <domain> --base-domain <domain>
# 1. Build binary archive (cross-compiles all binaries for linux/amd64)
orama build
# Creates: /tmp/orama-<version>-linux-amd64.tar.gz
# Or upgrade an existing VPS
./bin/orama node upgrade --restart
# 2. Push archive to all nodes (fanout via hub node)
orama node push --env testnet
# 3. Rolling upgrade (one node at a time, followers first, leader last)
orama node upgrade --env testnet
```
The `orama node install` command automatically:
1. Uploads the source archive via SCP
2. Extracts source to `/opt/orama/src` and installs the CLI to `/usr/local/bin/orama`
3. Runs `orama node install` on the VPS which builds all binaries from source (Go, CoreDNS, Caddy, Olric, etc.)
### Fresh Node Install
```bash
# Build the archive first (if not already built)
orama build
# Install on a new VPS (auto-uploads binary archive, zero compilation)
orama node install --vps-ip <ip> --nameserver --domain <domain> --base-domain <domain>
```
The installer auto-detects the binary archive at `/opt/orama/manifest.json` and copies pre-built binaries instead of compiling from source.
### Upgrading a Multi-Node Cluster (CRITICAL)
**NEVER restart all nodes simultaneously.** RQLite uses Raft consensus and requires a majority (quorum) to function. Restarting all nodes at once can cause cluster splits where nodes elect different leaders or form isolated clusters.
**NEVER restart all nodes simultaneously.** RQLite uses Raft consensus and requires a majority (quorum) to function.
#### Safe Upgrade Procedure (Rolling Restart)
Always upgrade nodes **one at a time**, waiting for each to rejoin before proceeding:
#### Safe Upgrade Procedure
```bash
# 1. Build CLI + generate archive
make build-linux
./scripts/generate-source-archive.sh
# Creates: /tmp/network-source.tar.gz
# Full rollout (build + push + rolling upgrade, one command)
orama node rollout --env testnet
# 2. Upload to ONE node first (the "hub" node)
sshpass -p '<password>' scp /tmp/network-source.tar.gz ubuntu@<hub-ip>:/tmp/
# Or with more control:
orama node push --env testnet # Push archive to all nodes
orama node upgrade --env testnet # Rolling upgrade (auto-detects leader)
orama node upgrade --env testnet --node 1.2.3.4 # Single node only
orama node upgrade --env testnet --delay 60 # 60s between nodes
```
# 3. Fan out from hub to all other nodes (server-to-server is faster)
ssh ubuntu@<hub-ip>
for ip in <ip2> <ip3> <ip4> <ip5> <ip6>; do
scp /tmp/network-source.tar.gz ubuntu@$ip:/tmp/
done
exit
The rolling upgrade automatically:
1. Upgrades **follower** nodes first
2. Upgrades the **leader** last
3. Waits a configurable delay between nodes (default: 30s)
# 4. Extract on ALL nodes (can be done in parallel, no restart yet)
for ip in <ip1> <ip2> <ip3> <ip4> <ip5> <ip6>; do
ssh ubuntu@$ip 'sudo bash -s' < scripts/extract-deploy.sh
done
# 5. Find the RQLite leader (upgrade this one LAST)
orama monitor report --env <env>
# Check "rqlite_leader" in summary output
# 6. Upgrade FOLLOWER nodes one at a time
ssh ubuntu@<follower-ip> 'sudo orama node stop && sudo orama node upgrade --restart'
# IMPORTANT: Verify FULL health before proceeding to next node:
orama monitor report --env <env> --node <follower-ip>
# Check:
# - All services active, 0 restart loops
# - RQLite: Follower state, applied_index matches cluster
# - All RQLite peers reachable (no partition alerts)
# - WireGuard peers connected with recent handshakes
# Only proceed to next node after ALL checks pass.
#
# NOTE: After restarting a node, other nodes may briefly report it as
# "unreachable" with "broken pipe" errors. This is normal — Raft TCP
# connections need ~1-2 minutes to re-establish. Wait and re-check
# before escalating.
# Repeat for each follower...
# 7. Upgrade the LEADER node last
ssh ubuntu@<leader-ip> 'sudo orama node stop && sudo orama node upgrade --restart'
# Verify the new leader was elected and cluster is fully healthy:
orama monitor report --env <env>
After each node, verify health:
```bash
orama monitor report --env testnet
```
#### What NOT to Do
@ -121,31 +98,38 @@ orama monitor report --env <env>
If nodes get stuck in "Candidate" state or show "leader not found" errors:
1. Identify which node has the most recent data (usually the old leader)
2. Keep that node running as the new leader
3. On each other node, clear RQLite data and restart:
```bash
sudo orama node stop
sudo rm -rf /opt/orama/.orama/data/rqlite
sudo systemctl start orama-node
```
4. The node should automatically rejoin using its configured `rqlite_join_address`
If automatic rejoin fails, the node may have started without the `-join` flag. Check:
```bash
ps aux | grep rqlited
# Should include: -join 10.0.0.1:7001 (or similar)
# Recover the Raft cluster (specify the node with highest commit index as leader)
orama node recover-raft --env testnet --leader 1.2.3.4
```
If `-join` is missing, the node bootstrapped standalone. You'll need to either:
- Restart orama-node (it should detect empty data and use join)
- Or do a full cluster rebuild from CLEAN_NODE.md
This will:
1. Stop orama-node on ALL nodes
2. Backup + delete raft/ on non-leader nodes
3. Start the leader, wait for Leader state
4. Start remaining nodes in batches
5. Verify cluster health
### Deploying to Multiple Nodes
### Cleaning Nodes for Reinstallation
To deploy to all nodes, repeat steps 3-5 (dev) or 3-4 (production) for each VPS IP.
```bash
# Wipe all data and services (preserves Anyone relay keys)
orama node clean --env testnet --force
**Important:** When using `--restart`, do nodes one at a time (see "Upgrading a Multi-Node Cluster" above).
# Also remove shared binaries (rqlited, ipfs, caddy, etc.)
orama node clean --env testnet --nuclear --force
# Single node only
orama node clean --env testnet --node 1.2.3.4 --force
```
### Push Options
```bash
orama node push --env devnet # Fanout via hub (default, fastest)
orama node push --env testnet --node 1.2.3.4 # Single node
orama node push --env testnet --direct # Sequential, no fanout
```
### CLI Flags Reference
@ -189,11 +173,56 @@ To deploy to all nodes, repeat steps 3-5 (dev) or 3-4 (production) for each VPS
| Flag | Description |
|------|-------------|
| `--restart` | Restart all services after upgrade |
| `--restart` | Restart all services after upgrade (local mode) |
| `--env <env>` | Target environment for remote rolling upgrade |
| `--node <ip>` | Upgrade a single node only |
| `--delay <seconds>` | Delay between nodes during rolling upgrade (default: 30) |
| `--anyone-relay` | Enable Anyone relay (same flags as install) |
| `--anyone-bandwidth <pct>` | Limit relay to N% of VPS bandwidth (default: 30, 0=unlimited) |
| `--anyone-accounting <GB>` | Monthly data cap for relay in GB (0=unlimited) |
#### `orama build`
| Flag | Description |
|------|-------------|
| `--arch <arch>` | Target architecture (default: amd64) |
| `--output <path>` | Output archive path |
| `--verbose` | Verbose build output |
#### `orama node push`
| Flag | Description |
|------|-------------|
| `--env <env>` | Target environment (required) |
| `--node <ip>` | Push to a single node only |
| `--direct` | Sequential upload (no hub fanout) |
#### `orama node rollout`
| Flag | Description |
|------|-------------|
| `--env <env>` | Target environment (required) |
| `--no-build` | Skip the build step |
| `--yes` | Skip confirmation |
| `--delay <seconds>` | Delay between nodes (default: 30) |
#### `orama node clean`
| Flag | Description |
|------|-------------|
| `--env <env>` | Target environment (required) |
| `--node <ip>` | Clean a single node only |
| `--nuclear` | Also remove shared binaries |
| `--force` | Skip confirmation (DESTRUCTIVE) |
#### `orama node recover-raft`
| Flag | Description |
|------|-------------|
| `--env <env>` | Target environment (required) |
| `--leader <ip>` | Leader node IP — highest commit index (required) |
| `--force` | Skip confirmation (DESTRUCTIVE) |
#### `orama node` (Service Management)
Use these commands to manage services on production nodes:
@ -291,7 +320,35 @@ is properly configured, always use the HTTPS domain URL.
UFW from external access. The join request goes through Caddy on port 80 (HTTP) or 443 (HTTPS),
which proxies to the gateway internally.
## Pre-Install Checklist
## OramaOS Enrollment
For OramaOS nodes (mainnet, devnet, testnet), use the enrollment flow instead of `orama node install`:
```bash
# 1. Flash OramaOS image to VPS (via provider dashboard)
# 2. Generate invite token on existing cluster node
orama node invite --expiry 24h
# 3. Enroll the OramaOS node
orama node enroll --node-ip <vps-public-ip> --token <invite-token> --gateway <gateway-url>
# 4. For genesis node reboots (before 5+ peers exist)
orama node unlock --genesis --node-ip <wg-ip>
```
OramaOS nodes have no SSH access. All management happens through the Gateway API:
```bash
# Status, logs, commands — all via Gateway proxy
curl "https://gateway.example.com/v1/node/status?node_id=<id>"
curl "https://gateway.example.com/v1/node/logs?node_id=<id>&service=gateway"
```
See [ORAMAOS_DEPLOYMENT.md](ORAMAOS_DEPLOYMENT.md) for the full guide.
**Note:** `orama node clean` does not work on OramaOS nodes (no SSH). Use `orama node leave` for graceful departure, or reflash the image for a factory reset.
## Pre-Install Checklist (Ubuntu Only)
Before running `orama node install` on a VPS, ensure:

10
docs/INSPECTOR.md
View File

@ -167,18 +167,18 @@ The inspector reads node definitions from a pipe-delimited config file (default:
### Format
```
# environment|user@host|password|role|ssh_key
devnet|ubuntu@1.2.3.4|mypassword|node|
devnet|ubuntu@5.6.7.8|mypassword|nameserver-ns1|/path/to/key
# environment|user@host|role
devnet|ubuntu@1.2.3.4|node
devnet|ubuntu@5.6.7.8|nameserver-ns1
```
| Field | Description |
|-------|-------------|
| `environment` | Cluster name (`devnet`, `testnet`) |
| `user@host` | SSH credentials |
| `password` | SSH password |
| `role` | `node` or `nameserver-ns1`, `nameserver-ns2`, etc. |
| `ssh_key` | Optional path to SSH private key |
SSH keys are resolved from rootwallet (`rw vault ssh get <host>/<user> --priv`).
Blank lines and lines starting with `#` are ignored.

233
docs/ORAMAOS_DEPLOYMENT.md Normal file
View File

@ -0,0 +1,233 @@
# OramaOS Deployment Guide
OramaOS is a custom minimal Linux image built with Buildroot. It replaces the standard Ubuntu-based node deployment for mainnet, devnet, and testnet environments. Sandbox clusters remain on Ubuntu for development convenience.
## What is OramaOS?
OramaOS is a locked-down operating system designed specifically for Orama node operators. Key properties:
- **No SSH, no shell** — operators cannot access the filesystem or run commands on the machine
- **LUKS full-disk encryption** — the data partition is encrypted; the key is split via Shamir's Secret Sharing across peer nodes
- **Read-only rootfs** — the OS image uses SquashFS with dm-verity integrity verification
- **A/B partition updates** — signed OS images are applied atomically with automatic rollback on failure
- **Service sandboxing** — each service runs in its own Linux namespace with seccomp syscall filtering
- **Signed binaries** — all updates are cryptographically signed with the Orama rootwallet
## Architecture
```
Partition Layout:
/dev/sda1 — ESP (EFI System Partition, systemd-boot)
/dev/sda2 — rootfs-A (SquashFS, read-only, dm-verity)
/dev/sda3 — rootfs-B (standby, for A/B updates)
/dev/sda4 — data (LUKS2 encrypted, ext4)
Boot Flow:
systemd-boot → dm-verity rootfs → orama-agent → WireGuard → services
```
The **orama-agent** is the only root process. It manages:
- Boot sequence and LUKS key reconstruction
- WireGuard tunnel setup
- Service lifecycle (start, stop, restart in sandboxed namespaces)
- Command reception from the Gateway over WireGuard
- OS updates (download, verify signature, A/B swap, reboot)
## Enrollment Flow
OramaOS nodes join the cluster through an enrollment process (different from the Ubuntu `orama node install` flow):
### Step 1: Flash OramaOS to VPS
Download the OramaOS image and flash it to your VPS:
```bash
# Download image (URL provided upon acceptance)
wget https://releases.orama.network/oramaos-v1.0.0-amd64.qcow2
# Flash to VPS (provider-specific — Hetzner, Vultr, etc.)
# Most providers support uploading custom images via their dashboard
```
### Step 2: First Boot — Enrollment Mode
On first boot, the agent:
1. Generates a random 8-character registration code
2. Starts a temporary HTTP server on port 9999
3. Opens an outbound WebSocket to the Gateway
4. Waits for enrollment to complete
The registration code is displayed on the VPS console (if available) and served at `http://<vps-ip>:9999/`.
### Step 3: Run Enrollment from CLI
On your local machine (where you have the `orama` CLI and rootwallet):
```bash
# Generate an invite token on any existing cluster node
orama node invite --expiry 24h
# Enroll the OramaOS node
orama node enroll --node-ip <vps-public-ip> --token <invite-token> --gateway <gateway-url>
```
The enrollment command:
1. Fetches the registration code from the node (port 9999)
2. Sends the code + invite token to the Gateway
3. Gateway validates everything, assigns a WireGuard IP, and pushes config to the node
4. Node configures WireGuard, formats the LUKS-encrypted data partition
5. LUKS key is split via Shamir and distributed to peer vault-guardians
6. Services start in sandboxed namespaces
7. Port 9999 closes permanently
### Step 4: Verify
```bash
# Check the node is online and healthy
orama monitor report --env <env>
```
## Genesis Node
The first OramaOS node in a cluster is the **genesis node**. It has a special boot path because there are no peers yet for Shamir key distribution:
1. Genesis generates a LUKS key and encrypts the data partition
2. The LUKS key is encrypted with a rootwallet-derived key and stored on the unencrypted rootfs
3. On reboot (before enough peers exist), the operator must manually unlock:
```bash
orama node unlock --genesis --node-ip <wg-ip>
```
This command:
1. Fetches the encrypted genesis key from the node
2. Decrypts it using the rootwallet (`rw decrypt`)
3. Sends the decrypted LUKS key to the agent over WireGuard
Once 5+ peers have joined, the genesis node distributes Shamir shares to peers, deletes the local encrypted key, and transitions to normal Shamir-based unlock. After this transition, `orama node unlock` is no longer needed.
## Normal Reboot (Shamir Unlock)
When an enrolled OramaOS node reboots:
1. Agent starts, brings up WireGuard
2. Contacts peer vault-guardians over WireGuard
3. Fetches K Shamir shares (K = threshold, typically `max(3, N/3)`)
4. Reconstructs LUKS key via Lagrange interpolation over GF(256)
5. Decrypts and mounts data partition
6. Starts all services
7. Zeros key from memory
If not enough peers are available, the agent enters a degraded "waiting for peers" state and retries with exponential backoff (1s, 2s, 4s, 8s, 16s, max 5 retries per cycle).
## Node Management
Since OramaOS has no SSH, all management happens through the Gateway API:
```bash
# Check node status
curl "https://gateway.example.com/v1/node/status?node_id=<id>"
# Send a command (e.g., restart a service)
curl -X POST "https://gateway.example.com/v1/node/command?node_id=<id>" \
-H "Content-Type: application/json" \
-d '{"action":"restart","service":"rqlite"}'
# View logs
curl "https://gateway.example.com/v1/node/logs?node_id=<id>&service=gateway&lines=100"
# Graceful node departure
curl -X POST "https://gateway.example.com/v1/node/leave" \
-H "Content-Type: application/json" \
-d '{"node_id":"<id>"}'
```
The Gateway proxies these requests to the agent over WireGuard (port 9998). The agent is never directly accessible from the public internet.
## OS Updates
OramaOS uses an A/B partition scheme for atomic, rollback-safe updates:
1. Agent periodically checks for new versions
2. Downloads the signed image (P2P over WireGuard between nodes)
3. Verifies the rootwallet EVM signature against the embedded public key
4. Writes to the standby partition (if running from A, writes to B)
5. Sets systemd-boot to boot from B with `tries_left=3`
6. Reboots
7. If B boots successfully (agent starts, WG connects, services healthy): marks B as "good"
8. If B fails 3 times: systemd-boot automatically falls back to A
No operator intervention is needed for updates. Failed updates are automatically rolled back.
## Service Sandboxing
Each service on OramaOS runs in an isolated environment:
- **Mount namespace** — each service only sees its own data directory as writable; everything else is read-only
- **UTS namespace** — isolated hostname
- **Dedicated UID/GID** — each service runs as a different user (not root)
- **Seccomp filtering** — per-service syscall allowlist (initially in audit mode, then enforce mode)
Services and their sandbox profiles:
| Service | Writable Path | Extra Syscalls |
|---------|--------------|----------------|
| RQLite | `/opt/orama/.orama/data/rqlite` | fsync, fdatasync (Raft + SQLite WAL) |
| Olric | `/opt/orama/.orama/data/olric` | sendmmsg, recvmmsg (gossip) |
| IPFS | `/opt/orama/.orama/data/ipfs` | sendfile, splice (data transfer) |
| Gateway | `/opt/orama/.orama/data/gateway` | sendfile, splice (HTTP) |
| CoreDNS | `/opt/orama/.orama/data/coredns` | sendmmsg, recvmmsg (DNS) |
## OramaOS vs Ubuntu Deployment
| Feature | Ubuntu | OramaOS |
|---------|--------|---------|
| SSH access | Yes | No |
| Shell access | Yes | No |
| Disk encryption | No | LUKS2 (Shamir) |
| OS updates | Manual (`orama node upgrade`) | Automatic (signed, A/B) |
| Service isolation | systemd only | Namespaces + seccomp |
| Rootfs integrity | None | dm-verity |
| Binary signing | Optional | Required |
| Operator data access | Full | None |
| Environments | All (including sandbox) | Mainnet, devnet, testnet |
## Cleaning / Factory Reset
OramaOS nodes cannot be cleaned with the standard `orama node clean` command (no SSH access). Instead:
- **Graceful departure:** `orama node leave` via the Gateway API — stops services, redistributes Shamir shares, removes WG peer
- **Factory reset:** Reflash the OramaOS image on the VPS via the hosting provider's dashboard
- **Data is unrecoverable:** Since the LUKS key is distributed across peers, reflashing destroys all data permanently
## Troubleshooting
### Node stuck in enrollment mode
The node boots but enrollment never completes.
**Check:** Can you reach `http://<vps-ip>:9999/` from your machine? If not, the VPS firewall may be blocking port 9999.
**Fix:** Ensure port 9999 is open in the VPS provider's firewall. OramaOS opens it automatically via its internal firewall, but external provider firewalls (Hetzner, AWS security groups) must be configured separately.
### LUKS unlock fails (not enough peers)
After reboot, the node can't reconstruct its LUKS key.
**Check:** How many peer nodes are online? The node needs at least K peers (threshold) to be reachable over WireGuard.
**Fix:** Ensure enough cluster nodes are online. If this is the genesis node and fewer than 5 peers exist, use:
```bash
orama node unlock --genesis --node-ip <wg-ip>
```
### Update failed, node rolled back
The node applied an update but reverted to the previous version.
**Check:** The agent logs will show why the new partition failed to boot (accessible via `GET /v1/node/logs?service=agent`).
**Common causes:** Corrupted download (signature verification should catch this), hardware issue, or incompatible configuration.
### Services not starting after reboot
The node rebooted and LUKS unlocked, but services are unhealthy.
**Check:** `GET /v1/node/status` — which services are down?
**Fix:** Try restarting the specific service via `POST /v1/node/command` with `{"action":"restart","service":"<name>"}`. If the issue persists, check service logs.

208
docs/SANDBOX.md Normal file
View File

@ -0,0 +1,208 @@
# Sandbox: Ephemeral Hetzner Cloud Clusters
Spin up temporary 5-node Orama clusters on Hetzner Cloud for development and testing. Total cost: ~€0.04/hour.
## Quick Start
```bash
# One-time setup (API key, domain, floating IPs, SSH key)
orama sandbox setup
# Create a cluster (~5 minutes)
orama sandbox create --name my-feature
# Check health
orama sandbox status
# SSH into a node
orama sandbox ssh 1
# Deploy code changes
orama sandbox rollout
# Tear it down
orama sandbox destroy
```
## Prerequisites
### 1. Hetzner Cloud Account
Create a project at [console.hetzner.cloud](https://console.hetzner.cloud) and generate an API token with read/write permissions under **Security > API Tokens**.
### 2. Domain with Glue Records
You need a domain (or subdomain) that points to Hetzner Floating IPs. The `orama sandbox setup` wizard will guide you through this.
**Example:** Using `sbx.dbrs.space`
At your domain registrar:
1. Create glue records (Personal DNS Servers):
- `ns1.sbx.dbrs.space` → `<floating-ip-1>`
- `ns2.sbx.dbrs.space` → `<floating-ip-2>`
2. Set custom nameservers for `sbx.dbrs.space`:
- `ns1.sbx.dbrs.space`
- `ns2.sbx.dbrs.space`
DNS propagation can take up to 48 hours.
### 3. Binary Archive
Build the binary archive before creating a cluster:
```bash
orama build
```
This creates `/tmp/orama-<version>-linux-amd64.tar.gz` with all pre-compiled binaries.
## Setup
Run the interactive setup wizard:
```bash
orama sandbox setup
```
This will:
1. Prompt for your Hetzner API token and validate it
2. Ask for your sandbox domain
3. Create or reuse 2 Hetzner Floating IPs (~$0.005/hr each)
4. Create a firewall with sandbox rules
5. Create a rootwallet SSH entry (`sandbox/root`) if it doesn't exist
6. Upload the wallet-derived public key to Hetzner
7. Display DNS configuration instructions
Config is saved to `~/.orama/sandbox.yaml`.
## Commands
### `orama sandbox create [--name <name>]`
Creates a new 5-node cluster. If `--name` is omitted, a random name is generated (e.g., "swift-falcon").
**Cluster layout:**
- Nodes 1-2: Nameservers (CoreDNS + Caddy + all services)
- Nodes 3-5: Regular nodes (all services except CoreDNS)
**Phases:**
1. Provision 5 CX22 servers on Hetzner (parallel, ~90s)
2. Assign floating IPs to nameserver nodes (~10s)
3. Upload binary archive to all nodes (parallel, ~60s)
4. Install genesis node + generate invite tokens (~120s)
5. Join remaining 4 nodes (serial with health checks, ~180s)
6. Verify cluster health (~15s)
**One sandbox at a time.** Since the floating IPs are shared, only one sandbox can own the nameservers. Destroy the active sandbox before creating a new one.
### `orama sandbox destroy [--name <name>] [--force]`
Tears down a cluster:
1. Unassigns floating IPs
2. Deletes all 5 servers (parallel)
3. Removes state file
Use `--force` to skip confirmation.
### `orama sandbox list`
Lists all sandboxes with their status. Also checks Hetzner for orphaned servers that don't have a corresponding state file.
### `orama sandbox status [--name <name>]`
Shows per-node health including:
- Service status (active/inactive)
- RQLite role (Leader/Follower)
- Cluster summary (commit index, voter count)
### `orama sandbox rollout [--name <name>]`
Deploys code changes:
1. Uses the latest binary archive from `/tmp/` (run `orama build` first)
2. Pushes to all nodes
3. Rolling upgrade: followers first, leader last, 15s between nodes
### `orama sandbox ssh <node-number>`
Opens an interactive SSH session to a sandbox node (1-5).
```bash
orama sandbox ssh 1 # SSH into node 1 (genesis/ns1)
orama sandbox ssh 3 # SSH into node 3 (regular node)
```
## Architecture
### Floating IPs
Hetzner Floating IPs are persistent IPv4 addresses that can be reassigned between servers. They solve the DNS chicken-and-egg problem:
- Glue records at the registrar point to 2 Floating IPs (configured once)
- Each new sandbox assigns the Floating IPs to its nameserver nodes
- DNS works instantly — no propagation delay between clusters
### SSH Authentication
Sandbox uses a rootwallet-derived SSH key (`sandbox/root` vault entry), the same mechanism as production. The wallet must be unlocked (`rw unlock`) before running sandbox commands that use SSH. The public key is uploaded to Hetzner during setup and injected into every server at creation time.
### Server Naming
Servers: `sbx-<name>-<N>` (e.g., `sbx-swift-falcon-1` through `sbx-swift-falcon-5`)
### State Files
Sandbox state is stored at `~/.orama/sandboxes/<name>.yaml`. This tracks server IDs, IPs, roles, and cluster status.
## Cost
| Resource | Cost | Qty | Total |
|----------|------|-----|-------|
| CX22 (2 vCPU, 4GB) | €0.006/hr | 5 | €0.03/hr |
| Floating IPv4 | €0.005/hr | 2 | €0.01/hr |
| **Total** | | | **~€0.04/hr** |
Servers are billed per hour. Floating IPs are billed as long as they exist (even unassigned). Destroy the sandbox when not in use to save on server costs.
## Troubleshooting
### "sandbox not configured"
Run `orama sandbox setup` first.
### "no binary archive found"
Run `orama build` to create the binary archive.
### "sandbox X is already active"
Only one sandbox can be active at a time. Destroy it first:
```bash
orama sandbox destroy --name <name>
```
### Server creation fails
Check:
- Hetzner API token is valid and has read/write permissions
- You haven't hit Hetzner's server limit (default: 10 per project)
- The selected location has CX22 capacity
### Genesis install fails
SSH into the node to debug:
```bash
orama sandbox ssh 1
journalctl -u orama-node -f
```
The sandbox will be left in "error" state. You can destroy and recreate it.
### DNS not resolving
1. Verify glue records are configured at your registrar
2. Check propagation: `dig NS sbx.dbrs.space @8.8.8.8`
3. Propagation can take 24-48 hours for new domains
### Orphaned servers
If `orama sandbox list` shows orphaned servers, delete them manually at [console.hetzner.cloud](https://console.hetzner.cloud). Sandbox servers are labeled `orama-sandbox=<name>` for easy identification.

194
docs/SECURITY.md Normal file
View File

@ -0,0 +1,194 @@
# Security Hardening
This document describes all security measures applied to the Orama Network, covering both Phase 1 (service hardening on existing Ubuntu nodes) and Phase 2 (OramaOS locked-down image).
## Phase 1: Service Hardening
These measures apply to all nodes (Ubuntu and OramaOS).
### Network Isolation
**CIDR Validation (Step 1.1)**
- WireGuard subnet restricted to `10.0.0.0/24` across all components: firewall rules, rate limiter, auth module, and WireGuard PostUp/PostDown iptables rules
- Prevents other tenants on shared VPS providers from bypassing the firewall via overlapping `10.x.x.x` ranges
**IPv6 Disabled (Step 1.2)**
- IPv6 disabled system-wide via sysctl: `net.ipv6.conf.all.disable_ipv6=1`
- Prevents services bound to `0.0.0.0` from being reachable via IPv6 (which had no firewall rules)
### Authentication
**Internal Endpoint Auth (Step 1.3)**
- `/v1/internal/wg/peers` and `/v1/internal/wg/peer/remove` now require cluster secret validation
- Peer removal additionally validates the request originates from a WireGuard subnet IP
**RQLite Authentication (Step 1.7)**
- RQLite runs with `-auth` flag pointing to a credentials file
- All RQLite HTTP requests include `Authorization: Basic <base64>` headers
- Credentials generated at cluster genesis, distributed to joining nodes via join response
- Both the central RQLite client wrapper and the standalone CoreDNS RQLite client send auth
**Olric Gossip Encryption (Step 1.8)**
- Olric memberlist uses a 32-byte encryption key for all gossip traffic
- Key generated at genesis, distributed via join response
- Prevents rogue nodes from joining the gossip ring and poisoning caches
- Note: encryption is all-or-nothing (coordinated restart required when enabling)
**IPFS Cluster TrustedPeers (Step 1.9)**
- IPFS Cluster `TrustedPeers` populated with actual cluster peer IDs (was `["*"]`)
- New peers added to TrustedPeers on all existing nodes during join
- Prevents unauthorized peers from controlling IPFS pinning
**Vault V1 Auth Enforcement (Step 1.14)**
- V1 push/pull endpoints require a valid session token when vault-guardian is configured
- Previously, auth was optional for backward compatibility — any WG peer could read/overwrite Shamir shares
### Token & Key Storage
**Refresh Token Hashing (Step 1.5)**
- Refresh tokens stored as SHA-256 hashes in RQLite (never plaintext)
- On lookup: hash the incoming token, query by hash
- On revocation: hash before revoking (both single-token and by-subject)
- Existing tokens invalidated on upgrade (users re-authenticate)
**API Key Hashing (Step 1.6)**
- API keys stored as HMAC-SHA256 hashes using a server-side secret
- HMAC secret generated at cluster genesis, stored in `~/.orama/secrets/api-key-hmac-secret`
- On lookup: compute HMAC, query by hash — fast enough for every request (unlike bcrypt)
- In-memory cache uses raw key as cache key (never persisted)
- During rolling upgrade: dual lookup (HMAC first, then raw as fallback) until all nodes upgraded
**TURN Secret Encryption (Step 1.15)**
- TURN shared secrets encrypted at rest in RQLite using AES-256-GCM
- Encryption key derived via HKDF from the cluster secret with purpose string `"turn-encryption"`
### TLS & Transport
**InsecureSkipVerify Fix (Step 1.10)**
- During node join, TLS verification uses TOFU (Trust On First Use)
- Invite token output includes the CA certificate fingerprint (SHA-256)
- Joining node verifies the server cert fingerprint matches before proceeding
- After join: CA cert stored locally for future connections
**WebSocket Origin Validation (Step 1.4)**
- All WebSocket upgraders validate the `Origin` header against the node's configured domain
- Non-browser clients (no Origin header) are still allowed
- Prevents cross-site WebSocket hijacking attacks
### Process Isolation
**Dedicated User (Step 1.11)**
- All services run as the `orama` user (not root)
- Caddy and CoreDNS get `AmbientCapabilities=CAP_NET_BIND_SERVICE` for ports 80/443 and 53
- WireGuard stays as root (kernel netlink requires it)
- vault-guardian already had proper hardening
**systemd Hardening (Step 1.12)**
- All service units include:
```ini
ProtectSystem=strict
ProtectHome=yes
NoNewPrivileges=yes
PrivateDevices=yes
ProtectKernelTunables=yes
ProtectKernelModules=yes
RestrictNamespaces=yes
ReadWritePaths=/opt/orama/.orama
```
- Applied to both template files (`pkg/environments/templates/`) and hardcoded unit generators (`pkg/environments/production/services.go`)
### Supply Chain
**Binary Signing (Step 1.13)**
- Build archives include `manifest.sig` — a rootwallet EVM signature of the manifest hash
- During install, the signature is verified against the embedded Orama public key
- Unsigned or tampered archives are rejected
## Phase 2: OramaOS
These measures apply only to OramaOS nodes (mainnet, devnet, testnet).
### Immutable OS
- **Read-only rootfs** — SquashFS with dm-verity integrity verification
- **No shell** — `/bin/sh` symlinked to `/bin/false`, no bash/ash/ssh
- **No SSH** — OpenSSH not included in the image
- **Minimal packages** — only what's needed for systemd, cryptsetup, and the agent
### Full-Disk Encryption
- **LUKS2** with AES-XTS-Plain64 on the data partition
- **Shamir's Secret Sharing** over GF(256) — LUKS key split across peer vault-guardians
- **Adaptive threshold** — K = max(3, N/3) where N is the number of peers
- **Key zeroing** — LUKS key wiped from memory immediately after use
- **Malicious share detection** — fetch K+1 shares when possible, verify consistency
### Service Sandboxing
Each service runs in isolated Linux namespaces:
- **CLONE_NEWNS** — mount namespace (filesystem isolation)
- **CLONE_NEWUTS** — hostname namespace
- **Dedicated UID/GID** — each service has its own user
- **Seccomp filtering** — per-service syscall allowlist
Note: CLONE_NEWPID is intentionally omitted — it makes services PID 1 in their namespace, which changes signal semantics (SIGTERM ignored by default for PID 1).
### Signed Updates
- A/B partition scheme with systemd-boot and boot counting (`tries_left=3`)
- All updates signed with rootwallet EVM signature (secp256k1 + keccak256)
- Signer address: `0xb5d8a496c8b2412990d7D467E17727fdF5954afC`
- P2P distribution over WireGuard between nodes
- Automatic rollback on 3 consecutive boot failures
### Zero Operator Access
- Operators cannot read data on the machine (LUKS encrypted, no shell)
- Management only through Gateway API → agent over WireGuard
- All commands are logged and auditable
- No root access, no console access, no file system access
## Rollout Strategy
### Phase 1 Batches
```
Batch 1 (zero-risk, no restart):
- CIDR fix
- IPv6 disable
- Internal endpoint auth
- WebSocket origin check
Batch 2 (medium-risk, restart needed):
- Hash refresh tokens
- Hash API keys
- Binary signing
- Vault V1 auth enforcement
- TURN secret encryption
Batch 3 (high-risk, coordinated rollout):
- RQLite auth (followers first, leader last)
- Olric encryption (simultaneous restart)
- IPFS Cluster TrustedPeers
Batch 4 (infrastructure changes):
- InsecureSkipVerify fix
- Dedicated user
- systemd hardening
```
### Phase 2
1. Build and test OramaOS image in QEMU
2. Deploy to sandbox cluster alongside Ubuntu nodes
3. Verify interop and stability
4. Gradual migration: testnet → devnet → mainnet (one node at a time, maintaining Raft quorum)
## Verification
All changes verified on sandbox cluster before production deployment:
- `make test` — all unit tests pass
- `orama monitor report --env sandbox` — full cluster health
- Manual endpoint testing (e.g., curl without auth → 401)
- Security-specific checks (IPv6 listeners, RQLite auth, binary signatures)

8
go.mod
View File

@ -20,6 +20,10 @@ require (
github.com/miekg/dns v1.1.70
github.com/multiformats/go-multiaddr v0.16.0
github.com/olric-data/olric v0.7.0
github.com/pion/interceptor v0.1.40
github.com/pion/rtcp v1.2.15
github.com/pion/turn/v4 v4.0.2
github.com/pion/webrtc/v4 v4.1.2
github.com/rqlite/gorqlite v0.0.0-20250609141355-ac86a4a1c9a8
github.com/spf13/cobra v1.10.2
github.com/stretchr/testify v1.11.1
@ -123,11 +127,9 @@ require (
github.com/pion/dtls/v2 v2.2.12 // indirect
github.com/pion/dtls/v3 v3.0.6 // indirect
github.com/pion/ice/v4 v4.0.10 // indirect
github.com/pion/interceptor v0.1.40 // indirect
github.com/pion/logging v0.2.3 // indirect
github.com/pion/mdns/v2 v2.0.7 // indirect
github.com/pion/randutil v0.1.0 // indirect
github.com/pion/rtcp v1.2.15 // indirect
github.com/pion/rtp v1.8.19 // indirect
github.com/pion/sctp v1.8.39 // indirect
github.com/pion/sdp/v3 v3.0.13 // indirect
@ -136,8 +138,6 @@ require (
github.com/pion/stun/v3 v3.0.0 // indirect
github.com/pion/transport/v2 v2.2.10 // indirect
github.com/pion/transport/v3 v3.0.7 // indirect
github.com/pion/turn/v4 v4.0.2 // indirect
github.com/pion/webrtc/v4 v4.1.2 // indirect
github.com/pkg/errors v0.9.1 // indirect
github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
github.com/prometheus/client_golang v1.23.0 // indirect

4
migrations/019_invalidate_plaintext_refresh_tokens.sql Normal file
View File

@ -0,0 +1,4 @@
-- Invalidate all existing refresh tokens.
-- Tokens were stored in plaintext; the application now stores SHA-256 hashes.
-- Users will need to re-authenticate (tokens have 30-day expiry anyway).
UPDATE refresh_tokens SET revoked_at = datetime('now') WHERE revoked_at IS NULL;

318
pkg/cli/build/archive.go Normal file
View File

@ -0,0 +1,318 @@
package build
import (
"archive/tar"
"compress/gzip"
"crypto/sha256"
"encoding/hex"
"encoding/json"
"fmt"
"io"
"net/http"
"os"
"os/exec"
"path/filepath"
"strings"
"time"
)
// Manifest describes the contents of a binary archive.
type Manifest struct {
Version string `json:"version"`
Commit string `json:"commit"`
Date string `json:"date"`
Arch string `json:"arch"`
Checksums map[string]string `json:"checksums"` // filename -> sha256
}
// generateManifest creates the manifest with SHA256 checksums of all binaries.
func (b *Builder) generateManifest() (*Manifest, error) {
m := &Manifest{
Version: b.version,
Commit: b.commit,
Date: b.date,
Arch: b.flags.Arch,
Checksums: make(map[string]string),
}
entries, err := os.ReadDir(b.binDir)
if err != nil {
return nil, err
}
for _, entry := range entries {
if entry.IsDir() {
continue
}
path := filepath.Join(b.binDir, entry.Name())
hash, err := sha256File(path)
if err != nil {
return nil, fmt.Errorf("failed to hash %s: %w", entry.Name(), err)
}
m.Checksums[entry.Name()] = hash
}
return m, nil
}
// createArchive creates the tar.gz archive from the build directory.
func (b *Builder) createArchive(outputPath string, manifest *Manifest) error {
fmt.Printf("\nCreating archive: %s\n", outputPath)
// Write manifest.json to tmpDir
manifestData, err := json.MarshalIndent(manifest, "", " ")
if err != nil {
return err
}
if err := os.WriteFile(filepath.Join(b.tmpDir, "manifest.json"), manifestData, 0644); err != nil {
return err
}
// Create output file
f, err := os.Create(outputPath)
if err != nil {
return err
}
defer f.Close()
gw := gzip.NewWriter(f)
defer gw.Close()
tw := tar.NewWriter(gw)
defer tw.Close()
// Add bin/ directory
if err := addDirToTar(tw, b.binDir, "bin"); err != nil {
return err
}
// Add systemd/ directory
systemdDir := filepath.Join(b.tmpDir, "systemd")
if _, err := os.Stat(systemdDir); err == nil {
if err := addDirToTar(tw, systemdDir, "systemd"); err != nil {
return err
}
}
// Add packages/ directory if it exists
packagesDir := filepath.Join(b.tmpDir, "packages")
if _, err := os.Stat(packagesDir); err == nil {
if err := addDirToTar(tw, packagesDir, "packages"); err != nil {
return err
}
}
// Add manifest.json
if err := addFileToTar(tw, filepath.Join(b.tmpDir, "manifest.json"), "manifest.json"); err != nil {
return err
}
// Add manifest.sig if it exists (created by --sign)
sigPath := filepath.Join(b.tmpDir, "manifest.sig")
if _, err := os.Stat(sigPath); err == nil {
if err := addFileToTar(tw, sigPath, "manifest.sig"); err != nil {
return err
}
}
// Print summary
fmt.Printf(" bin/: %d binaries\n", len(manifest.Checksums))
fmt.Printf(" systemd/: namespace templates\n")
fmt.Printf(" manifest: v%s (%s) linux/%s\n", manifest.Version, manifest.Commit, manifest.Arch)
info, err := f.Stat()
if err == nil {
fmt.Printf(" size: %s\n", formatBytes(info.Size()))
}
return nil
}
// signManifest signs the manifest hash using rootwallet CLI.
// Produces manifest.sig containing the hex-encoded EVM signature.
func (b *Builder) signManifest(manifest *Manifest) error {
fmt.Printf("\nSigning manifest with rootwallet...\n")
// Serialize manifest deterministically (compact JSON, sorted keys via json.Marshal)
manifestData, err := json.Marshal(manifest)
if err != nil {
return fmt.Errorf("failed to marshal manifest: %w", err)
}
// Hash the manifest JSON
hash := sha256.Sum256(manifestData)
hashHex := hex.EncodeToString(hash[:])
// Call rw sign <hash> --chain evm
cmd := exec.Command("rw", "sign", hashHex, "--chain", "evm")
var stdout, stderr strings.Builder
cmd.Stdout = &stdout
cmd.Stderr = &stderr
if err := cmd.Run(); err != nil {
return fmt.Errorf("rw sign failed: %w\n%s", err, stderr.String())
}
signature := strings.TrimSpace(stdout.String())
if signature == "" {
return fmt.Errorf("rw sign produced empty signature")
}
// Write signature file
sigPath := filepath.Join(b.tmpDir, "manifest.sig")
if err := os.WriteFile(sigPath, []byte(signature), 0644); err != nil {
return fmt.Errorf("failed to write manifest.sig: %w", err)
}
fmt.Printf(" Manifest signed (SHA256: %s...)\n", hashHex[:16])
return nil
}
// addDirToTar adds all files in a directory to the tar archive under the given prefix.
func addDirToTar(tw *tar.Writer, srcDir, prefix string) error {
return filepath.Walk(srcDir, func(path string, info os.FileInfo, err error) error {
if err != nil {
return err
}
// Calculate relative path
relPath, err := filepath.Rel(srcDir, path)
if err != nil {
return err
}
tarPath := filepath.Join(prefix, relPath)
if info.IsDir() {
header := &tar.Header{
Name: tarPath + "/",
Mode: 0755,
Typeflag: tar.TypeDir,
}
return tw.WriteHeader(header)
}
return addFileToTar(tw, path, tarPath)
})
}
// addFileToTar adds a single file to the tar archive.
func addFileToTar(tw *tar.Writer, srcPath, tarPath string) error {
f, err := os.Open(srcPath)
if err != nil {
return err
}
defer f.Close()
info, err := f.Stat()
if err != nil {
return err
}
header := &tar.Header{
Name: tarPath,
Size: info.Size(),
Mode: int64(info.Mode()),
}
if err := tw.WriteHeader(header); err != nil {
return err
}
_, err = io.Copy(tw, f)
return err
}
// sha256File computes the SHA256 hash of a file.
func sha256File(path string) (string, error) {
f, err := os.Open(path)
if err != nil {
return "", err
}
defer f.Close()
h := sha256.New()
if _, err := io.Copy(h, f); err != nil {
return "", err
}
return hex.EncodeToString(h.Sum(nil)), nil
}
// downloadFile downloads a URL to a local file path.
func downloadFile(url, destPath string) error {
client := &http.Client{Timeout: 5 * time.Minute}
resp, err := client.Get(url)
if err != nil {
return fmt.Errorf("failed to download %s: %w", url, err)
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
return fmt.Errorf("download %s returned status %d", url, resp.StatusCode)
}
f, err := os.Create(destPath)
if err != nil {
return err
}
defer f.Close()
_, err = io.Copy(f, resp.Body)
return err
}
// extractFileFromTarball extracts a single file from a tar.gz archive.
func extractFileFromTarball(tarPath, targetFile, destPath string) error {
f, err := os.Open(tarPath)
if err != nil {
return err
}
defer f.Close()
gr, err := gzip.NewReader(f)
if err != nil {
return err
}
defer gr.Close()
tr := tar.NewReader(gr)
for {
header, err := tr.Next()
if err == io.EOF {
break
}
if err != nil {
return err
}
// Match the target file (strip leading ./ if present)
name := strings.TrimPrefix(header.Name, "./")
if name == targetFile {
out, err := os.OpenFile(destPath, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0755)
if err != nil {
return err
}
defer out.Close()
if _, err := io.Copy(out, tr); err != nil {
return err
}
return nil
}
}
return fmt.Errorf("file %s not found in archive %s", targetFile, tarPath)
}
// formatBytes formats bytes into a human-readable string.
func formatBytes(b int64) string {
const unit = 1024
if b < unit {
return fmt.Sprintf("%d B", b)
}
div, exp := int64(unit), 0
for n := b / unit; n >= unit; n /= unit {
div *= unit
exp++
}
return fmt.Sprintf("%.1f %cB", float64(b)/float64(div), "KMGTPE"[exp])
}

829
pkg/cli/build/builder.go Normal file
View File

@ -0,0 +1,829 @@
package build
import (
"fmt"
"os"
"os/exec"
"path/filepath"
"strings"
"time"
"github.com/DeBrosOfficial/network/pkg/constants"
)
// oramaBinary defines a binary to cross-compile from the project source.
type oramaBinary struct {
Name string // output binary name
Package string // Go package path relative to project root
// Extra ldflags beyond the standard ones
ExtraLDFlags string
}
// Builder orchestrates the entire build process.
type Builder struct {
flags *Flags
projectDir string
tmpDir string
binDir string
version string
commit string
date string
}
// NewBuilder creates a new Builder.
func NewBuilder(flags *Flags) *Builder {
return &Builder{flags: flags}
}
// Build runs the full build pipeline.
func (b *Builder) Build() error {
start := time.Now()
// Find project root
projectDir, err := findProjectRoot()
if err != nil {
return err
}
b.projectDir = projectDir
// Read version from Makefile or use "dev"
b.version = b.readVersion()
b.commit = b.readCommit()
b.date = time.Now().UTC().Format("2006-01-02T15:04:05Z")
// Create temp build directory
b.tmpDir, err = os.MkdirTemp("", "orama-build-*")
if err != nil {
return fmt.Errorf("failed to create temp dir: %w", err)
}
defer os.RemoveAll(b.tmpDir)
b.binDir = filepath.Join(b.tmpDir, "bin")
if err := os.MkdirAll(b.binDir, 0755); err != nil {
return fmt.Errorf("failed to create bin dir: %w", err)
}
fmt.Printf("Building orama %s for linux/%s\n", b.version, b.flags.Arch)
fmt.Printf("Project: %s\n\n", b.projectDir)
// Step 1: Cross-compile Orama binaries
if err := b.buildOramaBinaries(); err != nil {
return fmt.Errorf("failed to build orama binaries: %w", err)
}
// Step 2: Cross-compile Vault Guardian (Zig)
if err := b.buildVaultGuardian(); err != nil {
return fmt.Errorf("failed to build vault-guardian: %w", err)
}
// Step 3: Cross-compile Olric
if err := b.buildOlric(); err != nil {
return fmt.Errorf("failed to build olric: %w", err)
}
// Step 4: Cross-compile IPFS Cluster
if err := b.buildIPFSCluster(); err != nil {
return fmt.Errorf("failed to build ipfs-cluster: %w", err)
}
// Step 5: Build CoreDNS with RQLite plugin
if err := b.buildCoreDNS(); err != nil {
return fmt.Errorf("failed to build coredns: %w", err)
}
// Step 6: Build Caddy with Orama DNS module
if err := b.buildCaddy(); err != nil {
return fmt.Errorf("failed to build caddy: %w", err)
}
// Step 7: Download pre-built IPFS Kubo
if err := b.downloadIPFS(); err != nil {
return fmt.Errorf("failed to download ipfs: %w", err)
}
// Step 8: Download pre-built RQLite
if err := b.downloadRQLite(); err != nil {
return fmt.Errorf("failed to download rqlite: %w", err)
}
// Step 9: Copy systemd templates
if err := b.copySystemdTemplates(); err != nil {
return fmt.Errorf("failed to copy systemd templates: %w", err)
}
// Step 10: Generate manifest
manifest, err := b.generateManifest()
if err != nil {
return fmt.Errorf("failed to generate manifest: %w", err)
}
// Step 11: Sign manifest (optional)
if b.flags.Sign {
if err := b.signManifest(manifest); err != nil {
return fmt.Errorf("failed to sign manifest: %w", err)
}
}
// Step 12: Create archive
outputPath := b.flags.Output
if outputPath == "" {
outputPath = fmt.Sprintf("/tmp/orama-%s-linux-%s.tar.gz", b.version, b.flags.Arch)
}
if err := b.createArchive(outputPath, manifest); err != nil {
return fmt.Errorf("failed to create archive: %w", err)
}
elapsed := time.Since(start).Round(time.Second)
fmt.Printf("\nBuild complete in %s\n", elapsed)
fmt.Printf("Archive: %s\n", outputPath)
return nil
}
func (b *Builder) buildOramaBinaries() error {
fmt.Println("[1/8] Cross-compiling Orama binaries...")
ldflags := fmt.Sprintf("-s -w -X 'main.version=%s' -X 'main.commit=%s' -X 'main.date=%s'",
b.version, b.commit, b.date)
gatewayLDFlags := fmt.Sprintf("%s -X 'github.com/DeBrosOfficial/network/pkg/gateway.BuildVersion=%s' -X 'github.com/DeBrosOfficial/network/pkg/gateway.BuildCommit=%s' -X 'github.com/DeBrosOfficial/network/pkg/gateway.BuildTime=%s'",
ldflags, b.version, b.commit, b.date)
binaries := []oramaBinary{
{Name: "orama", Package: "./cmd/cli/"},
{Name: "orama-node", Package: "./cmd/node/"},
{Name: "gateway", Package: "./cmd/gateway/", ExtraLDFlags: gatewayLDFlags},
{Name: "identity", Package: "./cmd/identity/"},
{Name: "sfu", Package: "./cmd/sfu/"},
{Name: "turn", Package: "./cmd/turn/"},
}
for _, bin := range binaries {
flags := ldflags
if bin.ExtraLDFlags != "" {
flags = bin.ExtraLDFlags
}
output := filepath.Join(b.binDir, bin.Name)
cmd := exec.Command("go", "build",
"-ldflags", flags,
"-trimpath",
"-o", output,
bin.Package)
cmd.Dir = b.projectDir
cmd.Env = b.crossEnv()
cmd.Stdout = os.Stdout
cmd.Stderr = os.Stderr
if b.flags.Verbose {
fmt.Printf(" go build -o %s %s\n", bin.Name, bin.Package)
}
if err := cmd.Run(); err != nil {
return fmt.Errorf("failed to build %s: %w", bin.Name, err)
}
fmt.Printf(" ✓ %s\n", bin.Name)
}
return nil
}
func (b *Builder) buildVaultGuardian() error {
fmt.Println("[2/8] Cross-compiling Vault Guardian (Zig)...")
// Ensure zig is available
if _, err := exec.LookPath("zig"); err != nil {
return fmt.Errorf("zig not found in PATH — install from https://ziglang.org/download/")
}
// Vault source is sibling to orama project
vaultDir := filepath.Join(b.projectDir, "..", "orama-vault")
if _, err := os.Stat(filepath.Join(vaultDir, "build.zig")); err != nil {
return fmt.Errorf("vault source not found at %s — expected orama-vault as sibling directory: %w", vaultDir, err)
}
// Map Go arch to Zig target triple
var zigTarget string
switch b.flags.Arch {
case "amd64":
zigTarget = "x86_64-linux-musl"
case "arm64":
zigTarget = "aarch64-linux-musl"
default:
return fmt.Errorf("unsupported architecture for vault: %s", b.flags.Arch)
}
if b.flags.Verbose {
fmt.Printf(" zig build -Dtarget=%s -Doptimize=ReleaseSafe\n", zigTarget)
}
cmd := exec.Command("zig", "build",
fmt.Sprintf("-Dtarget=%s", zigTarget),
"-Doptimize=ReleaseSafe")
cmd.Dir = vaultDir
cmd.Stdout = os.Stdout
cmd.Stderr = os.Stderr
if err := cmd.Run(); err != nil {
return fmt.Errorf("zig build failed: %w", err)
}
// Copy output binary to build bin dir
src := filepath.Join(vaultDir, "zig-out", "bin", "vault-guardian")
dst := filepath.Join(b.binDir, "vault-guardian")
if err := copyFile(src, dst); err != nil {
return fmt.Errorf("failed to copy vault-guardian binary: %w", err)
}
fmt.Println(" ✓ vault-guardian")
return nil
}
// copyFile copies a file from src to dst, preserving executable permissions.
func copyFile(src, dst string) error {
srcFile, err := os.Open(src)
if err != nil {
return err
}
defer srcFile.Close()
dstFile, err := os.OpenFile(dst, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0755)
if err != nil {
return err
}
defer dstFile.Close()
if _, err := srcFile.WriteTo(dstFile); err != nil {
return err
}
return nil
}
func (b *Builder) buildOlric() error {
fmt.Printf("[3/8] Cross-compiling Olric %s...\n", constants.OlricVersion)
// go install doesn't support cross-compilation with GOBIN set,
// so we create a temporary module and use go build -o instead.
tmpDir, err := os.MkdirTemp("", "olric-build-*")
if err != nil {
return fmt.Errorf("create temp dir: %w", err)
}
defer os.RemoveAll(tmpDir)
modInit := exec.Command("go", "mod", "init", "olric-build")
modInit.Dir = tmpDir
modInit.Stderr = os.Stderr
if err := modInit.Run(); err != nil {
return fmt.Errorf("go mod init: %w", err)
}
modGet := exec.Command("go", "get",
fmt.Sprintf("github.com/olric-data/olric/cmd/olric-server@%s", constants.OlricVersion))
modGet.Dir = tmpDir
modGet.Env = append(os.Environ(),
"GOPROXY=https://proxy.golang.org|direct",
"GONOSUMDB=*")
modGet.Stderr = os.Stderr
if err := modGet.Run(); err != nil {
return fmt.Errorf("go get olric: %w", err)
}
cmd := exec.Command("go", "build",
"-ldflags", "-s -w",
"-trimpath",
"-o", filepath.Join(b.binDir, "olric-server"),
fmt.Sprintf("github.com/olric-data/olric/cmd/olric-server"))
cmd.Dir = tmpDir
cmd.Env = append(b.crossEnv(),
"GOPROXY=https://proxy.golang.org|direct",
"GONOSUMDB=*")
cmd.Stdout = os.Stdout
cmd.Stderr = os.Stderr
if err := cmd.Run(); err != nil {
return err
}
fmt.Println(" ✓ olric-server")
return nil
}
func (b *Builder) buildIPFSCluster() error {
fmt.Printf("[4/8] Cross-compiling IPFS Cluster %s...\n", constants.IPFSClusterVersion)
tmpDir, err := os.MkdirTemp("", "ipfs-cluster-build-*")
if err != nil {
return fmt.Errorf("create temp dir: %w", err)
}
defer os.RemoveAll(tmpDir)
modInit := exec.Command("go", "mod", "init", "ipfs-cluster-build")
modInit.Dir = tmpDir
modInit.Stderr = os.Stderr
if err := modInit.Run(); err != nil {
return fmt.Errorf("go mod init: %w", err)
}
modGet := exec.Command("go", "get",
fmt.Sprintf("github.com/ipfs-cluster/ipfs-cluster/cmd/ipfs-cluster-service@%s", constants.IPFSClusterVersion))
modGet.Dir = tmpDir
modGet.Env = append(os.Environ(),
"GOPROXY=https://proxy.golang.org|direct",
"GONOSUMDB=*")
modGet.Stderr = os.Stderr
if err := modGet.Run(); err != nil {
return fmt.Errorf("go get ipfs-cluster: %w", err)
}
cmd := exec.Command("go", "build",
"-ldflags", "-s -w",
"-trimpath",
"-o", filepath.Join(b.binDir, "ipfs-cluster-service"),
"github.com/ipfs-cluster/ipfs-cluster/cmd/ipfs-cluster-service")
cmd.Dir = tmpDir
cmd.Env = append(b.crossEnv(),
"GOPROXY=https://proxy.golang.org|direct",
"GONOSUMDB=*")
cmd.Stdout = os.Stdout
cmd.Stderr = os.Stderr
if err := cmd.Run(); err != nil {
return err
}
fmt.Println(" ✓ ipfs-cluster-service")
return nil
}
func (b *Builder) buildCoreDNS() error {
fmt.Printf("[5/8] Building CoreDNS %s with RQLite plugin...\n", constants.CoreDNSVersion)
buildDir := filepath.Join(b.tmpDir, "coredns-build")
// Clone CoreDNS
fmt.Println(" Cloning CoreDNS...")
cmd := exec.Command("git", "clone", "--depth", "1",
"--branch", "v"+constants.CoreDNSVersion,
"https://github.com/coredns/coredns.git", buildDir)
cmd.Stdout = os.Stdout
cmd.Stderr = os.Stderr
if err := cmd.Run(); err != nil {
return fmt.Errorf("failed to clone coredns: %w", err)
}
// Copy RQLite plugin from local source
pluginSrc := filepath.Join(b.projectDir, "pkg", "coredns", "rqlite")
pluginDst := filepath.Join(buildDir, "plugin", "rqlite")
if err := os.MkdirAll(pluginDst, 0755); err != nil {
return err
}
entries, err := os.ReadDir(pluginSrc)
if err != nil {
return fmt.Errorf("failed to read rqlite plugin source at %s: %w", pluginSrc, err)
}
for _, entry := range entries {
if entry.IsDir() || filepath.Ext(entry.Name()) != ".go" {
continue
}
data, err := os.ReadFile(filepath.Join(pluginSrc, entry.Name()))
if err != nil {
return err
}
if err := os.WriteFile(filepath.Join(pluginDst, entry.Name()), data, 0644); err != nil {
return err
}
}
// Write plugin.cfg (same as build-linux-coredns.sh)
pluginCfg := `metadata:metadata
cancel:cancel
tls:tls
reload:reload
nsid:nsid
bufsize:bufsize
root:root
bind:bind
debug:debug
trace:trace
ready:ready
health:health
pprof:pprof
prometheus:metrics
errors:errors
log:log
dnstap:dnstap
local:local
dns64:dns64
acl:acl
any:any
chaos:chaos
loadbalance:loadbalance
cache:cache
rewrite:rewrite
header:header
dnssec:dnssec
autopath:autopath
minimal:minimal
template:template
transfer:transfer
hosts:hosts
file:file
auto:auto
secondary:secondary
loop:loop
forward:forward
grpc:grpc
erratic:erratic
whoami:whoami
on:github.com/coredns/caddy/onevent
sign:sign
view:view
rqlite:rqlite
`
if err := os.WriteFile(filepath.Join(buildDir, "plugin.cfg"), []byte(pluginCfg), 0644); err != nil {
return err
}
// Add dependencies
fmt.Println(" Adding dependencies...")
goPath := os.Getenv("PATH")
baseEnv := append(os.Environ(),
"PATH="+goPath,
"GOPROXY=https://proxy.golang.org|direct",
"GONOSUMDB=*")
for _, dep := range []string{"github.com/miekg/dns@latest", "go.uber.org/zap@latest"} {
cmd := exec.Command("go", "get", dep)
cmd.Dir = buildDir
cmd.Env = baseEnv
cmd.Stderr = os.Stderr
if err := cmd.Run(); err != nil {
return fmt.Errorf("failed to get %s: %w", dep, err)
}
}
cmd = exec.Command("go", "mod", "tidy")
cmd.Dir = buildDir
cmd.Env = baseEnv
cmd.Stderr = os.Stderr
if err := cmd.Run(); err != nil {
return fmt.Errorf("go mod tidy failed: %w", err)
}
// Generate plugin code
fmt.Println(" Generating plugin code...")
cmd = exec.Command("go", "generate")
cmd.Dir = buildDir
cmd.Env = baseEnv
cmd.Stderr = os.Stderr
if err := cmd.Run(); err != nil {
return fmt.Errorf("go generate failed: %w", err)
}
// Cross-compile
fmt.Println(" Building binary...")
cmd = exec.Command("go", "build",
"-ldflags", "-s -w",
"-trimpath",
"-o", filepath.Join(b.binDir, "coredns"))
cmd.Dir = buildDir
cmd.Env = append(baseEnv,
"GOOS=linux",
fmt.Sprintf("GOARCH=%s", b.flags.Arch),
"CGO_ENABLED=0")
cmd.Stdout = os.Stdout
cmd.Stderr = os.Stderr
if err := cmd.Run(); err != nil {
return fmt.Errorf("build failed: %w", err)
}
fmt.Println(" ✓ coredns")
return nil
}
func (b *Builder) buildCaddy() error {
fmt.Printf("[6/8] Building Caddy %s with Orama DNS module...\n", constants.CaddyVersion)
// Ensure xcaddy is available
if _, err := exec.LookPath("xcaddy"); err != nil {
return fmt.Errorf("xcaddy not found in PATH — install with: go install github.com/caddyserver/xcaddy/cmd/xcaddy@latest")
}
moduleDir := filepath.Join(b.tmpDir, "caddy-dns-orama")
if err := os.MkdirAll(moduleDir, 0755); err != nil {
return err
}
// Write go.mod
goMod := fmt.Sprintf(`module github.com/DeBrosOfficial/caddy-dns-orama
go 1.22
require (
github.com/caddyserver/caddy/v2 v2.%s
github.com/libdns/libdns v1.1.0
)
`, constants.CaddyVersion[2:])
if err := os.WriteFile(filepath.Join(moduleDir, "go.mod"), []byte(goMod), 0644); err != nil {
return err
}
// Write provider.go — read from the caddy installer's generated code
// We inline the same provider code used by the VPS-side caddy installer
providerCode := generateCaddyProviderCode()
if err := os.WriteFile(filepath.Join(moduleDir, "provider.go"), []byte(providerCode), 0644); err != nil {
return err
}
// go mod tidy
cmd := exec.Command("go", "mod", "tidy")
cmd.Dir = moduleDir
cmd.Env = append(os.Environ(),
"GOPROXY=https://proxy.golang.org|direct",
"GONOSUMDB=*")
cmd.Stderr = os.Stderr
if err := cmd.Run(); err != nil {
return fmt.Errorf("go mod tidy failed: %w", err)
}
// Build with xcaddy
fmt.Println(" Building binary...")
cmd = exec.Command("xcaddy", "build",
"v"+constants.CaddyVersion,
"--with", "github.com/DeBrosOfficial/caddy-dns-orama="+moduleDir,
"--output", filepath.Join(b.binDir, "caddy"))
cmd.Env = append(os.Environ(),
"GOOS=linux",
fmt.Sprintf("GOARCH=%s", b.flags.Arch),
"GOPROXY=https://proxy.golang.org|direct",
"GONOSUMDB=*")
cmd.Stdout = os.Stdout
cmd.Stderr = os.Stderr
if err := cmd.Run(); err != nil {
return fmt.Errorf("xcaddy build failed: %w", err)
}
fmt.Println(" ✓ caddy")
return nil
}
func (b *Builder) downloadIPFS() error {
fmt.Printf("[7/8] Downloading IPFS Kubo %s...\n", constants.IPFSKuboVersion)
arch := b.flags.Arch
tarball := fmt.Sprintf("kubo_%s_linux-%s.tar.gz", constants.IPFSKuboVersion, arch)
url := fmt.Sprintf("https://dist.ipfs.tech/kubo/%s/%s", constants.IPFSKuboVersion, tarball)
tarPath := filepath.Join(b.tmpDir, tarball)
if err := downloadFile(url, tarPath); err != nil {
return err
}
// Extract ipfs binary from kubo/ipfs
if err := extractFileFromTarball(tarPath, "kubo/ipfs", filepath.Join(b.binDir, "ipfs")); err != nil {
return err
}
fmt.Println(" ✓ ipfs")
return nil
}
func (b *Builder) downloadRQLite() error {
fmt.Printf("[8/8] Downloading RQLite %s...\n", constants.RQLiteVersion)
arch := b.flags.Arch
tarball := fmt.Sprintf("rqlite-v%s-linux-%s.tar.gz", constants.RQLiteVersion, arch)
url := fmt.Sprintf("https://github.com/rqlite/rqlite/releases/download/v%s/%s", constants.RQLiteVersion, tarball)
tarPath := filepath.Join(b.tmpDir, tarball)
if err := downloadFile(url, tarPath); err != nil {
return err
}
// Extract rqlited binary
extractDir := fmt.Sprintf("rqlite-v%s-linux-%s", constants.RQLiteVersion, arch)
if err := extractFileFromTarball(tarPath, extractDir+"/rqlited", filepath.Join(b.binDir, "rqlited")); err != nil {
return err
}
fmt.Println(" ✓ rqlited")
return nil
}
func (b *Builder) copySystemdTemplates() error {
systemdSrc := filepath.Join(b.projectDir, "systemd")
systemdDst := filepath.Join(b.tmpDir, "systemd")
if err := os.MkdirAll(systemdDst, 0755); err != nil {
return err
}
entries, err := os.ReadDir(systemdSrc)
if err != nil {
return fmt.Errorf("failed to read systemd dir: %w", err)
}
for _, entry := range entries {
if entry.IsDir() || !strings.HasSuffix(entry.Name(), ".service") {
continue
}
data, err := os.ReadFile(filepath.Join(systemdSrc, entry.Name()))
if err != nil {
return err
}
if err := os.WriteFile(filepath.Join(systemdDst, entry.Name()), data, 0644); err != nil {
return err
}
}
return nil
}
// crossEnv returns the environment for cross-compilation.
func (b *Builder) crossEnv() []string {
return append(os.Environ(),
"GOOS=linux",
fmt.Sprintf("GOARCH=%s", b.flags.Arch),
"CGO_ENABLED=0")
}
func (b *Builder) readVersion() string {
// Try to read from Makefile
data, err := os.ReadFile(filepath.Join(b.projectDir, "Makefile"))
if err != nil {
return "dev"
}
for _, line := range strings.Split(string(data), "\n") {
line = strings.TrimSpace(line)
if strings.HasPrefix(line, "VERSION") {
parts := strings.SplitN(line, ":=", 2)
if len(parts) == 2 {
return strings.TrimSpace(parts[1])
}
}
}
return "dev"
}
func (b *Builder) readCommit() string {
cmd := exec.Command("git", "rev-parse", "--short", "HEAD")
cmd.Dir = b.projectDir
out, err := cmd.Output()
if err != nil {
return "unknown"
}
return strings.TrimSpace(string(out))
}
// generateCaddyProviderCode returns the Caddy DNS provider Go source.
// This is the same code used by the VPS-side caddy installer.
func generateCaddyProviderCode() string {
return `// Package orama implements a DNS provider for Caddy that uses the Orama Network
// gateway's internal ACME API for DNS-01 challenge validation.
package orama
import (
"bytes"
"context"
"encoding/json"
"fmt"
"net/http"
"time"
"github.com/caddyserver/caddy/v2"
"github.com/caddyserver/caddy/v2/caddyconfig/caddyfile"
"github.com/libdns/libdns"
)
func init() {
caddy.RegisterModule(Provider{})
}
// Provider wraps the Orama DNS provider for Caddy.
type Provider struct {
// Endpoint is the URL of the Orama gateway's ACME API
// Default: http://localhost:6001/v1/internal/acme
Endpoint string ` + "`json:\"endpoint,omitempty\"`" + `
}
// CaddyModule returns the Caddy module information.
func (Provider) CaddyModule() caddy.ModuleInfo {
return caddy.ModuleInfo{
ID: "dns.providers.orama",
New: func() caddy.Module { return new(Provider) },
}
}
// Provision sets up the module.
func (p *Provider) Provision(ctx caddy.Context) error {
if p.Endpoint == "" {
p.Endpoint = "http://localhost:6001/v1/internal/acme"
}
return nil
}
// UnmarshalCaddyfile parses the Caddyfile configuration.
func (p *Provider) UnmarshalCaddyfile(d *caddyfile.Dispenser) error {
for d.Next() {
for d.NextBlock(0) {
switch d.Val() {
case "endpoint":
if !d.NextArg() {
return d.ArgErr()
}
p.Endpoint = d.Val()
default:
return d.Errf("unrecognized option: %s", d.Val())
}
}
}
return nil
}
// AppendRecords adds records to the zone.
func (p *Provider) AppendRecords(ctx context.Context, zone string, records []libdns.Record) ([]libdns.Record, error) {
var added []libdns.Record
for _, rec := range records {
rr := rec.RR()
if rr.Type != "TXT" {
continue
}
fqdn := rr.Name + "." + zone
payload := map[string]string{"fqdn": fqdn, "value": rr.Data}
body, err := json.Marshal(payload)
if err != nil {
return added, fmt.Errorf("failed to marshal request: %w", err)
}
req, err := http.NewRequestWithContext(ctx, "POST", p.Endpoint+"/present", bytes.NewReader(body))
if err != nil {
return added, fmt.Errorf("failed to create request: %w", err)
}
req.Header.Set("Content-Type", "application/json")
client := &http.Client{Timeout: 30 * time.Second}
resp, err := client.Do(req)
if err != nil {
return added, fmt.Errorf("failed to present challenge: %w", err)
}
resp.Body.Close()
if resp.StatusCode != http.StatusOK {
return added, fmt.Errorf("present failed with status %d", resp.StatusCode)
}
added = append(added, rec)
}
return added, nil
}
// DeleteRecords removes records from the zone.
func (p *Provider) DeleteRecords(ctx context.Context, zone string, records []libdns.Record) ([]libdns.Record, error) {
var deleted []libdns.Record
for _, rec := range records {
rr := rec.RR()
if rr.Type != "TXT" {
continue
}
fqdn := rr.Name + "." + zone
payload := map[string]string{"fqdn": fqdn, "value": rr.Data}
body, err := json.Marshal(payload)
if err != nil {
return deleted, fmt.Errorf("failed to marshal request: %w", err)
}
req, err := http.NewRequestWithContext(ctx, "POST", p.Endpoint+"/cleanup", bytes.NewReader(body))
if err != nil {
return deleted, fmt.Errorf("failed to create request: %w", err)
}
req.Header.Set("Content-Type", "application/json")
client := &http.Client{Timeout: 30 * time.Second}
resp, err := client.Do(req)
if err != nil {
return deleted, fmt.Errorf("failed to cleanup challenge: %w", err)
}
resp.Body.Close()
if resp.StatusCode != http.StatusOK {
return deleted, fmt.Errorf("cleanup failed with status %d", resp.StatusCode)
}
deleted = append(deleted, rec)
}
return deleted, nil
}
// GetRecords returns the records in the zone. Not used for ACME.
func (p *Provider) GetRecords(ctx context.Context, zone string) ([]libdns.Record, error) {
return nil, nil
}
// SetRecords sets the records in the zone. Not used for ACME.
func (p *Provider) SetRecords(ctx context.Context, zone string, records []libdns.Record) ([]libdns.Record, error) {
return nil, nil
}
// Interface guards
var (
_ caddy.Module = (*Provider)(nil)
_ caddy.Provisioner = (*Provider)(nil)
_ caddyfile.Unmarshaler = (*Provider)(nil)
_ libdns.RecordAppender = (*Provider)(nil)
_ libdns.RecordDeleter = (*Provider)(nil)
_ libdns.RecordGetter = (*Provider)(nil)
_ libdns.RecordSetter = (*Provider)(nil)
)
`
}

82
pkg/cli/build/command.go Normal file
View File

@ -0,0 +1,82 @@
package build
import (
"flag"
"fmt"
"os"
"path/filepath"
"runtime"
)
// Flags represents build command flags.
type Flags struct {
Arch string
Output string
Verbose bool
Sign bool // Sign the archive manifest with rootwallet
}
// Handle is the entry point for the build command.
func Handle(args []string) {
flags, err := parseFlags(args)
if err != nil {
if err == flag.ErrHelp {
return
}
fmt.Fprintf(os.Stderr, "Error: %v\n", err)
os.Exit(1)
}
b := NewBuilder(flags)
if err := b.Build(); err != nil {
fmt.Fprintf(os.Stderr, "Error: %v\n", err)
os.Exit(1)
}
}
func parseFlags(args []string) (*Flags, error) {
fs := flag.NewFlagSet("build", flag.ContinueOnError)
fs.SetOutput(os.Stderr)
flags := &Flags{}
fs.StringVar(&flags.Arch, "arch", "amd64", "Target architecture (amd64, arm64)")
fs.StringVar(&flags.Output, "output", "", "Output archive path (default: /tmp/orama-<version>-linux-<arch>.tar.gz)")
fs.BoolVar(&flags.Verbose, "verbose", false, "Verbose output")
fs.BoolVar(&flags.Sign, "sign", false, "Sign the manifest with rootwallet (requires rw in PATH)")
if err := fs.Parse(args); err != nil {
return nil, err
}
return flags, nil
}
// findProjectRoot walks up from the current directory looking for go.mod.
func findProjectRoot() (string, error) {
dir, err := os.Getwd()
if err != nil {
return "", err
}
for {
if _, err := os.Stat(filepath.Join(dir, "go.mod")); err == nil {
// Verify it's the network project
if _, err := os.Stat(filepath.Join(dir, "cmd", "cli")); err == nil {
return dir, nil
}
}
parent := filepath.Dir(dir)
if parent == dir {
break
}
dir = parent
}
return "", fmt.Errorf("could not find project root (no go.mod with cmd/cli found)")
}
// detectHostArch returns the host architecture in Go naming convention.
func detectHostArch() string {
return runtime.GOARCH
}

2
pkg/cli/cluster/commands.go
View File

@ -61,7 +61,7 @@ func ShowHelp() {
fmt.Printf("Subcommands:\n")
fmt.Printf(" status - Show cluster node status (RQLite + Olric)\n")
fmt.Printf(" Options:\n")
fmt.Printf(" --all - SSH into all nodes from remote-nodes.conf (TODO)\n")
fmt.Printf(" --all - SSH into all nodes from nodes.conf (TODO)\n")
fmt.Printf(" health - Run cluster health checks\n")
fmt.Printf(" rqlite <subcommand> - RQLite-specific commands\n")
fmt.Printf(" status - Show detailed Raft state for local node\n")

24
pkg/cli/cmd/buildcmd/build.go Normal file
View File

@ -0,0 +1,24 @@
package buildcmd
import (
"github.com/DeBrosOfficial/network/pkg/cli/build"
"github.com/spf13/cobra"
)
// Cmd is the top-level build command.
var Cmd = &cobra.Command{
Use: "build",
Short: "Build pre-compiled binary archive for deployment",
Long: `Cross-compile all Orama binaries and dependencies for Linux,
then package them into a deployment archive. The archive includes:
- Orama binaries (CLI, node, gateway, identity, SFU, TURN)
- Olric, IPFS Kubo, IPFS Cluster, RQLite, CoreDNS, Caddy
- Systemd namespace templates
- manifest.json with checksums
The resulting archive can be pushed to nodes with 'orama node push'.`,
Run: func(cmd *cobra.Command, args []string) {
build.Handle(args)
},
DisableFlagParsing: true,
}

2
pkg/cli/cmd/monitorcmd/monitor.go
View File

@ -34,7 +34,7 @@ func init() {
Cmd.PersistentFlags().StringVar(&flagEnv, "env", "", "Environment: devnet, testnet, mainnet (required)")
Cmd.PersistentFlags().BoolVar(&flagJSON, "json", false, "Machine-readable JSON output")
Cmd.PersistentFlags().StringVar(&flagNode, "node", "", "Filter to specific node host/IP")
Cmd.PersistentFlags().StringVar(&flagConfig, "config", "scripts/remote-nodes.conf", "Path to remote-nodes.conf")
Cmd.PersistentFlags().StringVar(&flagConfig, "config", "scripts/nodes.conf", "Path to nodes.conf")
Cmd.MarkPersistentFlagRequired("env")
Cmd.AddCommand(liveCmd)

25
pkg/cli/cmd/node/clean.go Normal file
View File

@ -0,0 +1,25 @@
package node
import (
"github.com/DeBrosOfficial/network/pkg/cli/production/clean"
"github.com/spf13/cobra"
)
var cleanCmd = &cobra.Command{
Use: "clean",
Short: "Clean (wipe) remote nodes for reinstallation",
Long: `Remove all Orama data, services, and configuration from remote nodes.
Anyone relay keys at /var/lib/anon/ are preserved.
This is a DESTRUCTIVE operation. Use --force to skip confirmation.
Examples:
orama node clean --env testnet # Clean all testnet nodes
orama node clean --env testnet --node 1.2.3.4 # Clean specific node
orama node clean --env testnet --nuclear # Also remove shared binaries
orama node clean --env testnet --force # Skip confirmation`,
Run: func(cmd *cobra.Command, args []string) {
clean.Handle(args)
},
DisableFlagParsing: true,
}

26
pkg/cli/cmd/node/enroll.go Normal file
View File

@ -0,0 +1,26 @@
package node
import (
"github.com/DeBrosOfficial/network/pkg/cli/production/enroll"
"github.com/spf13/cobra"
)
var enrollCmd = &cobra.Command{
Use: "enroll",
Short: "Enroll an OramaOS node into the cluster",
Long: `Enroll a freshly booted OramaOS node into the cluster.
The OramaOS node displays a registration code on port 9999. Provide this code
along with an invite token to complete enrollment. The Gateway pushes cluster
configuration (WireGuard, secrets, peer list) to the node.
Usage:
orama node enroll --node-ip <ip> --code <code> --token <invite-token> --env <environment>
The node must be reachable over the public internet on port 9999 (enrollment only).
After enrollment, port 9999 is permanently closed and all communication goes over WireGuard.`,
Run: func(cmd *cobra.Command, args []string) {
enroll.Handle(args)
},
DisableFlagParsing: true,
}

6
pkg/cli/cmd/node/node.go
View File

@ -26,4 +26,10 @@ func init() {
Cmd.AddCommand(migrateCmd)
Cmd.AddCommand(doctorCmd)
Cmd.AddCommand(reportCmd)
Cmd.AddCommand(pushCmd)
Cmd.AddCommand(rolloutCmd)
Cmd.AddCommand(cleanCmd)
Cmd.AddCommand(recoverRaftCmd)
Cmd.AddCommand(enrollCmd)
Cmd.AddCommand(unlockCmd)
}

24
pkg/cli/cmd/node/push.go Normal file
View File

@ -0,0 +1,24 @@
package node
import (
"github.com/DeBrosOfficial/network/pkg/cli/production/push"
"github.com/spf13/cobra"
)
var pushCmd = &cobra.Command{
Use: "push",
Short: "Push binary archive to remote nodes",
Long: `Upload a pre-built binary archive to remote nodes.
By default, uses fanout distribution: uploads to one hub node,
then distributes to all others via server-to-server SCP.
Examples:
orama node push --env devnet # Fanout to all devnet nodes
orama node push --env testnet --node 1.2.3.4 # Single node
orama node push --env testnet --direct # Sequential upload to each node`,
Run: func(cmd *cobra.Command, args []string) {
push.Handle(args)
},
DisableFlagParsing: true,
}

31
pkg/cli/cmd/node/recover_raft.go Normal file
View File

@ -0,0 +1,31 @@
package node
import (
"github.com/DeBrosOfficial/network/pkg/cli/production/recover"
"github.com/spf13/cobra"
)
var recoverRaftCmd = &cobra.Command{
Use: "recover-raft",
Short: "Recover RQLite cluster from split-brain",
Long: `Recover the RQLite Raft cluster from split-brain failure.
Strategy:
1. Stop orama-node on ALL nodes simultaneously
2. Backup and delete raft/ on non-leader nodes
3. Start leader node, wait for Leader state
4. Start remaining nodes in batches
5. Verify cluster health
The --leader flag must point to the node with the highest commit index.
This is a DESTRUCTIVE operation. Use --force to skip confirmation.
Examples:
orama node recover-raft --env testnet --leader 1.2.3.4
orama node recover-raft --env devnet --leader 1.2.3.4 --force`,
Run: func(cmd *cobra.Command, args []string) {
recover.Handle(args)
},
DisableFlagParsing: true,
}

22
pkg/cli/cmd/node/rollout.go Normal file
View File

@ -0,0 +1,22 @@
package node
import (
"github.com/DeBrosOfficial/network/pkg/cli/production/rollout"
"github.com/spf13/cobra"
)
var rolloutCmd = &cobra.Command{
Use: "rollout",
Short: "Build, push, and rolling upgrade all nodes in an environment",
Long: `Full deployment pipeline: build binary archive locally, push to all nodes,
then perform a rolling upgrade (one node at a time).
Examples:
orama node rollout --env testnet # Full: build + push + rolling upgrade
orama node rollout --env testnet --no-build # Skip build, use existing archive
orama node rollout --env testnet --yes # Skip confirmation`,
Run: func(cmd *cobra.Command, args []string) {
rollout.Handle(args)
},
DisableFlagParsing: true,
}

26
pkg/cli/cmd/node/unlock.go Normal file
View File

@ -0,0 +1,26 @@
package node
import (
"github.com/DeBrosOfficial/network/pkg/cli/production/unlock"
"github.com/spf13/cobra"
)
var unlockCmd = &cobra.Command{
Use: "unlock",
Short: "Unlock an OramaOS genesis node",
Long: `Manually unlock a genesis OramaOS node that cannot reconstruct its LUKS key
via Shamir shares (not enough peers online).
This is only needed for the genesis node before enough peers have joined for
Shamir-based unlock. Once 5+ peers exist, the genesis node transitions to
normal Shamir unlock and this command is no longer needed.
Usage:
orama node unlock --genesis --node-ip <wg-ip>
The node must be reachable over WireGuard on port 9998.`,
Run: func(cmd *cobra.Command, args []string) {
unlock.Handle(args)
},
DisableFlagParsing: true,
}

140
pkg/cli/cmd/sandboxcmd/sandbox.go Normal file
View File

@ -0,0 +1,140 @@
package sandboxcmd
import (
"fmt"
"os"
"github.com/DeBrosOfficial/network/pkg/cli/sandbox"
"github.com/spf13/cobra"
)
// Cmd is the root command for sandbox operations.
var Cmd = &cobra.Command{
Use: "sandbox",
Short: "Manage ephemeral Hetzner Cloud clusters for testing",
Long: `Spin up temporary 5-node Orama clusters on Hetzner Cloud for development and testing.
Setup (one-time):
orama sandbox setup
Usage:
orama sandbox create [--name <name>] Create a new 5-node cluster
orama sandbox destroy [--name <name>] Tear down a cluster
orama sandbox list List active sandboxes
orama sandbox status [--name <name>] Show cluster health
orama sandbox rollout [--name <name>] Build + push + rolling upgrade
orama sandbox ssh <node-number> SSH into a sandbox node (1-5)
orama sandbox reset Delete all infra and config to start fresh`,
}
var setupCmd = &cobra.Command{
Use: "setup",
Short: "Interactive setup: Hetzner API key, domain, floating IPs, SSH key",
RunE: func(cmd *cobra.Command, args []string) error {
return sandbox.Setup()
},
}
var createCmd = &cobra.Command{
Use: "create",
Short: "Create a new 5-node sandbox cluster (~5 min)",
RunE: func(cmd *cobra.Command, args []string) error {
name, _ := cmd.Flags().GetString("name")
return sandbox.Create(name)
},
}
var destroyCmd = &cobra.Command{
Use: "destroy",
Short: "Destroy a sandbox cluster and release resources",
RunE: func(cmd *cobra.Command, args []string) error {
name, _ := cmd.Flags().GetString("name")
force, _ := cmd.Flags().GetBool("force")
return sandbox.Destroy(name, force)
},
}
var listCmd = &cobra.Command{
Use: "list",
Short: "List active sandbox clusters",
RunE: func(cmd *cobra.Command, args []string) error {
return sandbox.List()
},
}
var statusCmd = &cobra.Command{
Use: "status",
Short: "Show cluster health report",
RunE: func(cmd *cobra.Command, args []string) error {
name, _ := cmd.Flags().GetString("name")
return sandbox.Status(name)
},
}
var rolloutCmd = &cobra.Command{
Use: "rollout",
Short: "Build + push + rolling upgrade to sandbox cluster",
RunE: func(cmd *cobra.Command, args []string) error {
name, _ := cmd.Flags().GetString("name")
anyoneClient, _ := cmd.Flags().GetBool("anyone-client")
return sandbox.Rollout(name, sandbox.RolloutFlags{
AnyoneClient: anyoneClient,
})
},
}
var resetCmd = &cobra.Command{
Use: "reset",
Short: "Delete all sandbox infrastructure and config to start fresh",
Long: `Deletes floating IPs, firewall, and SSH key from Hetzner Cloud,
then removes the local config (~/.orama/sandbox.yaml) and SSH keys.
Use this when you need to switch datacenter locations (floating IPs are
location-bound) or to completely start over with sandbox setup.`,
RunE: func(cmd *cobra.Command, args []string) error {
return sandbox.Reset()
},
}
var sshCmd = &cobra.Command{
Use: "ssh <node-number>",
Short: "SSH into a sandbox node (1-5)",
Args: cobra.ExactArgs(1),
RunE: func(cmd *cobra.Command, args []string) error {
name, _ := cmd.Flags().GetString("name")
var nodeNum int
if _, err := fmt.Sscanf(args[0], "%d", &nodeNum); err != nil {
fmt.Fprintf(os.Stderr, "Invalid node number: %s (expected 1-5)\n", args[0])
os.Exit(1)
}
return sandbox.SSHInto(name, nodeNum)
},
}
func init() {
// create flags
createCmd.Flags().String("name", "", "Sandbox name (random if not specified)")
// destroy flags
destroyCmd.Flags().String("name", "", "Sandbox name (uses active if not specified)")
destroyCmd.Flags().Bool("force", false, "Skip confirmation")
// status flags
statusCmd.Flags().String("name", "", "Sandbox name (uses active if not specified)")
// rollout flags
rolloutCmd.Flags().String("name", "", "Sandbox name (uses active if not specified)")
rolloutCmd.Flags().Bool("anyone-client", false, "Enable Anyone client (SOCKS5 proxy) on all nodes")
// ssh flags
sshCmd.Flags().String("name", "", "Sandbox name (uses active if not specified)")
Cmd.AddCommand(setupCmd)
Cmd.AddCommand(createCmd)
Cmd.AddCommand(destroyCmd)
Cmd.AddCommand(listCmd)
Cmd.AddCommand(statusCmd)
Cmd.AddCommand(rolloutCmd)
Cmd.AddCommand(sshCmd)
Cmd.AddCommand(resetCmd)
}

18
pkg/cli/environment.go
View File

@ -26,16 +26,16 @@ type EnvironmentConfig struct {
// Default environments
var DefaultEnvironments = []Environment{
{
Name: "production",
Name: "sandbox",
GatewayURL: "https://dbrs.space",
Description: "Production network (dbrs.space)",
IsActive: false,
Description: "Sandbox cluster (dbrs.space)",
IsActive: true,
},
{
Name: "devnet",
GatewayURL: "https://orama-devnet.network",
Description: "Development network (testnet)",
IsActive: true,
Description: "Development network",
IsActive: false,
},
{
Name: "testnet",
@ -65,7 +65,7 @@ func LoadEnvironmentConfig() (*EnvironmentConfig, error) {
if _, err := os.Stat(path); os.IsNotExist(err) {
return &EnvironmentConfig{
Environments: DefaultEnvironments,
ActiveEnvironment: "devnet",
ActiveEnvironment: "sandbox",
}, nil
}
@ -120,9 +120,9 @@ func GetActiveEnvironment() (*Environment, error) {
}
}
// Fallback to devnet if active environment not found
// Fallback to sandbox if active environment not found
for _, env := range envConfig.Environments {
if env.Name == "devnet" {
if env.Name == "sandbox" {
return &env, nil
}
}
@ -184,7 +184,7 @@ func InitializeEnvironments() error {
envConfig := &EnvironmentConfig{
Environments: DefaultEnvironments,
ActiveEnvironment: "devnet",
ActiveEnvironment: "sandbox",
}
return SaveEnvironmentConfig(envConfig)

11
pkg/cli/inspect_command.go
View File

@ -9,6 +9,7 @@ import (
"strings"
"time"
"github.com/DeBrosOfficial/network/pkg/cli/remotessh"
"github.com/DeBrosOfficial/network/pkg/inspector"
// Import checks package so init() registers the checkers
_ "github.com/DeBrosOfficial/network/pkg/inspector/checks"
@ -49,7 +50,7 @@ func HandleInspectCommand(args []string) {
fs := flag.NewFlagSet("inspect", flag.ExitOnError)
configPath := fs.String("config", "scripts/remote-nodes.conf", "Path to remote-nodes.conf")
configPath := fs.String("config", "scripts/nodes.conf", "Path to nodes.conf")
env := fs.String("env", "", "Environment to inspect (devnet, testnet)")
subsystem := fs.String("subsystem", "all", "Subsystem to inspect (rqlite,olric,ipfs,dns,wg,system,network,anyone,all)")
format := fs.String("format", "table", "Output format (table, json)")
@ -98,6 +99,14 @@ func HandleInspectCommand(args []string) {
os.Exit(1)
}
// Prepare wallet-derived SSH keys
cleanup, err := remotessh.PrepareNodeKeys(nodes)
if err != nil {
fmt.Fprintf(os.Stderr, "Error preparing SSH keys: %v\n", err)
os.Exit(1)
}
defer cleanup()
// Parse subsystems
var subsystems []string
if *subsystem != "all" {

79
pkg/cli/monitor/collector.go
View File

@ -8,6 +8,8 @@ import (
"time"
"github.com/DeBrosOfficial/network/pkg/cli/production/report"
"github.com/DeBrosOfficial/network/pkg/cli/remotessh"
"github.com/DeBrosOfficial/network/pkg/cli/sandbox"
"github.com/DeBrosOfficial/network/pkg/inspector"
)
@ -22,17 +24,11 @@ type CollectorConfig struct {
// CollectOnce runs `sudo orama node report --json` on all matching nodes
// in parallel and returns a ClusterSnapshot.
func CollectOnce(ctx context.Context, cfg CollectorConfig) (*ClusterSnapshot, error) {
nodes, err := inspector.LoadNodes(cfg.ConfigPath)
nodes, cleanup, err := loadNodes(cfg)
if err != nil {
return nil, fmt.Errorf("load nodes: %w", err)
}
nodes = inspector.FilterByEnv(nodes, cfg.Env)
if cfg.NodeFilter != "" {
nodes = filterByHost(nodes, cfg.NodeFilter)
}
if len(nodes) == 0 {
return nil, fmt.Errorf("no nodes found for env %q", cfg.Env)
return nil, err
}
defer cleanup()
timeout := cfg.Timeout
if timeout == 0 {
@ -87,7 +83,7 @@ func collectNodeReport(ctx context.Context, node inspector.Node, timeout time.Du
return cs
}
// Enrich with node metadata from remote-nodes.conf
// Enrich with node metadata from nodes.conf
if rpt.Hostname == "" {
rpt.Hostname = node.Host
}
@ -113,3 +109,66 @@ func truncate(s string, maxLen int) string {
}
return s[:maxLen] + "..."
}
// loadNodes resolves the node list and SSH keys based on the environment.
// For "sandbox", nodes are loaded from the active sandbox state file with
// the sandbox SSH key already set. For other environments, nodes come from
// nodes.conf and use wallet-derived SSH keys.
func loadNodes(cfg CollectorConfig) ([]inspector.Node, func(), error) {
noop := func() {}
if cfg.Env == "sandbox" {
return loadSandboxNodes(cfg)
}
nodes, err := inspector.LoadNodes(cfg.ConfigPath)
if err != nil {
return nil, noop, fmt.Errorf("load nodes: %w", err)
}
nodes = inspector.FilterByEnv(nodes, cfg.Env)
if cfg.NodeFilter != "" {
nodes = filterByHost(nodes, cfg.NodeFilter)
}
if len(nodes) == 0 {
return nil, noop, fmt.Errorf("no nodes found for env %q", cfg.Env)
}
cleanup, err := remotessh.PrepareNodeKeys(nodes)
if err != nil {
return nil, noop, fmt.Errorf("prepare SSH keys: %w", err)
}
return nodes, cleanup, nil
}
// loadSandboxNodes loads nodes from the active sandbox state file.
func loadSandboxNodes(cfg CollectorConfig) ([]inspector.Node, func(), error) {
noop := func() {}
sbxCfg, err := sandbox.LoadConfig()
if err != nil {
return nil, noop, fmt.Errorf("load sandbox config: %w", err)
}
state, err := sandbox.FindActiveSandbox()
if err != nil {
return nil, noop, fmt.Errorf("find active sandbox: %w", err)
}
if state == nil {
return nil, noop, fmt.Errorf("no active sandbox found")
}
nodes := state.ToNodes(sbxCfg.SSHKey.VaultTarget)
if cfg.NodeFilter != "" {
nodes = filterByHost(nodes, cfg.NodeFilter)
}
if len(nodes) == 0 {
return nil, noop, fmt.Errorf("no nodes found for sandbox %q", state.Name)
}
cleanup, err := remotessh.PrepareNodeKeys(nodes)
if err != nil {
return nil, noop, fmt.Errorf("prepare SSH keys: %w", err)
}
return nodes, cleanup, nil
}

189
pkg/cli/production/clean/clean.go Normal file
View File

@ -0,0 +1,189 @@
package clean
import (
"bufio"
"flag"
"fmt"
"os"
"strings"
"github.com/DeBrosOfficial/network/pkg/cli/remotessh"
"github.com/DeBrosOfficial/network/pkg/inspector"
)
// Flags holds clean command flags.
type Flags struct {
Env string // Target environment
Node string // Single node IP
Nuclear bool // Also remove shared binaries
Force bool // Skip confirmation
}
// Handle is the entry point for the clean command.
func Handle(args []string) {
flags, err := parseFlags(args)
if err != nil {
if err == flag.ErrHelp {
return
}
fmt.Fprintf(os.Stderr, "Error: %v\n", err)
os.Exit(1)
}
if err := execute(flags); err != nil {
fmt.Fprintf(os.Stderr, "Error: %v\n", err)
os.Exit(1)
}
}
func parseFlags(args []string) (*Flags, error) {
fs := flag.NewFlagSet("clean", flag.ContinueOnError)
fs.SetOutput(os.Stderr)
flags := &Flags{}
fs.StringVar(&flags.Env, "env", "", "Target environment (devnet, testnet) [required]")
fs.StringVar(&flags.Node, "node", "", "Clean a single node IP only")
fs.BoolVar(&flags.Nuclear, "nuclear", false, "Also remove shared binaries (rqlited, ipfs, caddy, etc.)")
fs.BoolVar(&flags.Force, "force", false, "Skip confirmation (DESTRUCTIVE)")
if err := fs.Parse(args); err != nil {
return nil, err
}
if flags.Env == "" {
return nil, fmt.Errorf("--env is required\nUsage: orama node clean --env <devnet|testnet> --force")
}
return flags, nil
}
func execute(flags *Flags) error {
nodes, err := remotessh.LoadEnvNodes(flags.Env)
if err != nil {
return err
}
cleanup, err := remotessh.PrepareNodeKeys(nodes)
if err != nil {
return err
}
defer cleanup()
if flags.Node != "" {
nodes = remotessh.FilterByIP(nodes, flags.Node)
if len(nodes) == 0 {
return fmt.Errorf("node %s not found in %s environment", flags.Node, flags.Env)
}
}
fmt.Printf("Clean %s: %d node(s)\n", flags.Env, len(nodes))
if flags.Nuclear {
fmt.Printf(" Mode: NUCLEAR (removes binaries too)\n")
}
for _, n := range nodes {
fmt.Printf(" - %s (%s)\n", n.Host, n.Role)
}
fmt.Println()
// Confirm unless --force
if !flags.Force {
fmt.Printf("This will DESTROY all data on these nodes. Anyone relay keys are preserved.\n")
fmt.Printf("Type 'yes' to confirm: ")
reader := bufio.NewReader(os.Stdin)
input, _ := reader.ReadString('\n')
if strings.TrimSpace(input) != "yes" {
fmt.Println("Aborted.")
return nil
}
fmt.Println()
}
// Clean each node
var failed []string
for i, node := range nodes {
fmt.Printf("[%d/%d] Cleaning %s...\n", i+1, len(nodes), node.Host)
if err := cleanNode(node, flags.Nuclear); err != nil {
fmt.Fprintf(os.Stderr, " ✗ %s: %v\n", node.Host, err)
failed = append(failed, node.Host)
continue
}
fmt.Printf(" ✓ %s cleaned\n\n", node.Host)
}
if len(failed) > 0 {
return fmt.Errorf("clean failed on %d node(s): %s", len(failed), strings.Join(failed, ", "))
}
fmt.Printf("✓ Clean complete (%d nodes)\n", len(nodes))
fmt.Printf(" Anyone relay keys preserved at /var/lib/anon/\n")
fmt.Printf(" To reinstall: orama node install --vps-ip <ip> ...\n")
return nil
}
func cleanNode(node inspector.Node, nuclear bool) error {
sudo := remotessh.SudoPrefix(node)
nuclearFlag := ""
if nuclear {
nuclearFlag = "NUCLEAR=1"
}
// The cleanup script runs on the remote node
script := fmt.Sprintf(`%sbash -c '
%s
# Stop services
for svc in caddy coredns orama-node orama-gateway orama-ipfs-cluster orama-ipfs orama-olric orama-anyone-relay orama-anyone-client; do
systemctl stop "$svc" 2>/dev/null
systemctl disable "$svc" 2>/dev/null
done
# Kill stragglers
pkill -9 -f "orama-node" 2>/dev/null || true
pkill -9 -f "olric-server" 2>/dev/null || true
pkill -9 -f "ipfs" 2>/dev/null || true
# Remove systemd units
rm -f /etc/systemd/system/orama-*.service
rm -f /etc/systemd/system/coredns.service
rm -f /etc/systemd/system/caddy.service
systemctl daemon-reload 2>/dev/null
# Tear down WireGuard
ip link delete wg0 2>/dev/null || true
rm -f /etc/wireguard/wg0.conf
# Reset firewall
ufw --force reset 2>/dev/null || true
ufw default deny incoming 2>/dev/null || true
ufw default allow outgoing 2>/dev/null || true
ufw allow 22/tcp 2>/dev/null || true
ufw --force enable 2>/dev/null || true
# Remove data
rm -rf /opt/orama
# Clean configs
rm -rf /etc/coredns
rm -rf /etc/caddy
rm -f /tmp/orama-*.sh /tmp/network-source.tar.gz /tmp/orama-*.tar.gz
# Nuclear: remove binaries
if [ -n "$NUCLEAR" ]; then
rm -f /usr/local/bin/orama /usr/local/bin/orama-node /usr/local/bin/gateway
rm -f /usr/local/bin/identity /usr/local/bin/sfu /usr/local/bin/turn
rm -f /usr/local/bin/olric-server /usr/local/bin/ipfs /usr/local/bin/ipfs-cluster-service
rm -f /usr/local/bin/rqlited /usr/local/bin/coredns
rm -f /usr/bin/caddy
fi
# Verify Anyone keys preserved
if [ -d /var/lib/anon ]; then
echo " Anyone relay keys preserved at /var/lib/anon/"
fi
echo " Node cleaned successfully"
'`, sudo, nuclearFlag)
return remotessh.RunSSHStreaming(node, script)
}

123
pkg/cli/production/enroll/command.go Normal file
View File

@ -0,0 +1,123 @@
// Package enroll implements the OramaOS node enrollment command.
//
// Flow:
// 1. Operator fetches registration code from the OramaOS node (port 9999)
// 2. Operator provides code + invite token to Gateway
// 3. Gateway validates, generates cluster config, pushes to node
// 4. Node configures WireGuard, encrypts data partition, starts services
package enroll
import (
"bytes"
"encoding/json"
"fmt"
"io"
"net/http"
"os"
"time"
)
// Handle processes the enroll command.
func Handle(args []string) {
flags, err := ParseFlags(args)
if err != nil {
fmt.Fprintf(os.Stderr, "Error: %v\n", err)
os.Exit(1)
}
// Step 1: Fetch registration code from the OramaOS node
fmt.Printf("Fetching registration code from %s:9999...\n", flags.NodeIP)
var code string
if flags.Code != "" {
// Code provided directly — skip fetch
code = flags.Code
} else {
fetchedCode, err := fetchRegistrationCode(flags.NodeIP)
if err != nil {
fmt.Fprintf(os.Stderr, "Error: could not reach OramaOS node: %v\n", err)
fmt.Fprintf(os.Stderr, "Make sure the node is booted and port 9999 is reachable.\n")
os.Exit(1)
}
code = fetchedCode
}
fmt.Printf("Registration code: %s\n", code)
// Step 2: Send enrollment request to the Gateway
fmt.Printf("Sending enrollment to Gateway at %s...\n", flags.GatewayURL)
if err := enrollWithGateway(flags.GatewayURL, flags.Token, code, flags.NodeIP); err != nil {
fmt.Fprintf(os.Stderr, "Error: enrollment failed: %v\n", err)
os.Exit(1)
}
fmt.Printf("Node %s enrolled successfully.\n", flags.NodeIP)
fmt.Printf("The node is now configuring WireGuard and encrypting its data partition.\n")
fmt.Printf("This may take a few minutes. Check status with: orama node status --env %s\n", flags.Env)
}
// fetchRegistrationCode retrieves the one-time registration code from the OramaOS node.
func fetchRegistrationCode(nodeIP string) (string, error) {
client := &http.Client{Timeout: 10 * time.Second}
resp, err := client.Get(fmt.Sprintf("http://%s:9999/", nodeIP))
if err != nil {
return "", fmt.Errorf("GET failed: %w", err)
}
defer resp.Body.Close()
if resp.StatusCode == http.StatusGone {
return "", fmt.Errorf("registration code already served (node may be partially enrolled)")
}
if resp.StatusCode != http.StatusOK {
return "", fmt.Errorf("unexpected status %d", resp.StatusCode)
}
var result struct {
Code string `json:"code"`
Expires string `json:"expires"`
}
if err := json.NewDecoder(resp.Body).Decode(&result); err != nil {
return "", fmt.Errorf("invalid response: %w", err)
}
return result.Code, nil
}
// enrollWithGateway sends the enrollment request to the Gateway, which validates
// the code and token, then pushes cluster configuration to the OramaOS node.
func enrollWithGateway(gatewayURL, token, code, nodeIP string) error {
body, _ := json.Marshal(map[string]string{
"code": code,
"token": token,
"node_ip": nodeIP,
})
req, err := http.NewRequest("POST", gatewayURL+"/v1/node/enroll", bytes.NewReader(body))
if err != nil {
return err
}
req.Header.Set("Content-Type", "application/json")
req.Header.Set("Authorization", "Bearer "+token)
client := &http.Client{Timeout: 60 * time.Second}
resp, err := client.Do(req)
if err != nil {
return fmt.Errorf("request failed: %w", err)
}
defer resp.Body.Close()
if resp.StatusCode == http.StatusUnauthorized {
return fmt.Errorf("invalid or expired invite token")
}
if resp.StatusCode == http.StatusBadRequest {
respBody, _ := io.ReadAll(resp.Body)
return fmt.Errorf("bad request: %s", string(respBody))
}
if resp.StatusCode != http.StatusOK {
respBody, _ := io.ReadAll(resp.Body)
return fmt.Errorf("gateway returned %d: %s", resp.StatusCode, string(respBody))
}
return nil
}

46
pkg/cli/production/enroll/flags.go Normal file
View File

@ -0,0 +1,46 @@
package enroll
import (
"flag"
"fmt"
"os"
)
// Flags holds the parsed command-line flags for the enroll command.
type Flags struct {
NodeIP string // Public IP of the OramaOS node
Code string // Registration code (optional — fetched automatically if not provided)
Token string // Invite token for cluster joining
GatewayURL string // Gateway HTTPS URL
Env string // Environment name (for display only)
}
// ParseFlags parses the enroll command flags.
func ParseFlags(args []string) (*Flags, error) {
fs := flag.NewFlagSet("enroll", flag.ContinueOnError)
fs.SetOutput(os.Stderr)
flags := &Flags{}
fs.StringVar(&flags.NodeIP, "node-ip", "", "Public IP of the OramaOS node (required)")
fs.StringVar(&flags.Code, "code", "", "Registration code from the node (auto-fetched if not provided)")
fs.StringVar(&flags.Token, "token", "", "Invite token for cluster joining (required)")
fs.StringVar(&flags.GatewayURL, "gateway", "", "Gateway URL (required, e.g. https://gateway.example.com)")
fs.StringVar(&flags.Env, "env", "production", "Environment name")
if err := fs.Parse(args); err != nil {
return nil, err
}
if flags.NodeIP == "" {
return nil, fmt.Errorf("--node-ip is required")
}
if flags.Token == "" {
return nil, fmt.Errorf("--token is required")
}
if flags.GatewayURL == "" {
return nil, fmt.Errorf("--gateway is required")
}
return flags, nil
}

4
pkg/cli/production/install/flags.go
View File

@ -28,7 +28,8 @@ type Flags struct {
IPFSClusterAddrs string
// Security flags
SkipFirewall bool // Skip UFW firewall setup (for users who manage their own firewall)
SkipFirewall bool // Skip UFW firewall setup (for users who manage their own firewall)
CAFingerprint string // SHA-256 fingerprint of server TLS cert for TOFU verification
// Anyone flags
AnyoneClient bool // Run Anyone as client-only (SOCKS5 proxy on port 9050, no relay)
@ -74,6 +75,7 @@ func ParseFlags(args []string) (*Flags, error) {
// Security flags
fs.BoolVar(&flags.SkipFirewall, "skip-firewall", false, "Skip UFW firewall setup (for users who manage their own firewall)")
fs.StringVar(&flags.CAFingerprint, "ca-fingerprint", "", "SHA-256 fingerprint of server TLS cert (from orama invite output)")
// Anyone flags
fs.BoolVar(&flags.AnyoneClient, "anyone-client", false, "Install Anyone as client-only (SOCKS5 proxy on port 9050, no relay)")

67
pkg/cli/production/install/orchestrator.go
View File

@ -2,8 +2,12 @@ package install
import (
"bufio"
"bytes"
"crypto/rand"
"crypto/sha256"
"crypto/tls"
"crypto/x509"
"encoding/hex"
"encoding/json"
"fmt"
"io"
@ -366,12 +370,35 @@ func (o *Orchestrator) callJoinEndpoint(wgPubKey string) (*joinhandlers.JoinResp
}
url := strings.TrimRight(o.flags.JoinAddress, "/") + "/v1/internal/join"
tlsConfig := &tls.Config{}
if o.flags.CAFingerprint != "" {
// TOFU: verify the server's TLS cert fingerprint matches the one from the invite
expectedFP, err := hex.DecodeString(o.flags.CAFingerprint)
if err != nil {
return nil, fmt.Errorf("invalid --ca-fingerprint: must be hex-encoded SHA-256: %w", err)
}
tlsConfig.InsecureSkipVerify = true
tlsConfig.VerifyPeerCertificate = func(rawCerts [][]byte, _ [][]*x509.Certificate) error {
if len(rawCerts) == 0 {
return fmt.Errorf("server presented no TLS certificates")
}
hash := sha256.Sum256(rawCerts[0])
if !bytes.Equal(hash[:], expectedFP) {
return fmt.Errorf("TLS certificate fingerprint mismatch: expected %s, got %x (possible MITM attack)",
o.flags.CAFingerprint, hash[:])
}
return nil
}
} else {
// No fingerprint provided — fall back to insecure for backward compatibility
tlsConfig.InsecureSkipVerify = true
}
client := &http.Client{
Timeout: 30 * time.Second,
Transport: &http.Transport{
TLSClientConfig: &tls.Config{
InsecureSkipVerify: true, // Self-signed certs during initial setup
},
TLSClientConfig: tlsConfig,
},
}
@ -419,6 +446,40 @@ func (o *Orchestrator) saveSecretsFromJoinResponse(resp *joinhandlers.JoinRespon
}
}
// Write API key HMAC secret
if resp.APIKeyHMACSecret != "" {
if err := os.WriteFile(filepath.Join(secretsDir, "api-key-hmac-secret"), []byte(resp.APIKeyHMACSecret), 0600); err != nil {
return fmt.Errorf("failed to write api-key-hmac-secret: %w", err)
}
}
// Write RQLite password and generate auth JSON file
if resp.RQLitePassword != "" {
if err := os.WriteFile(filepath.Join(secretsDir, "rqlite-password"), []byte(resp.RQLitePassword), 0600); err != nil {
return fmt.Errorf("failed to write rqlite-password: %w", err)
}
// Also generate the auth JSON file that rqlited uses with -auth flag
authJSON := fmt.Sprintf(`[{"username": "orama", "password": "%s", "perms": ["all"]}]`, resp.RQLitePassword)
if err := os.WriteFile(filepath.Join(secretsDir, "rqlite-auth.json"), []byte(authJSON), 0600); err != nil {
return fmt.Errorf("failed to write rqlite-auth.json: %w", err)
}
}
// Write Olric encryption key
if resp.OlricEncryptionKey != "" {
if err := os.WriteFile(filepath.Join(secretsDir, "olric-encryption-key"), []byte(resp.OlricEncryptionKey), 0600); err != nil {
return fmt.Errorf("failed to write olric-encryption-key: %w", err)
}
}
// Write IPFS Cluster trusted peer IDs
if len(resp.IPFSClusterPeerIDs) > 0 {
content := strings.Join(resp.IPFSClusterPeerIDs, "\n") + "\n"
if err := os.WriteFile(filepath.Join(secretsDir, "ipfs-cluster-trusted-peers"), []byte(content), 0600); err != nil {
return fmt.Errorf("failed to write ipfs-cluster-trusted-peers: %w", err)
}
}
return nil
}

122
pkg/cli/production/install/remote.go
View File

@ -2,9 +2,12 @@ package install
import (
"fmt"
"os"
"path/filepath"
"strconv"
"strings"
"github.com/DeBrosOfficial/network/pkg/cli/remotessh"
"github.com/DeBrosOfficial/network/pkg/inspector"
)
@ -12,34 +15,75 @@ import (
// It uploads the source archive, extracts it on the VPS, and runs
// the actual install command remotely.
type RemoteOrchestrator struct {
flags *Flags
node inspector.Node
flags *Flags
node inspector.Node
cleanup func()
}
// NewRemoteOrchestrator creates a new remote orchestrator.
// It resolves SSH credentials and checks prerequisites.
// Resolves SSH credentials via wallet-derived keys and checks prerequisites.
func NewRemoteOrchestrator(flags *Flags) (*RemoteOrchestrator, error) {
if flags.VpsIP == "" {
return nil, fmt.Errorf("--vps-ip is required\nExample: orama install --vps-ip 1.2.3.4 --nameserver --domain orama-testnet.network")
}
// Resolve SSH credentials
node, err := resolveSSHCredentials(flags.VpsIP)
if err != nil {
return nil, fmt.Errorf("failed to resolve SSH credentials: %w", err)
// Try to find this IP in nodes.conf for the correct user
user := resolveUser(flags.VpsIP)
node := inspector.Node{
User: user,
Host: flags.VpsIP,
Role: "node",
}
// Prepare wallet-derived SSH key
nodes := []inspector.Node{node}
cleanup, err := remotessh.PrepareNodeKeys(nodes)
if err != nil {
return nil, fmt.Errorf("failed to prepare SSH key: %w\nEnsure you've run: rw vault ssh add %s/%s", err, flags.VpsIP, user)
}
// PrepareNodeKeys modifies nodes in place
node = nodes[0]
return &RemoteOrchestrator{
flags: flags,
node: node,
flags: flags,
node: node,
cleanup: cleanup,
}, nil
}
// resolveUser looks up the SSH user for a VPS IP from nodes.conf.
// Falls back to "root" if not found.
func resolveUser(vpsIP string) string {
confPath := remotessh.FindNodesConf()
if confPath != "" {
nodes, err := inspector.LoadNodes(confPath)
if err == nil {
for _, n := range nodes {
if n.Host == vpsIP {
return n.User
}
}
}
}
return "root"
}
// Execute runs the remote install process.
// Source must already be uploaded via: ./scripts/upload-source.sh <vps-ip>
// If a binary archive exists locally, uploads and extracts it on the VPS
// so Phase2b auto-detects pre-built mode. Otherwise, source must already
// be present on the VPS.
func (r *RemoteOrchestrator) Execute() error {
defer r.cleanup()
fmt.Printf("Installing on %s via SSH (%s@%s)...\n\n", r.flags.VpsIP, r.node.User, r.node.Host)
// Try to upload a binary archive if one exists locally
if err := r.uploadBinaryArchive(); err != nil {
fmt.Printf(" Binary archive upload skipped: %v\n", err)
fmt.Printf(" Proceeding with source mode (source must already be on VPS)\n\n")
}
// Run remote install
fmt.Printf("Running install on VPS...\n\n")
if err := r.runRemoteInstall(); err != nil {
@ -49,10 +93,66 @@ func (r *RemoteOrchestrator) Execute() error {
return nil
}
// uploadBinaryArchive finds a local binary archive and uploads + extracts it on the VPS.
// Returns nil on success, error if no archive found or upload failed.
func (r *RemoteOrchestrator) uploadBinaryArchive() error {
archivePath := r.findLocalArchive()
if archivePath == "" {
return fmt.Errorf("no binary archive found locally")
}
fmt.Printf("Uploading binary archive: %s\n", filepath.Base(archivePath))
// Upload to /tmp/ on VPS
remoteTmp := "/tmp/" + filepath.Base(archivePath)
if err := remotessh.UploadFile(r.node, archivePath, remoteTmp); err != nil {
return fmt.Errorf("failed to upload archive: %w", err)
}
// Extract to /opt/orama/ and install CLI to PATH
fmt.Printf("Extracting archive on VPS...\n")
extractCmd := fmt.Sprintf("%smkdir -p /opt/orama && tar xzf %s -C /opt/orama && rm -f %s && cp /opt/orama/bin/orama /usr/local/bin/orama && chmod +x /usr/local/bin/orama && echo ' ✓ Archive extracted, CLI installed'",
r.sudoPrefix(), remoteTmp, remoteTmp)
if err := remotessh.RunSSHStreaming(r.node, extractCmd); err != nil {
return fmt.Errorf("failed to extract archive on VPS: %w", err)
}
fmt.Println()
return nil
}
// findLocalArchive searches for a binary archive in common locations.
func (r *RemoteOrchestrator) findLocalArchive() string {
// Check /tmp/ for archives matching the naming pattern
entries, err := os.ReadDir("/tmp")
if err != nil {
return ""
}
// Look for orama-*-linux-*.tar.gz, prefer newest
var best string
var bestMod int64
for _, entry := range entries {
name := entry.Name()
if strings.HasPrefix(name, "orama-") && strings.Contains(name, "-linux-") && strings.HasSuffix(name, ".tar.gz") {
info, err := entry.Info()
if err != nil {
continue
}
if info.ModTime().Unix() > bestMod {
best = filepath.Join("/tmp", name)
bestMod = info.ModTime().Unix()
}
}
}
return best
}
// runRemoteInstall executes `orama install` on the VPS.
func (r *RemoteOrchestrator) runRemoteInstall() error {
cmd := r.buildRemoteCommand()
return runSSHStreaming(r.node, cmd)
return remotessh.RunSSHStreaming(r.node, cmd)
}
// buildRemoteCommand constructs the `sudo orama install` command string

153
pkg/cli/production/install/ssh.go
View File

@ -1,153 +0,0 @@
package install
import (
"bufio"
"fmt"
"os"
"os/exec"
"path/filepath"
"strings"
"github.com/DeBrosOfficial/network/pkg/inspector"
"golang.org/x/term"
)
const sourceArchivePath = "/tmp/network-source.tar.gz"
// resolveSSHCredentials finds SSH credentials for the given VPS IP.
// First checks remote-nodes.conf, then prompts interactively.
func resolveSSHCredentials(vpsIP string) (inspector.Node, error) {
confPath := findRemoteNodesConf()
if confPath != "" {
nodes, err := inspector.LoadNodes(confPath)
if err == nil {
for _, n := range nodes {
if n.Host == vpsIP {
// Expand ~ in SSH key path
if n.SSHKey != "" && strings.HasPrefix(n.SSHKey, "~") {
home, _ := os.UserHomeDir()
n.SSHKey = filepath.Join(home, n.SSHKey[1:])
}
return n, nil
}
}
}
}
// Not found in config — prompt interactively
return promptSSHCredentials(vpsIP), nil
}
// findRemoteNodesConf searches for the remote-nodes.conf file.
func findRemoteNodesConf() string {
candidates := []string{
"scripts/remote-nodes.conf",
"../scripts/remote-nodes.conf",
"network/scripts/remote-nodes.conf",
}
for _, c := range candidates {
if _, err := os.Stat(c); err == nil {
return c
}
}
return ""
}
// promptSSHCredentials asks the user for SSH credentials interactively.
func promptSSHCredentials(vpsIP string) inspector.Node {
reader := bufio.NewReader(os.Stdin)
fmt.Printf("\nSSH credentials for %s\n", vpsIP)
fmt.Print(" SSH user (default: ubuntu): ")
user, _ := reader.ReadString('\n')
user = strings.TrimSpace(user)
if user == "" {
user = "ubuntu"
}
fmt.Print(" SSH password: ")
passwordBytes, err := term.ReadPassword(int(os.Stdin.Fd()))
fmt.Println() // newline after hidden input
if err != nil {
// Fall back to plain read if terminal is not available
password, _ := reader.ReadString('\n')
return inspector.Node{
User: user,
Host: vpsIP,
Password: strings.TrimSpace(password),
}
}
password := string(passwordBytes)
return inspector.Node{
User: user,
Host: vpsIP,
Password: password,
}
}
// uploadFile copies a local file to a remote host via SCP.
func uploadFile(node inspector.Node, localPath, remotePath string) error {
dest := fmt.Sprintf("%s@%s:%s", node.User, node.Host, remotePath)
var cmd *exec.Cmd
if node.SSHKey != "" {
cmd = exec.Command("scp",
"-o", "StrictHostKeyChecking=no",
"-o", "ConnectTimeout=10",
"-i", node.SSHKey,
localPath, dest,
)
} else {
if _, err := exec.LookPath("sshpass"); err != nil {
return fmt.Errorf("sshpass not found — install it: brew install hudochenkov/sshpass/sshpass")
}
cmd = exec.Command("sshpass", "-p", node.Password,
"scp",
"-o", "StrictHostKeyChecking=no",
"-o", "ConnectTimeout=10",
localPath, dest,
)
}
cmd.Stdout = os.Stdout
cmd.Stderr = os.Stderr
if err := cmd.Run(); err != nil {
return fmt.Errorf("SCP failed: %w", err)
}
return nil
}
// runSSHStreaming executes a command on a remote host via SSH,
// streaming stdout/stderr to the local terminal in real-time.
func runSSHStreaming(node inspector.Node, command string) error {
var cmd *exec.Cmd
if node.SSHKey != "" {
cmd = exec.Command("ssh",
"-o", "StrictHostKeyChecking=no",
"-o", "ConnectTimeout=10",
"-i", node.SSHKey,
fmt.Sprintf("%s@%s", node.User, node.Host),
command,
)
} else {
cmd = exec.Command("sshpass", "-p", node.Password,
"ssh",
"-o", "StrictHostKeyChecking=no",
"-o", "ConnectTimeout=10",
fmt.Sprintf("%s@%s", node.User, node.Host),
command,
)
}
cmd.Stdout = os.Stdout
cmd.Stderr = os.Stderr
cmd.Stdin = os.Stdin // Allow password prompts from remote sudo
if err := cmd.Run(); err != nil {
return fmt.Errorf("SSH command failed: %w", err)
}
return nil
}

35
pkg/cli/production/invite/command.go
View File

@ -3,9 +3,12 @@ package invite
import (
"bytes"
"crypto/rand"
"crypto/sha256"
"crypto/tls"
"encoding/hex"
"encoding/json"
"fmt"
"net"
"net/http"
"os"
"time"
@ -59,13 +62,43 @@ func Handle(args []string) {
os.Exit(1)
}
// Get TLS certificate fingerprint for TOFU verification
certFingerprint := getTLSCertFingerprint(domain)
// Print the invite command
fmt.Printf("\nInvite token created (expires in %s)\n\n", expiry)
fmt.Printf("Run this on the new node:\n\n")
fmt.Printf(" sudo orama install --join https://%s --token %s --vps-ip <NEW_NODE_IP> --nameserver\n\n", domain, token)
if certFingerprint != "" {
fmt.Printf(" sudo orama install --join https://%s --token %s --ca-fingerprint %s --vps-ip <NEW_NODE_IP> --nameserver\n\n", domain, token, certFingerprint)
} else {
fmt.Printf(" sudo orama install --join https://%s --token %s --vps-ip <NEW_NODE_IP> --nameserver\n\n", domain, token)
}
fmt.Printf("Replace <NEW_NODE_IP> with the new node's public IP address.\n")
}
// getTLSCertFingerprint connects to the domain over TLS and returns the
// SHA-256 fingerprint of the leaf certificate. Returns empty string on failure.
func getTLSCertFingerprint(domain string) string {
conn, err := tls.DialWithDialer(
&net.Dialer{Timeout: 5 * time.Second},
"tcp",
domain+":443",
&tls.Config{InsecureSkipVerify: true},
)
if err != nil {
return ""
}
defer conn.Close()
certs := conn.ConnectionState().PeerCertificates
if len(certs) == 0 {
return ""
}
hash := sha256.Sum256(certs[0].Raw)
return hex.EncodeToString(hash[:])
}
// readNodeDomain reads the domain from the node config file
func readNodeDomain() (string, error) {
configPath := "/opt/orama/.orama/configs/node.yaml"

261
pkg/cli/production/push/push.go Normal file
View File

@ -0,0 +1,261 @@
package push
import (
"flag"
"fmt"
"os"
"path/filepath"
"strings"
"sync"
"github.com/DeBrosOfficial/network/pkg/cli/remotessh"
"github.com/DeBrosOfficial/network/pkg/inspector"
)
// Flags holds push command flags.
type Flags struct {
Env string // Target environment (devnet, testnet)
Node string // Single node IP (optional)
Direct bool // Sequential upload to each node (no fanout)
}
// Handle is the entry point for the push command.
func Handle(args []string) {
flags, err := parseFlags(args)
if err != nil {
if err == flag.ErrHelp {
return
}
fmt.Fprintf(os.Stderr, "Error: %v\n", err)
os.Exit(1)
}
if err := execute(flags); err != nil {
fmt.Fprintf(os.Stderr, "Error: %v\n", err)
os.Exit(1)
}
}
func parseFlags(args []string) (*Flags, error) {
fs := flag.NewFlagSet("push", flag.ContinueOnError)
fs.SetOutput(os.Stderr)
flags := &Flags{}
fs.StringVar(&flags.Env, "env", "", "Target environment (devnet, testnet) [required]")
fs.StringVar(&flags.Node, "node", "", "Push to a single node IP only")
fs.BoolVar(&flags.Direct, "direct", false, "Upload directly to each node (no hub fanout)")
if err := fs.Parse(args); err != nil {
return nil, err
}
if flags.Env == "" {
return nil, fmt.Errorf("--env is required\nUsage: orama node push --env <devnet|testnet>")
}
return flags, nil
}
func execute(flags *Flags) error {
// Find archive
archivePath := findNewestArchive()
if archivePath == "" {
return fmt.Errorf("no binary archive found in /tmp/ (run `orama build` first)")
}
info, _ := os.Stat(archivePath)
fmt.Printf("Archive: %s (%s)\n", filepath.Base(archivePath), formatBytes(info.Size()))
// Resolve nodes
nodes, err := remotessh.LoadEnvNodes(flags.Env)
if err != nil {
return err
}
// Prepare wallet-derived SSH keys
cleanup, err := remotessh.PrepareNodeKeys(nodes)
if err != nil {
return err
}
defer cleanup()
// Filter to single node if specified
if flags.Node != "" {
nodes = remotessh.FilterByIP(nodes, flags.Node)
if len(nodes) == 0 {
return fmt.Errorf("node %s not found in %s environment", flags.Node, flags.Env)
}
}
fmt.Printf("Environment: %s (%d nodes)\n\n", flags.Env, len(nodes))
if flags.Direct || len(nodes) == 1 {
return pushDirect(archivePath, nodes)
}
// Load keys into ssh-agent for fanout forwarding
if err := remotessh.LoadAgentKeys(nodes); err != nil {
return fmt.Errorf("load agent keys for fanout: %w", err)
}
return pushFanout(archivePath, nodes)
}
// pushDirect uploads the archive to each node sequentially.
func pushDirect(archivePath string, nodes []inspector.Node) error {
remotePath := "/tmp/" + filepath.Base(archivePath)
for i, node := range nodes {
fmt.Printf("[%d/%d] Pushing to %s...\n", i+1, len(nodes), node.Host)
if err := remotessh.UploadFile(node, archivePath, remotePath); err != nil {
return fmt.Errorf("upload to %s failed: %w", node.Host, err)
}
if err := extractOnNode(node, remotePath); err != nil {
return fmt.Errorf("extract on %s failed: %w", node.Host, err)
}
fmt.Printf(" ✓ %s done\n\n", node.Host)
}
fmt.Printf("✓ Push complete (%d nodes)\n", len(nodes))
return nil
}
// pushFanout uploads to a hub node, then fans out to all others via agent forwarding.
func pushFanout(archivePath string, nodes []inspector.Node) error {
hub := remotessh.PickHubNode(nodes)
remotePath := "/tmp/" + filepath.Base(archivePath)
// Step 1: Upload to hub
fmt.Printf("[hub] Uploading to %s...\n", hub.Host)
if err := remotessh.UploadFile(hub, archivePath, remotePath); err != nil {
return fmt.Errorf("upload to hub %s failed: %w", hub.Host, err)
}
if err := extractOnNode(hub, remotePath); err != nil {
return fmt.Errorf("extract on hub %s failed: %w", hub.Host, err)
}
fmt.Printf(" ✓ hub %s done\n\n", hub.Host)
// Step 2: Fan out from hub to remaining nodes in parallel (via agent forwarding)
remaining := make([]inspector.Node, 0, len(nodes)-1)
for _, n := range nodes {
if n.Host != hub.Host {
remaining = append(remaining, n)
}
}
if len(remaining) == 0 {
fmt.Printf("✓ Push complete (1 node)\n")
return nil
}
fmt.Printf("[fanout] Distributing from %s to %d nodes...\n", hub.Host, len(remaining))
var wg sync.WaitGroup
errors := make([]error, len(remaining))
for i, target := range remaining {
wg.Add(1)
go func(idx int, target inspector.Node) {
defer wg.Done()
// SCP from hub to target (agent forwarding serves the key)
scpCmd := fmt.Sprintf("scp -o StrictHostKeyChecking=accept-new -o ConnectTimeout=10 %s %s@%s:%s",
remotePath, target.User, target.Host, remotePath)
if err := remotessh.RunSSHStreaming(hub, scpCmd, remotessh.WithAgentForward()); err != nil {
errors[idx] = fmt.Errorf("fanout to %s failed: %w", target.Host, err)
return
}
if err := extractOnNodeVia(hub, target, remotePath); err != nil {
errors[idx] = fmt.Errorf("extract on %s failed: %w", target.Host, err)
return
}
fmt.Printf(" ✓ %s done\n", target.Host)
}(i, target)
}
wg.Wait()
// Check for errors
var failed []string
for i, err := range errors {
if err != nil {
fmt.Fprintf(os.Stderr, " ✗ %s: %v\n", remaining[i].Host, err)
failed = append(failed, remaining[i].Host)
}
}
if len(failed) > 0 {
return fmt.Errorf("push failed on %d node(s): %s", len(failed), strings.Join(failed, ", "))
}
fmt.Printf("\n✓ Push complete (%d nodes)\n", len(nodes))
return nil
}
// extractOnNode extracts the archive on a remote node.
func extractOnNode(node inspector.Node, remotePath string) error {
sudo := remotessh.SudoPrefix(node)
cmd := fmt.Sprintf("%smkdir -p /opt/orama && %star xzf %s -C /opt/orama && %srm -f %s",
sudo, sudo, remotePath, sudo, remotePath)
return remotessh.RunSSHStreaming(node, cmd)
}
// extractOnNodeVia extracts the archive on a target node by SSHing through the hub.
// Uses agent forwarding so the hub can authenticate to the target.
func extractOnNodeVia(hub, target inspector.Node, remotePath string) error {
sudo := remotessh.SudoPrefix(target)
extractCmd := fmt.Sprintf("%smkdir -p /opt/orama && %star xzf %s -C /opt/orama && %srm -f %s",
sudo, sudo, remotePath, sudo, remotePath)
// SSH from hub to target to extract (agent forwarding serves the key)
sshCmd := fmt.Sprintf("ssh -o StrictHostKeyChecking=accept-new -o ConnectTimeout=10 %s@%s '%s'",
target.User, target.Host, extractCmd)
return remotessh.RunSSHStreaming(hub, sshCmd, remotessh.WithAgentForward())
}
// findNewestArchive finds the newest binary archive in /tmp/.
func findNewestArchive() string {
entries, err := os.ReadDir("/tmp")
if err != nil {
return ""
}
var best string
var bestMod int64
for _, entry := range entries {
name := entry.Name()
if strings.HasPrefix(name, "orama-") && strings.Contains(name, "-linux-") && strings.HasSuffix(name, ".tar.gz") {
info, err := entry.Info()
if err != nil {
continue
}
if info.ModTime().Unix() > bestMod {
best = filepath.Join("/tmp", name)
bestMod = info.ModTime().Unix()
}
}
}
return best
}
func formatBytes(b int64) string {
const unit = 1024
if b < unit {
return fmt.Sprintf("%d B", b)
}
div, exp := int64(unit), 0
for n := b / unit; n >= unit; n /= unit {
div *= unit
exp++
}
return fmt.Sprintf("%.1f %cB", float64(b)/float64(div), "KMGTPE"[exp])
}

312
pkg/cli/production/recover/recover.go Normal file
View File

@ -0,0 +1,312 @@
package recover
import (
"bufio"
"flag"
"fmt"
"os"
"strings"
"time"
"github.com/DeBrosOfficial/network/pkg/cli/remotessh"
"github.com/DeBrosOfficial/network/pkg/inspector"
)
// Flags holds recover-raft command flags.
type Flags struct {
Env string // Target environment
Leader string // Leader node IP (highest commit index)
Force bool // Skip confirmation
}
const (
raftDir = "/opt/orama/.orama/data/rqlite/raft"
backupDir = "/tmp/rqlite-raft-backup"
)
// Handle is the entry point for the recover-raft command.
func Handle(args []string) {
flags, err := parseFlags(args)
if err != nil {
if err == flag.ErrHelp {
return
}
fmt.Fprintf(os.Stderr, "Error: %v\n", err)
os.Exit(1)
}
if err := execute(flags); err != nil {
fmt.Fprintf(os.Stderr, "Error: %v\n", err)
os.Exit(1)
}
}
func parseFlags(args []string) (*Flags, error) {
fs := flag.NewFlagSet("recover-raft", flag.ContinueOnError)
fs.SetOutput(os.Stderr)
flags := &Flags{}
fs.StringVar(&flags.Env, "env", "", "Target environment (devnet, testnet) [required]")
fs.StringVar(&flags.Leader, "leader", "", "Leader node IP (node with highest commit index) [required]")
fs.BoolVar(&flags.Force, "force", false, "Skip confirmation (DESTRUCTIVE)")
if err := fs.Parse(args); err != nil {
return nil, err
}
if flags.Env == "" {
return nil, fmt.Errorf("--env is required\nUsage: orama node recover-raft --env <devnet|testnet> --leader <ip>")
}
if flags.Leader == "" {
return nil, fmt.Errorf("--leader is required\nUsage: orama node recover-raft --env <devnet|testnet> --leader <ip>")
}
return flags, nil
}
func execute(flags *Flags) error {
nodes, err := remotessh.LoadEnvNodes(flags.Env)
if err != nil {
return err
}
cleanup, err := remotessh.PrepareNodeKeys(nodes)
if err != nil {
return err
}
defer cleanup()
// Find leader node
leaderNodes := remotessh.FilterByIP(nodes, flags.Leader)
if len(leaderNodes) == 0 {
return fmt.Errorf("leader %s not found in %s environment", flags.Leader, flags.Env)
}
leader := leaderNodes[0]
// Separate leader from followers
var followers []inspector.Node
for _, n := range nodes {
if n.Host != leader.Host {
followers = append(followers, n)
}
}
// Print plan
fmt.Printf("Recover Raft: %s (%d nodes)\n", flags.Env, len(nodes))
fmt.Printf(" Leader candidate: %s (%s) — raft/ data preserved\n", leader.Host, leader.Role)
for _, n := range followers {
fmt.Printf(" - %s (%s) — raft/ will be deleted\n", n.Host, n.Role)
}
fmt.Println()
// Confirm unless --force
if !flags.Force {
fmt.Printf("⚠️ THIS WILL:\n")
fmt.Printf(" 1. Stop orama-node on ALL %d nodes\n", len(nodes))
fmt.Printf(" 2. DELETE raft/ data on %d nodes (backup to %s)\n", len(followers), backupDir)
fmt.Printf(" 3. Keep raft/ data ONLY on %s (leader candidate)\n", leader.Host)
fmt.Printf(" 4. Restart all nodes to reform the cluster\n")
fmt.Printf("\nType 'yes' to confirm: ")
reader := bufio.NewReader(os.Stdin)
input, _ := reader.ReadString('\n')
if strings.TrimSpace(input) != "yes" {
fmt.Println("Aborted.")
return nil
}
fmt.Println()
}
// Phase 1: Stop orama-node on ALL nodes
if err := phase1StopAll(nodes); err != nil {
return fmt.Errorf("phase 1 (stop all): %w", err)
}
// Phase 2: Backup and delete raft/ on non-leader nodes
if err := phase2ClearFollowers(followers); err != nil {
return fmt.Errorf("phase 2 (clear followers): %w", err)
}
fmt.Printf(" Leader node %s raft/ data preserved.\n\n", leader.Host)
// Phase 3: Start leader node and wait for Leader state
if err := phase3StartLeader(leader); err != nil {
return fmt.Errorf("phase 3 (start leader): %w", err)
}
// Phase 4: Start remaining nodes in batches
if err := phase4StartFollowers(followers); err != nil {
return fmt.Errorf("phase 4 (start followers): %w", err)
}
// Phase 5: Verify cluster health
phase5Verify(nodes, leader)
return nil
}
func phase1StopAll(nodes []inspector.Node) error {
fmt.Printf("== Phase 1: Stopping orama-node on all %d nodes ==\n", len(nodes))
var failed []inspector.Node
for _, node := range nodes {
sudo := remotessh.SudoPrefix(node)
fmt.Printf(" Stopping %s ... ", node.Host)
cmd := fmt.Sprintf("%ssystemctl stop orama-node 2>&1 && echo STOPPED", sudo)
if err := remotessh.RunSSHStreaming(node, cmd); err != nil {
fmt.Printf("FAILED\n")
failed = append(failed, node)
continue
}
fmt.Println()
}
// Kill stragglers
if len(failed) > 0 {
fmt.Printf("\n⚠️ %d nodes failed to stop. Attempting kill...\n", len(failed))
for _, node := range failed {
sudo := remotessh.SudoPrefix(node)
cmd := fmt.Sprintf("%skillall -9 orama-node rqlited 2>/dev/null; echo KILLED", sudo)
_ = remotessh.RunSSHStreaming(node, cmd)
}
}
fmt.Printf("\nWaiting 5s for processes to fully stop...\n")
time.Sleep(5 * time.Second)
fmt.Println()
return nil
}
func phase2ClearFollowers(followers []inspector.Node) error {
fmt.Printf("== Phase 2: Clearing raft state on %d non-leader nodes ==\n", len(followers))
for _, node := range followers {
sudo := remotessh.SudoPrefix(node)
fmt.Printf(" Clearing %s ... ", node.Host)
script := fmt.Sprintf(`%sbash -c '
rm -rf %s
if [ -d %s ]; then
cp -r %s %s 2>/dev/null || true
rm -rf %s
echo "CLEARED (backup at %s)"
else
echo "NO_RAFT_DIR (nothing to clear)"
fi
'`, sudo, backupDir, raftDir, raftDir, backupDir, raftDir, backupDir)
if err := remotessh.RunSSHStreaming(node, script); err != nil {
fmt.Printf("FAILED: %v\n", err)
continue
}
fmt.Println()
}
return nil
}
func phase3StartLeader(leader inspector.Node) error {
fmt.Printf("== Phase 3: Starting leader node (%s) ==\n", leader.Host)
sudo := remotessh.SudoPrefix(leader)
startCmd := fmt.Sprintf("%ssystemctl start orama-node", sudo)
if err := remotessh.RunSSHStreaming(leader, startCmd); err != nil {
return fmt.Errorf("failed to start leader node %s: %w", leader.Host, err)
}
fmt.Printf(" Waiting for leader to become Leader...\n")
maxWait := 120
elapsed := 0
for elapsed < maxWait {
// Check raft state via RQLite status endpoint
checkCmd := `curl -s --max-time 3 http://localhost:5001/status 2>/dev/null | python3 -c "
import sys,json
try:
d=json.load(sys.stdin)
print(d.get('store',{}).get('raft',{}).get('state',''))
except:
print('')
" 2>/dev/null || echo ""`
// We can't easily capture output from RunSSHStreaming, so we use a simple approach
// Check via a combined command that prints a marker
stateCheckCmd := fmt.Sprintf(`state=$(%s); echo "RAFT_STATE=$state"`, checkCmd)
// Since RunSSHStreaming prints to stdout, we'll poll and let user see the state
fmt.Printf(" ... polling (%ds / %ds)\n", elapsed, maxWait)
// Try to check state - the output goes to stdout via streaming
_ = remotessh.RunSSHStreaming(leader, stateCheckCmd)
time.Sleep(5 * time.Second)
elapsed += 5
}
fmt.Printf(" Leader start complete. Check output above for state.\n\n")
return nil
}
func phase4StartFollowers(followers []inspector.Node) error {
fmt.Printf("== Phase 4: Starting %d remaining nodes ==\n", len(followers))
batchSize := 3
for i, node := range followers {
sudo := remotessh.SudoPrefix(node)
fmt.Printf(" Starting %s ... ", node.Host)
cmd := fmt.Sprintf("%ssystemctl start orama-node && echo STARTED", sudo)
if err := remotessh.RunSSHStreaming(node, cmd); err != nil {
fmt.Printf("FAILED: %v\n", err)
continue
}
fmt.Println()
// Batch delay for cluster stability
if (i+1)%batchSize == 0 && i+1 < len(followers) {
fmt.Printf(" (waiting 15s between batches for cluster stability)\n")
time.Sleep(15 * time.Second)
}
}
fmt.Println()
return nil
}
func phase5Verify(nodes []inspector.Node, leader inspector.Node) {
fmt.Printf("== Phase 5: Waiting for cluster to stabilize ==\n")
// Wait in 30s increments
for _, s := range []int{30, 60, 90, 120} {
time.Sleep(30 * time.Second)
fmt.Printf(" ... %ds\n", s)
}
fmt.Printf("\n== Cluster status ==\n")
for _, node := range nodes {
marker := ""
if node.Host == leader.Host {
marker = " ← LEADER"
}
checkCmd := `curl -s --max-time 5 http://localhost:5001/status 2>/dev/null | python3 -c "
import sys,json
try:
d=json.load(sys.stdin)
r=d.get('store',{}).get('raft',{})
n=d.get('store',{}).get('num_nodes','?')
print(f'state={r.get(\"state\",\"?\")} commit={r.get(\"commit_index\",\"?\")} leader={r.get(\"leader\",{}).get(\"node_id\",\"?\")} nodes={n}')
except:
print('NO_RESPONSE')
" 2>/dev/null || echo "SSH_FAILED"`
fmt.Printf(" %s%s: ", node.Host, marker)
_ = remotessh.RunSSHStreaming(node, checkCmd)
fmt.Println()
}
fmt.Printf("\n== Recovery complete ==\n\n")
fmt.Printf("Next steps:\n")
fmt.Printf(" 1. Run 'orama monitor report --env <env>' to verify full cluster health\n")
fmt.Printf(" 2. If some nodes show Candidate state, give them more time (up to 5 min)\n")
fmt.Printf(" 3. If nodes fail to join, check /opt/orama/.orama/logs/rqlite-node.log on the node\n")
}

102
pkg/cli/production/rollout/rollout.go Normal file
View File

@ -0,0 +1,102 @@
package rollout
import (
"flag"
"fmt"
"os"
"time"
"github.com/DeBrosOfficial/network/pkg/cli/build"
"github.com/DeBrosOfficial/network/pkg/cli/production/push"
"github.com/DeBrosOfficial/network/pkg/cli/production/upgrade"
)
// Flags holds rollout command flags.
type Flags struct {
Env string // Target environment (devnet, testnet)
NoBuild bool // Skip the build step
Yes bool // Skip confirmation
Delay int // Delay in seconds between nodes
}
// Handle is the entry point for the rollout command.
func Handle(args []string) {
flags, err := parseFlags(args)
if err != nil {
if err == flag.ErrHelp {
return
}
fmt.Fprintf(os.Stderr, "Error: %v\n", err)
os.Exit(1)
}
if err := execute(flags); err != nil {
fmt.Fprintf(os.Stderr, "Error: %v\n", err)
os.Exit(1)
}
}
func parseFlags(args []string) (*Flags, error) {
fs := flag.NewFlagSet("rollout", flag.ContinueOnError)
fs.SetOutput(os.Stderr)
flags := &Flags{}
fs.StringVar(&flags.Env, "env", "", "Target environment (devnet, testnet) [required]")
fs.BoolVar(&flags.NoBuild, "no-build", false, "Skip build step (use existing archive)")
fs.BoolVar(&flags.Yes, "yes", false, "Skip confirmation")
fs.IntVar(&flags.Delay, "delay", 30, "Delay in seconds between nodes during rolling upgrade")
if err := fs.Parse(args); err != nil {
return nil, err
}
if flags.Env == "" {
return nil, fmt.Errorf("--env is required\nUsage: orama node rollout --env <devnet|testnet>")
}
return flags, nil
}
func execute(flags *Flags) error {
start := time.Now()
fmt.Printf("Rollout to %s\n", flags.Env)
fmt.Printf(" Build: %s\n", boolStr(!flags.NoBuild, "yes", "skip"))
fmt.Printf(" Delay: %ds between nodes\n\n", flags.Delay)
// Step 1: Build
if !flags.NoBuild {
fmt.Printf("Step 1/3: Building binary archive...\n\n")
buildFlags := &build.Flags{
Arch: "amd64",
}
builder := build.NewBuilder(buildFlags)
if err := builder.Build(); err != nil {
return fmt.Errorf("build failed: %w", err)
}
fmt.Println()
} else {
fmt.Printf("Step 1/3: Build skipped (--no-build)\n\n")
}
// Step 2: Push
fmt.Printf("Step 2/3: Pushing to all %s nodes...\n\n", flags.Env)
push.Handle([]string{"--env", flags.Env})
fmt.Println()
// Step 3: Rolling upgrade
fmt.Printf("Step 3/3: Rolling upgrade across %s...\n\n", flags.Env)
upgrade.Handle([]string{"--env", flags.Env, "--delay", fmt.Sprintf("%d", flags.Delay)})
elapsed := time.Since(start).Round(time.Second)
fmt.Printf("\nRollout complete in %s\n", elapsed)
return nil
}
func boolStr(b bool, trueStr, falseStr string) string {
if b {
return trueStr
}
return falseStr
}

166
pkg/cli/production/unlock/command.go Normal file
View File

@ -0,0 +1,166 @@
// Package unlock implements the genesis node unlock command.
//
// When the genesis OramaOS node reboots before enough peers exist for
// Shamir-based LUKS key reconstruction, the operator must manually provide
// the LUKS key. This command reads the encrypted genesis key from the
// node's rootfs, decrypts it with the rootwallet, and sends it to the agent.
package unlock
import (
"bytes"
"encoding/base64"
"encoding/json"
"flag"
"fmt"
"io"
"net/http"
"os"
"os/exec"
"strings"
"time"
)
// Flags holds parsed command-line flags.
type Flags struct {
NodeIP string // WireGuard IP of the OramaOS node
Genesis bool // Must be set to confirm genesis unlock
KeyFile string // Path to the encrypted genesis key file (optional override)
}
// Handle processes the unlock command.
func Handle(args []string) {
flags, err := parseFlags(args)
if err != nil {
fmt.Fprintf(os.Stderr, "Error: %v\n", err)
os.Exit(1)
}
if !flags.Genesis {
fmt.Fprintf(os.Stderr, "Error: --genesis flag is required to confirm genesis unlock\n")
os.Exit(1)
}
// Step 1: Read the encrypted genesis key from the node
fmt.Printf("Fetching encrypted genesis key from %s...\n", flags.NodeIP)
encKey, err := fetchGenesisKey(flags.NodeIP)
if err != nil && flags.KeyFile == "" {
fmt.Fprintf(os.Stderr, "Error: could not fetch genesis key from node: %v\n", err)
fmt.Fprintf(os.Stderr, "You can provide the key file directly with --key-file\n")
os.Exit(1)
}
if flags.KeyFile != "" {
data, readErr := os.ReadFile(flags.KeyFile)
if readErr != nil {
fmt.Fprintf(os.Stderr, "Error: could not read key file: %v\n", readErr)
os.Exit(1)
}
encKey = strings.TrimSpace(string(data))
}
// Step 2: Decrypt with rootwallet
fmt.Println("Decrypting genesis key with rootwallet...")
luksKey, err := decryptGenesisKey(encKey)
if err != nil {
fmt.Fprintf(os.Stderr, "Error: decryption failed: %v\n", err)
os.Exit(1)
}
// Step 3: Send LUKS key to the agent over WireGuard
fmt.Printf("Sending LUKS key to agent at %s:9998...\n", flags.NodeIP)
if err := sendUnlockKey(flags.NodeIP, luksKey); err != nil {
fmt.Fprintf(os.Stderr, "Error: unlock failed: %v\n", err)
os.Exit(1)
}
fmt.Println("Genesis node unlocked successfully.")
fmt.Println("The node is decrypting and mounting its data partition.")
}
func parseFlags(args []string) (*Flags, error) {
fs := flag.NewFlagSet("unlock", flag.ContinueOnError)
fs.SetOutput(os.Stderr)
flags := &Flags{}
fs.StringVar(&flags.NodeIP, "node-ip", "", "WireGuard IP of the OramaOS node (required)")
fs.BoolVar(&flags.Genesis, "genesis", false, "Confirm genesis node unlock")
fs.StringVar(&flags.KeyFile, "key-file", "", "Path to encrypted genesis key file (optional)")
if err := fs.Parse(args); err != nil {
return nil, err
}
if flags.NodeIP == "" {
return nil, fmt.Errorf("--node-ip is required")
}
return flags, nil
}
// fetchGenesisKey retrieves the encrypted genesis key from the node.
// The agent serves it at GET /v1/agent/genesis-key (only during genesis unlock mode).
func fetchGenesisKey(nodeIP string) (string, error) {
client := &http.Client{Timeout: 10 * time.Second}
resp, err := client.Get(fmt.Sprintf("http://%s:9998/v1/agent/genesis-key", nodeIP))
if err != nil {
return "", fmt.Errorf("request failed: %w", err)
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
body, _ := io.ReadAll(resp.Body)
return "", fmt.Errorf("status %d: %s", resp.StatusCode, string(body))
}
var result struct {
EncryptedKey string `json:"encrypted_key"`
}
if err := json.NewDecoder(resp.Body).Decode(&result); err != nil {
return "", fmt.Errorf("invalid response: %w", err)
}
return result.EncryptedKey, nil
}
// decryptGenesisKey decrypts the AES-256-GCM encrypted LUKS key using rootwallet.
// The key was encrypted with: AES-256-GCM(luksKey, HKDF(rootwalletKey, "genesis-luks"))
// For now, we use `rw decrypt` if available, or a local HKDF+AES-GCM implementation.
func decryptGenesisKey(encryptedKey string) ([]byte, error) {
// Try rw decrypt first
cmd := exec.Command("rw", "decrypt", encryptedKey, "--purpose", "genesis-luks", "--chain", "evm")
output, err := cmd.Output()
if err == nil {
decoded, decErr := base64.StdEncoding.DecodeString(strings.TrimSpace(string(output)))
if decErr != nil {
return nil, fmt.Errorf("failed to decode decrypted key: %w", decErr)
}
return decoded, nil
}
return nil, fmt.Errorf("rw decrypt failed: %w (is rootwallet installed and initialized?)", err)
}
// sendUnlockKey sends the decrypted LUKS key to the agent's unlock endpoint.
func sendUnlockKey(nodeIP string, luksKey []byte) error {
body, _ := json.Marshal(map[string]string{
"key": base64.StdEncoding.EncodeToString(luksKey),
})
client := &http.Client{Timeout: 30 * time.Second}
resp, err := client.Post(
fmt.Sprintf("http://%s:9998/v1/agent/unlock", nodeIP),
"application/json",
bytes.NewReader(body),
)
if err != nil {
return fmt.Errorf("request failed: %w", err)
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
respBody, _ := io.ReadAll(resp.Body)
return fmt.Errorf("status %d: %s", resp.StatusCode, string(respBody))
}
return nil
}

12
pkg/cli/production/upgrade/command.go
View File

@ -14,7 +14,17 @@ func Handle(args []string) {
os.Exit(1)
}
// Check root privileges
// Remote rolling upgrade when --env is specified
if flags.Env != "" {
remote := NewRemoteUpgrader(flags)
if err := remote.Execute(); err != nil {
fmt.Fprintf(os.Stderr, "❌ %v\n", err)
os.Exit(1)
}
return
}
// Local upgrade: requires root
if os.Geteuid() != 0 {
fmt.Fprintf(os.Stderr, "❌ Production upgrade must be run as root (use sudo)\n")
os.Exit(1)

10
pkg/cli/production/upgrade/flags.go
View File

@ -13,6 +13,11 @@ type Flags struct {
SkipChecks bool
Nameserver *bool // Pointer so we can detect if explicitly set vs default
// Remote upgrade flags
Env string // Target environment for remote rolling upgrade
NodeFilter string // Single node IP to upgrade (optional)
Delay int // Delay in seconds between nodes during rolling upgrade
// Anyone flags
AnyoneClient bool
AnyoneRelay bool
@ -38,6 +43,11 @@ func ParseFlags(args []string) (*Flags, error) {
fs.BoolVar(&flags.RestartServices, "restart", false, "Automatically restart services after upgrade")
fs.BoolVar(&flags.SkipChecks, "skip-checks", false, "Skip minimum resource checks (RAM/CPU)")
// Remote upgrade flags
fs.StringVar(&flags.Env, "env", "", "Target environment for remote rolling upgrade (devnet, testnet)")
fs.StringVar(&flags.NodeFilter, "node", "", "Upgrade a single node IP only")
fs.IntVar(&flags.Delay, "delay", 30, "Delay in seconds between nodes during rolling upgrade")
// Nameserver flag - use pointer to detect if explicitly set
nameserver := fs.Bool("nameserver", false, "Make this node a nameserver (uses saved preference if not specified)")

26
pkg/cli/production/upgrade/orchestrator.go
View File

@ -41,7 +41,8 @@ func NewOrchestrator(flags *Flags) *Orchestrator {
setup := production.NewProductionSetup(oramaHome, os.Stdout, flags.Force, flags.SkipChecks)
setup.SetNameserver(isNameserver)
// Configure Anyone mode (flag > saved preference > auto-detect)
// Configure Anyone mode (explicit flags > saved preferences > auto-detect)
// Explicit flags always win — they represent the user's current intent.
if flags.AnyoneRelay {
setup.SetAnyoneRelayConfig(&production.AnyoneRelayConfig{
Enabled: true,
@ -55,6 +56,9 @@ func NewOrchestrator(flags *Flags) *Orchestrator {
BandwidthPct: flags.AnyoneBandwidth,
AccountingMax: flags.AnyoneAccounting,
})
} else if flags.AnyoneClient {
// Explicit --anyone-client flag overrides saved relay prefs and auto-detect.
setup.SetAnyoneClient(true)
} else if prefs.AnyoneRelay {
// Restore relay config from saved preferences (for firewall rules)
orPort := prefs.AnyoneORPort
@ -65,6 +69,8 @@ func NewOrchestrator(flags *Flags) *Orchestrator {
Enabled: true,
ORPort: orPort,
})
} else if prefs.AnyoneClient {
setup.SetAnyoneClient(true)
} else if detectAnyoneRelay(oramaDir) {
// Auto-detect: relay is installed but preferences weren't saved.
// This happens when upgrading from older versions that didn't persist
@ -79,8 +85,6 @@ func NewOrchestrator(flags *Flags) *Orchestrator {
prefs.AnyoneORPort = orPort
_ = production.SavePreferences(oramaDir, prefs)
fmt.Printf(" Auto-detected Anyone relay (ORPort: %d), saved to preferences\n", orPort)
} else if flags.AnyoneClient || prefs.AnyoneClient {
setup.SetAnyoneClient(true)
}
return &Orchestrator{
@ -207,15 +211,15 @@ func (o *Orchestrator) handleBranchPreferences() error {
fmt.Printf(" Nameserver mode: enabled (CoreDNS + Caddy)\n")
}
// If anyone-client was explicitly provided, update it
// Anyone client and relay are mutually exclusive — setting one clears the other.
if o.flags.AnyoneClient {
prefs.AnyoneClient = true
prefs.AnyoneRelay = false
prefs.AnyoneORPort = 0
prefsChanged = true
}
// If anyone-relay was explicitly provided, update it
if o.flags.AnyoneRelay {
} else if o.flags.AnyoneRelay {
prefs.AnyoneRelay = true
prefs.AnyoneClient = false
prefs.AnyoneORPort = o.flags.AnyoneORPort
if prefs.AnyoneORPort == 0 {
prefs.AnyoneORPort = 9001
@ -424,7 +428,11 @@ func (o *Orchestrator) stopAllNamespaceServices(serviceController *production.Sy
// installNamespaceTemplates installs systemd template unit files for namespace services
func (o *Orchestrator) installNamespaceTemplates() error {
sourceDir := filepath.Join(o.oramaHome, "src", "systemd")
// Check pre-built archive path first, fall back to source path
sourceDir := production.OramaSystemdDir
if _, err := os.Stat(sourceDir); os.IsNotExist(err) {
sourceDir = filepath.Join(o.oramaHome, "src", "systemd")
}
systemdDir := "/etc/systemd/system"
templates := []string{

75
pkg/cli/production/upgrade/remote.go Normal file
View File

@ -0,0 +1,75 @@
package upgrade
import (
"fmt"
"time"
"github.com/DeBrosOfficial/network/pkg/cli/remotessh"
"github.com/DeBrosOfficial/network/pkg/inspector"
)
// RemoteUpgrader handles rolling upgrades across remote nodes.
type RemoteUpgrader struct {
flags *Flags
}
// NewRemoteUpgrader creates a new remote upgrader.
func NewRemoteUpgrader(flags *Flags) *RemoteUpgrader {
return &RemoteUpgrader{flags: flags}
}
// Execute runs the remote rolling upgrade.
func (r *RemoteUpgrader) Execute() error {
nodes, err := remotessh.LoadEnvNodes(r.flags.Env)
if err != nil {
return err
}
cleanup, err := remotessh.PrepareNodeKeys(nodes)
if err != nil {
return err
}
defer cleanup()
// Filter to single node if specified
if r.flags.NodeFilter != "" {
nodes = remotessh.FilterByIP(nodes, r.flags.NodeFilter)
if len(nodes) == 0 {
return fmt.Errorf("node %s not found in %s environment", r.flags.NodeFilter, r.flags.Env)
}
}
fmt.Printf("Rolling upgrade: %s (%d nodes, %ds delay)\n\n", r.flags.Env, len(nodes), r.flags.Delay)
// Print execution plan
for i, node := range nodes {
fmt.Printf(" %d. %s (%s)\n", i+1, node.Host, node.Role)
}
fmt.Println()
for i, node := range nodes {
fmt.Printf("[%d/%d] Upgrading %s (%s)...\n", i+1, len(nodes), node.Host, node.Role)
if err := r.upgradeNode(node); err != nil {
return fmt.Errorf("upgrade failed on %s: %w\nStopping rollout — remaining nodes not upgraded", node.Host, err)
}
fmt.Printf(" ✓ %s upgraded\n", node.Host)
// Wait between nodes (except after the last one)
if i < len(nodes)-1 && r.flags.Delay > 0 {
fmt.Printf(" Waiting %ds before next node...\n\n", r.flags.Delay)
time.Sleep(time.Duration(r.flags.Delay) * time.Second)
}
}
fmt.Printf("\n✓ Rolling upgrade complete (%d nodes)\n", len(nodes))
return nil
}
// upgradeNode runs `orama node upgrade --restart` on a single remote node.
func (r *RemoteUpgrader) upgradeNode(node inspector.Node) error {
sudo := remotessh.SudoPrefix(node)
cmd := fmt.Sprintf("%sorama node upgrade --restart", sudo)
return remotessh.RunSSHStreaming(node, cmd)
}

69
pkg/cli/remotessh/config.go Normal file
View File

@ -0,0 +1,69 @@
package remotessh
import (
"fmt"
"os"
"path/filepath"
"github.com/DeBrosOfficial/network/pkg/inspector"
)
// FindNodesConf searches for the nodes.conf file
// in common locations relative to the current directory or project root.
func FindNodesConf() string {
candidates := []string{
"scripts/nodes.conf",
"../scripts/nodes.conf",
"network/scripts/nodes.conf",
}
// Also check from home dir
home, _ := os.UserHomeDir()
if home != "" {
candidates = append(candidates, filepath.Join(home, ".orama", "nodes.conf"))
}
for _, c := range candidates {
if _, err := os.Stat(c); err == nil {
return c
}
}
return ""
}
// LoadEnvNodes loads all nodes for a given environment from nodes.conf.
// SSHKey fields are NOT set — caller must call PrepareNodeKeys() after this.
func LoadEnvNodes(env string) ([]inspector.Node, error) {
confPath := FindNodesConf()
if confPath == "" {
return nil, fmt.Errorf("nodes.conf not found (checked scripts/, ../scripts/, network/scripts/)")
}
nodes, err := inspector.LoadNodes(confPath)
if err != nil {
return nil, fmt.Errorf("failed to load %s: %w", confPath, err)
}
filtered := inspector.FilterByEnv(nodes, env)
if len(filtered) == 0 {
return nil, fmt.Errorf("no nodes found for environment %q in %s", env, confPath)
}
return filtered, nil
}
// PickHubNode selects the first node as the hub for fanout distribution.
func PickHubNode(nodes []inspector.Node) inspector.Node {
return nodes[0]
}
// FilterByIP returns nodes matching the given IP address.
func FilterByIP(nodes []inspector.Node, ip string) []inspector.Node {
var filtered []inspector.Node
for _, n := range nodes {
if n.Host == ip {
filtered = append(filtered, n)
}
}
return filtered
}

104
pkg/cli/remotessh/ssh.go Normal file
View File

@ -0,0 +1,104 @@
package remotessh
import (
"fmt"
"os"
"os/exec"
"github.com/DeBrosOfficial/network/pkg/inspector"
)
// SSHOption configures SSH command behavior.
type SSHOption func(*sshOptions)
type sshOptions struct {
agentForward bool
noHostKeyCheck bool
}
// WithAgentForward enables SSH agent forwarding (-A flag).
// Used by push fanout so the hub can reach targets via the forwarded agent.
func WithAgentForward() SSHOption {
return func(o *sshOptions) { o.agentForward = true }
}
// WithNoHostKeyCheck disables host key verification and uses /dev/null as known_hosts.
// Use for ephemeral servers (sandbox) where IPs are frequently recycled.
func WithNoHostKeyCheck() SSHOption {
return func(o *sshOptions) { o.noHostKeyCheck = true }
}
// UploadFile copies a local file to a remote host via SCP.
// Requires node.SSHKey to be set (via PrepareNodeKeys).
func UploadFile(node inspector.Node, localPath, remotePath string, opts ...SSHOption) error {
if node.SSHKey == "" {
return fmt.Errorf("no SSH key for %s (call PrepareNodeKeys first)", node.Name())
}
var cfg sshOptions
for _, o := range opts {
o(&cfg)
}
dest := fmt.Sprintf("%s@%s:%s", node.User, node.Host, remotePath)
args := []string{"-o", "ConnectTimeout=10", "-i", node.SSHKey}
if cfg.noHostKeyCheck {
args = append([]string{"-o", "StrictHostKeyChecking=no", "-o", "UserKnownHostsFile=/dev/null"}, args...)
} else {
args = append([]string{"-o", "StrictHostKeyChecking=accept-new"}, args...)
}
args = append(args, localPath, dest)
cmd := exec.Command("scp", args...)
cmd.Stdout = os.Stdout
cmd.Stderr = os.Stderr
if err := cmd.Run(); err != nil {
return fmt.Errorf("SCP to %s failed: %w", node.Host, err)
}
return nil
}
// RunSSHStreaming executes a command on a remote host via SSH,
// streaming stdout/stderr to the local terminal in real-time.
// Requires node.SSHKey to be set (via PrepareNodeKeys).
func RunSSHStreaming(node inspector.Node, command string, opts ...SSHOption) error {
if node.SSHKey == "" {
return fmt.Errorf("no SSH key for %s (call PrepareNodeKeys first)", node.Name())
}
var cfg sshOptions
for _, o := range opts {
o(&cfg)
}
args := []string{"-o", "ConnectTimeout=10", "-i", node.SSHKey}
if cfg.noHostKeyCheck {
args = append([]string{"-o", "StrictHostKeyChecking=no", "-o", "UserKnownHostsFile=/dev/null"}, args...)
} else {
args = append([]string{"-o", "StrictHostKeyChecking=accept-new"}, args...)
}
if cfg.agentForward {
args = append(args, "-A")
}
args = append(args, fmt.Sprintf("%s@%s", node.User, node.Host), command)
cmd := exec.Command("ssh", args...)
cmd.Stdout = os.Stdout
cmd.Stderr = os.Stderr
cmd.Stdin = os.Stdin
if err := cmd.Run(); err != nil {
return fmt.Errorf("SSH to %s failed: %w", node.Host, err)
}
return nil
}
// SudoPrefix returns "sudo " for non-root users, empty for root.
func SudoPrefix(node inspector.Node) string {
if node.User == "root" {
return ""
}
return "sudo "
}

242
pkg/cli/remotessh/wallet.go Normal file
View File

@ -0,0 +1,242 @@
package remotessh
import (
"fmt"
"os"
"os/exec"
"path/filepath"
"strings"
"github.com/DeBrosOfficial/network/pkg/inspector"
)
// PrepareNodeKeys resolves wallet-derived SSH keys for all nodes.
// Calls `rw vault ssh get <host>/<user> --priv` for each unique host/user,
// writes PEMs to temp files, and sets node.SSHKey for each node.
//
// The nodes slice is modified in place — each node.SSHKey is set to
// the path of the temporary key file.
//
// Returns a cleanup function that zero-overwrites and removes all temp files.
// Caller must defer cleanup().
func PrepareNodeKeys(nodes []inspector.Node) (cleanup func(), err error) {
rw, err := rwBinary()
if err != nil {
return nil, err
}
// Create temp dir for all keys
tmpDir, err := os.MkdirTemp("", "orama-ssh-")
if err != nil {
return nil, fmt.Errorf("create temp dir: %w", err)
}
// Track resolved keys by host/user to avoid duplicate rw calls
keyPaths := make(map[string]string) // "host/user" → temp file path
var allKeyPaths []string
for i := range nodes {
// Use VaultTarget if set, otherwise default to Host/User
var key string
if nodes[i].VaultTarget != "" {
key = nodes[i].VaultTarget
} else {
key = nodes[i].Host + "/" + nodes[i].User
}
if existing, ok := keyPaths[key]; ok {
nodes[i].SSHKey = existing
continue
}
// Call rw to get the private key PEM
host, user := parseVaultTarget(key)
pem, err := resolveWalletKey(rw, host, user)
if err != nil {
// Cleanup any keys already written before returning error
cleanupKeys(tmpDir, allKeyPaths)
return nil, fmt.Errorf("resolve key for %s: %w", nodes[i].Name(), err)
}
// Write PEM to temp file with restrictive perms
keyFile := filepath.Join(tmpDir, fmt.Sprintf("id_%d", i))
if err := os.WriteFile(keyFile, []byte(pem), 0600); err != nil {
cleanupKeys(tmpDir, allKeyPaths)
return nil, fmt.Errorf("write key for %s: %w", nodes[i].Name(), err)
}
keyPaths[key] = keyFile
allKeyPaths = append(allKeyPaths, keyFile)
nodes[i].SSHKey = keyFile
}
cleanup = func() {
cleanupKeys(tmpDir, allKeyPaths)
}
return cleanup, nil
}
// LoadAgentKeys loads SSH keys for the given nodes into the system ssh-agent.
// Used by push fanout to enable agent forwarding.
// Calls `rw vault ssh agent-load <host1/user1> <host2/user2> ...`
func LoadAgentKeys(nodes []inspector.Node) error {
rw, err := rwBinary()
if err != nil {
return err
}
// Deduplicate host/user pairs
seen := make(map[string]bool)
var targets []string
for _, n := range nodes {
var key string
if n.VaultTarget != "" {
key = n.VaultTarget
} else {
key = n.Host + "/" + n.User
}
if seen[key] {
continue
}
seen[key] = true
targets = append(targets, key)
}
if len(targets) == 0 {
return nil
}
args := append([]string{"vault", "ssh", "agent-load"}, targets...)
cmd := exec.Command(rw, args...)
cmd.Stderr = os.Stderr
cmd.Stdout = os.Stderr // info messages go to stderr
if err := cmd.Run(); err != nil {
return fmt.Errorf("rw vault ssh agent-load failed: %w", err)
}
return nil
}
// EnsureVaultEntry creates a wallet SSH entry if it doesn't already exist.
// Checks existence via `rw vault ssh get <target> --pub`, and if missing,
// runs `rw vault ssh add <target>` to create it.
func EnsureVaultEntry(vaultTarget string) error {
rw, err := rwBinary()
if err != nil {
return err
}
// Check if entry exists by trying to get the public key
cmd := exec.Command(rw, "vault", "ssh", "get", vaultTarget, "--pub")
if err := cmd.Run(); err == nil {
return nil // entry already exists
}
// Entry doesn't exist — try to create it
addCmd := exec.Command(rw, "vault", "ssh", "add", vaultTarget)
addCmd.Stdin = os.Stdin
addCmd.Stdout = os.Stderr
addCmd.Stderr = os.Stderr
if err := addCmd.Run(); err != nil {
if exitErr, ok := err.(*exec.ExitError); ok {
stderr := strings.TrimSpace(string(exitErr.Stderr))
if strings.Contains(stderr, "not unlocked") || strings.Contains(stderr, "session") {
return fmt.Errorf("wallet is locked — run: rw unlock")
}
}
return fmt.Errorf("rw vault ssh add %s failed: %w", vaultTarget, err)
}
return nil
}
// ResolveVaultPublicKey returns the OpenSSH public key string for a vault entry.
// Calls `rw vault ssh get <target> --pub`.
func ResolveVaultPublicKey(vaultTarget string) (string, error) {
rw, err := rwBinary()
if err != nil {
return "", err
}
cmd := exec.Command(rw, "vault", "ssh", "get", vaultTarget, "--pub")
out, err := cmd.Output()
if err != nil {
if exitErr, ok := err.(*exec.ExitError); ok {
stderr := strings.TrimSpace(string(exitErr.Stderr))
if strings.Contains(stderr, "No SSH entry") {
return "", fmt.Errorf("no vault SSH entry for %s — run: rw vault ssh add %s", vaultTarget, vaultTarget)
}
if strings.Contains(stderr, "not unlocked") || strings.Contains(stderr, "session") {
return "", fmt.Errorf("wallet is locked — run: rw unlock")
}
return "", fmt.Errorf("%s", stderr)
}
return "", fmt.Errorf("rw command failed: %w", err)
}
pubKey := strings.TrimSpace(string(out))
if !strings.HasPrefix(pubKey, "ssh-") {
return "", fmt.Errorf("rw returned invalid public key for %s", vaultTarget)
}
return pubKey, nil
}
// parseVaultTarget splits a "host/user" vault target string into host and user.
func parseVaultTarget(target string) (host, user string) {
idx := strings.Index(target, "/")
if idx < 0 {
return target, ""
}
return target[:idx], target[idx+1:]
}
// resolveWalletKey calls `rw vault ssh get <host>/<user> --priv`
// and returns the PEM string. Requires an active rw session.
func resolveWalletKey(rw string, host, user string) (string, error) {
target := host + "/" + user
cmd := exec.Command(rw, "vault", "ssh", "get", target, "--priv")
out, err := cmd.Output()
if err != nil {
if exitErr, ok := err.(*exec.ExitError); ok {
stderr := strings.TrimSpace(string(exitErr.Stderr))
if strings.Contains(stderr, "No SSH entry") {
return "", fmt.Errorf("no vault SSH entry for %s — run: rw vault ssh add %s", target, target)
}
if strings.Contains(stderr, "not unlocked") || strings.Contains(stderr, "session") {
return "", fmt.Errorf("wallet is locked — run: rw unlock")
}
return "", fmt.Errorf("%s", stderr)
}
return "", fmt.Errorf("rw command failed: %w", err)
}
pem := string(out)
if !strings.Contains(pem, "BEGIN OPENSSH PRIVATE KEY") {
return "", fmt.Errorf("rw returned invalid key for %s", target)
}
return pem, nil
}
// rwBinary returns the path to the `rw` binary.
// Checks RW_PATH env var first, then PATH.
func rwBinary() (string, error) {
if p := os.Getenv("RW_PATH"); p != "" {
if _, err := os.Stat(p); err == nil {
return p, nil
}
return "", fmt.Errorf("RW_PATH=%q not found", p)
}
p, err := exec.LookPath("rw")
if err != nil {
return "", fmt.Errorf("rw not found in PATH — install rootwallet CLI: https://github.com/DeBrosOfficial/rootwallet")
}
return p, nil
}
// cleanupKeys zero-overwrites and removes all key files, then removes the temp dir.
func cleanupKeys(tmpDir string, keyPaths []string) {
zeros := make([]byte, 512)
for _, p := range keyPaths {
_ = os.WriteFile(p, zeros, 0600) // zero-overwrite
_ = os.Remove(p)
}
_ = os.Remove(tmpDir)
}

29
pkg/cli/remotessh/wallet_test.go Normal file
View File

@ -0,0 +1,29 @@
package remotessh
import "testing"
func TestParseVaultTarget(t *testing.T) {
tests := []struct {
target string
wantHost string
wantUser string
}{
{"sandbox/root", "sandbox", "root"},
{"192.168.1.1/ubuntu", "192.168.1.1", "ubuntu"},
{"my-host/my-user", "my-host", "my-user"},
{"noslash", "noslash", ""},
{"a/b/c", "a", "b/c"},
}
for _, tt := range tests {
t.Run(tt.target, func(t *testing.T) {
host, user := parseVaultTarget(tt.target)
if host != tt.wantHost {
t.Errorf("parseVaultTarget(%q) host = %q, want %q", tt.target, host, tt.wantHost)
}
if user != tt.wantUser {
t.Errorf("parseVaultTarget(%q) user = %q, want %q", tt.target, user, tt.wantUser)
}
})
}
}

133
pkg/cli/sandbox/config.go Normal file
View File

@ -0,0 +1,133 @@
package sandbox
import (
"fmt"
"os"
"path/filepath"
"gopkg.in/yaml.v3"
)
// Config holds sandbox configuration, stored at ~/.orama/sandbox.yaml.
type Config struct {
HetznerAPIToken string `yaml:"hetzner_api_token"`
Domain string `yaml:"domain"`
Location string `yaml:"location"` // Hetzner datacenter (default: fsn1)
ServerType string `yaml:"server_type"` // Hetzner server type (default: cx22)
FloatingIPs []FloatIP `yaml:"floating_ips"`
SSHKey SSHKeyConfig `yaml:"ssh_key"`
FirewallID int64 `yaml:"firewall_id,omitempty"` // Hetzner firewall resource ID
}
// FloatIP holds a Hetzner floating IP reference.
type FloatIP struct {
ID int64 `yaml:"id"`
IP string `yaml:"ip"`
}
// SSHKeyConfig holds the wallet vault target and Hetzner resource ID.
type SSHKeyConfig struct {
HetznerID int64 `yaml:"hetzner_id"`
VaultTarget string `yaml:"vault_target"` // e.g. "sandbox/root"
}
// configDir returns ~/.orama/, creating it if needed.
func configDir() (string, error) {
home, err := os.UserHomeDir()
if err != nil {
return "", fmt.Errorf("get home directory: %w", err)
}
dir := filepath.Join(home, ".orama")
if err := os.MkdirAll(dir, 0700); err != nil {
return "", fmt.Errorf("create config directory: %w", err)
}
return dir, nil
}
// configPath returns the full path to ~/.orama/sandbox.yaml.
func configPath() (string, error) {
dir, err := configDir()
if err != nil {
return "", err
}
return filepath.Join(dir, "sandbox.yaml"), nil
}
// LoadConfig reads the sandbox config from ~/.orama/sandbox.yaml.
// Returns an error if the file doesn't exist (user must run setup first).
func LoadConfig() (*Config, error) {
path, err := configPath()
if err != nil {
return nil, err
}
data, err := os.ReadFile(path)
if err != nil {
if os.IsNotExist(err) {
return nil, fmt.Errorf("sandbox not configured, run: orama sandbox setup")
}
return nil, fmt.Errorf("read config: %w", err)
}
var cfg Config
if err := yaml.Unmarshal(data, &cfg); err != nil {
return nil, fmt.Errorf("parse config %s: %w", path, err)
}
if err := cfg.validate(); err != nil {
return nil, fmt.Errorf("invalid config: %w", err)
}
cfg.Defaults()
return &cfg, nil
}
// SaveConfig writes the sandbox config to ~/.orama/sandbox.yaml.
func SaveConfig(cfg *Config) error {
path, err := configPath()
if err != nil {
return err
}
data, err := yaml.Marshal(cfg)
if err != nil {
return fmt.Errorf("marshal config: %w", err)
}
if err := os.WriteFile(path, data, 0600); err != nil {
return fmt.Errorf("write config: %w", err)
}
return nil
}
// validate checks that required fields are present.
func (c *Config) validate() error {
if c.HetznerAPIToken == "" {
return fmt.Errorf("hetzner_api_token is required")
}
if c.Domain == "" {
return fmt.Errorf("domain is required")
}
if len(c.FloatingIPs) < 2 {
return fmt.Errorf("2 floating IPs required, got %d", len(c.FloatingIPs))
}
if c.SSHKey.VaultTarget == "" {
return fmt.Errorf("ssh_key.vault_target is required (run: orama sandbox setup)")
}
return nil
}
// Defaults fills in default values for optional fields.
func (c *Config) Defaults() {
if c.Location == "" {
c.Location = "nbg1"
}
if c.ServerType == "" {
c.ServerType = "cx23"
}
if c.SSHKey.VaultTarget == "" {
c.SSHKey.VaultTarget = "sandbox/root"
}
}

53
pkg/cli/sandbox/config_test.go Normal file
View File

@ -0,0 +1,53 @@
package sandbox
import "testing"
func TestConfig_Validate_EmptyVaultTarget(t *testing.T) {
cfg := &Config{
HetznerAPIToken: "test-token",
Domain: "test.example.com",
FloatingIPs: []FloatIP{{ID: 1, IP: "1.1.1.1"}, {ID: 2, IP: "2.2.2.2"}},
SSHKey: SSHKeyConfig{HetznerID: 1, VaultTarget: ""},
}
if err := cfg.validate(); err == nil {
t.Error("validate() should reject empty VaultTarget")
}
}
func TestConfig_Validate_WithVaultTarget(t *testing.T) {
cfg := &Config{
HetznerAPIToken: "test-token",
Domain: "test.example.com",
FloatingIPs: []FloatIP{{ID: 1, IP: "1.1.1.1"}, {ID: 2, IP: "2.2.2.2"}},
SSHKey: SSHKeyConfig{HetznerID: 1, VaultTarget: "sandbox/root"},
}
if err := cfg.validate(); err != nil {
t.Errorf("validate() unexpected error: %v", err)
}
}
func TestConfig_Defaults_SetsVaultTarget(t *testing.T) {
cfg := &Config{}
cfg.Defaults()
if cfg.SSHKey.VaultTarget != "sandbox/root" {
t.Errorf("Defaults() VaultTarget = %q, want sandbox/root", cfg.SSHKey.VaultTarget)
}
if cfg.Location != "nbg1" {
t.Errorf("Defaults() Location = %q, want nbg1", cfg.Location)
}
if cfg.ServerType != "cx23" {
t.Errorf("Defaults() ServerType = %q, want cx23", cfg.ServerType)
}
}
func TestConfig_Defaults_PreservesExistingVaultTarget(t *testing.T) {
cfg := &Config{
SSHKey: SSHKeyConfig{VaultTarget: "custom/user"},
}
cfg.Defaults()
if cfg.SSHKey.VaultTarget != "custom/user" {
t.Errorf("Defaults() should preserve existing VaultTarget, got %q", cfg.SSHKey.VaultTarget)
}
}

546
pkg/cli/sandbox/create.go Normal file
View File

@ -0,0 +1,546 @@
package sandbox
import (
"fmt"
"os"
"os/exec"
"path/filepath"
"strings"
"time"
"github.com/DeBrosOfficial/network/pkg/cli/remotessh"
"github.com/DeBrosOfficial/network/pkg/inspector"
)
// Create orchestrates the creation of a new sandbox cluster.
func Create(name string) error {
cfg, err := LoadConfig()
if err != nil {
return err
}
// Resolve wallet SSH key once for all phases
sshKeyPath, cleanup, err := resolveVaultKeyOnce(cfg.SSHKey.VaultTarget)
if err != nil {
return fmt.Errorf("prepare SSH key: %w", err)
}
defer cleanup()
// Check for existing active sandbox
active, err := FindActiveSandbox()
if err != nil {
return err
}
if active != nil {
return fmt.Errorf("sandbox %q is already active (status: %s)\nDestroy it first: orama sandbox destroy --name %s",
active.Name, active.Status, active.Name)
}
// Generate name if not provided
if name == "" {
name = GenerateName()
}
fmt.Printf("Creating sandbox %q (%s, %d nodes)\n\n", name, cfg.Domain, 5)
client := NewHetznerClient(cfg.HetznerAPIToken)
state := &SandboxState{
Name: name,
CreatedAt: time.Now().UTC(),
Domain: cfg.Domain,
Status: StatusCreating,
}
// Phase 1: Provision servers
fmt.Println("Phase 1: Provisioning servers...")
if err := phase1ProvisionServers(client, cfg, state); err != nil {
cleanupFailedCreate(client, state)
return fmt.Errorf("provision servers: %w", err)
}
SaveState(state)
// Phase 2: Assign floating IPs
fmt.Println("\nPhase 2: Assigning floating IPs...")
if err := phase2AssignFloatingIPs(client, cfg, state, sshKeyPath); err != nil {
return fmt.Errorf("assign floating IPs: %w", err)
}
SaveState(state)
// Phase 3: Upload binary archive
fmt.Println("\nPhase 3: Uploading binary archive...")
if err := phase3UploadArchive(state, sshKeyPath); err != nil {
return fmt.Errorf("upload archive: %w", err)
}
// Phase 4: Install genesis node
fmt.Println("\nPhase 4: Installing genesis node...")
tokens, err := phase4InstallGenesis(cfg, state, sshKeyPath)
if err != nil {
state.Status = StatusError
SaveState(state)
return fmt.Errorf("install genesis: %w", err)
}
// Phase 5: Join remaining nodes
fmt.Println("\nPhase 5: Joining remaining nodes...")
if err := phase5JoinNodes(cfg, state, tokens, sshKeyPath); err != nil {
state.Status = StatusError
SaveState(state)
return fmt.Errorf("join nodes: %w", err)
}
// Phase 6: Verify cluster
fmt.Println("\nPhase 6: Verifying cluster...")
phase6Verify(cfg, state, sshKeyPath)
state.Status = StatusRunning
SaveState(state)
printCreateSummary(cfg, state)
return nil
}
// resolveVaultKeyOnce resolves a wallet SSH key to a temp file.
// Returns the key path, cleanup function, and any error.
func resolveVaultKeyOnce(vaultTarget string) (string, func(), error) {
node := inspector.Node{User: "root", Host: "resolve-only", VaultTarget: vaultTarget}
nodes := []inspector.Node{node}
cleanup, err := remotessh.PrepareNodeKeys(nodes)
if err != nil {
return "", func() {}, err
}
return nodes[0].SSHKey, cleanup, nil
}
// phase1ProvisionServers creates 5 Hetzner servers in parallel.
func phase1ProvisionServers(client *HetznerClient, cfg *Config, state *SandboxState) error {
type serverResult struct {
index int
server *HetznerServer
err error
}
results := make(chan serverResult, 5)
for i := 0; i < 5; i++ {
go func(idx int) {
role := "node"
if idx < 2 {
role = "nameserver"
}
serverName := fmt.Sprintf("sbx-%s-%d", state.Name, idx+1)
labels := map[string]string{
"orama-sandbox": state.Name,
"orama-sandbox-role": role,
}
req := CreateServerRequest{
Name: serverName,
ServerType: cfg.ServerType,
Image: "ubuntu-24.04",
Location: cfg.Location,
SSHKeys: []int64{cfg.SSHKey.HetznerID},
Labels: labels,
}
if cfg.FirewallID > 0 {
req.Firewalls = []struct {
Firewall int64 `json:"firewall"`
}{{Firewall: cfg.FirewallID}}
}
srv, err := client.CreateServer(req)
results <- serverResult{index: idx, server: srv, err: err}
}(i)
}
servers := make([]ServerState, 5)
var firstErr error
for i := 0; i < 5; i++ {
r := <-results
if r.err != nil {
if firstErr == nil {
firstErr = fmt.Errorf("server %d: %w", r.index+1, r.err)
}
continue
}
fmt.Printf(" Created %s (ID: %d, initializing...)\n", r.server.Name, r.server.ID)
role := "node"
if r.index < 2 {
role = "nameserver"
}
servers[r.index] = ServerState{
ID: r.server.ID,
Name: r.server.Name,
Role: role,
}
}
state.Servers = servers // populate before returning so cleanup can delete created servers
if firstErr != nil {
return firstErr
}
// Wait for all servers to reach "running"
fmt.Print(" Waiting for servers to boot...")
for i := range servers {
srv, err := client.WaitForServer(servers[i].ID, 3*time.Minute)
if err != nil {
return fmt.Errorf("wait for %s: %w", servers[i].Name, err)
}
servers[i].IP = srv.PublicNet.IPv4.IP
fmt.Print(".")
}
fmt.Println(" OK")
// Assign floating IPs to nameserver entries
if len(cfg.FloatingIPs) >= 2 {
servers[0].FloatingIP = cfg.FloatingIPs[0].IP
servers[1].FloatingIP = cfg.FloatingIPs[1].IP
}
state.Servers = servers
for _, srv := range servers {
fmt.Printf(" %s: %s (%s)\n", srv.Name, srv.IP, srv.Role)
}
return nil
}
// phase2AssignFloatingIPs assigns floating IPs and configures loopback.
func phase2AssignFloatingIPs(client *HetznerClient, cfg *Config, state *SandboxState, sshKeyPath string) error {
for i := 0; i < 2 && i < len(cfg.FloatingIPs) && i < len(state.Servers); i++ {
fip := cfg.FloatingIPs[i]
srv := state.Servers[i]
// Unassign if currently assigned elsewhere (ignore "not assigned" errors)
fmt.Printf(" Assigning %s to %s...\n", fip.IP, srv.Name)
if err := client.UnassignFloatingIP(fip.ID); err != nil {
// Log but continue — may fail if not currently assigned, which is fine
fmt.Printf(" Note: unassign %s: %v (continuing)\n", fip.IP, err)
}
if err := client.AssignFloatingIP(fip.ID, srv.ID); err != nil {
return fmt.Errorf("assign %s to %s: %w", fip.IP, srv.Name, err)
}
// Configure floating IP on the server's loopback interface
// Hetzner floating IPs require this: ip addr add <floating_ip>/32 dev lo
node := inspector.Node{
User: "root",
Host: srv.IP,
SSHKey: sshKeyPath,
}
// Wait for SSH to be ready on freshly booted servers
if err := waitForSSH(node, 5*time.Minute); err != nil {
return fmt.Errorf("SSH not ready on %s: %w", srv.Name, err)
}
cmd := fmt.Sprintf("ip addr add %s/32 dev lo 2>/dev/null || true", fip.IP)
if err := remotessh.RunSSHStreaming(node, cmd, remotessh.WithNoHostKeyCheck()); err != nil {
return fmt.Errorf("configure loopback on %s: %w", srv.Name, err)
}
}
return nil
}
// waitForSSH polls until SSH is responsive on the node.
func waitForSSH(node inspector.Node, timeout time.Duration) error {
deadline := time.Now().Add(timeout)
for time.Now().Before(deadline) {
_, err := runSSHOutput(node, "echo ok")
if err == nil {
return nil
}
time.Sleep(3 * time.Second)
}
return fmt.Errorf("timeout after %s", timeout)
}
// phase3UploadArchive uploads the binary archive to the genesis node, then fans out
// to the remaining nodes server-to-server (much faster than uploading from local machine).
func phase3UploadArchive(state *SandboxState, sshKeyPath string) error {
archivePath := findNewestArchive()
if archivePath == "" {
fmt.Println(" No binary archive found, run `orama build` first")
return fmt.Errorf("no binary archive found in /tmp/ (run `orama build` first)")
}
info, _ := os.Stat(archivePath)
fmt.Printf(" Archive: %s (%s)\n", filepath.Base(archivePath), formatBytes(info.Size()))
if err := fanoutArchive(state.Servers, sshKeyPath, archivePath); err != nil {
return err
}
fmt.Println(" All nodes ready")
return nil
}
// phase4InstallGenesis installs the genesis node and generates invite tokens.
func phase4InstallGenesis(cfg *Config, state *SandboxState, sshKeyPath string) ([]string, error) {
genesis := state.GenesisServer()
node := inspector.Node{User: "root", Host: genesis.IP, SSHKey: sshKeyPath}
// Install genesis
installCmd := fmt.Sprintf("/opt/orama/bin/orama node install --vps-ip %s --domain %s --base-domain %s --nameserver --skip-checks",
genesis.IP, cfg.Domain, cfg.Domain)
fmt.Printf(" Installing on %s (%s)...\n", genesis.Name, genesis.IP)
if err := remotessh.RunSSHStreaming(node, installCmd, remotessh.WithNoHostKeyCheck()); err != nil {
return nil, fmt.Errorf("install genesis: %w", err)
}
// Wait for RQLite leader
fmt.Print(" Waiting for RQLite leader...")
if err := waitForRQLiteHealth(node, 3*time.Minute); err != nil {
return nil, fmt.Errorf("genesis health: %w", err)
}
fmt.Println(" OK")
// Generate invite tokens (one per remaining node)
fmt.Print(" Generating invite tokens...")
remaining := len(state.Servers) - 1
tokens := make([]string, remaining)
for i := 0; i < remaining; i++ {
token, err := generateInviteToken(node)
if err != nil {
return nil, fmt.Errorf("generate invite token %d: %w", i+1, err)
}
tokens[i] = token
fmt.Print(".")
}
fmt.Println(" OK")
return tokens, nil
}
// phase5JoinNodes joins the remaining 4 nodes to the cluster (serial).
func phase5JoinNodes(cfg *Config, state *SandboxState, tokens []string, sshKeyPath string) error {
genesisIP := state.GenesisServer().IP
for i := 1; i < len(state.Servers); i++ {
srv := state.Servers[i]
node := inspector.Node{User: "root", Host: srv.IP, SSHKey: sshKeyPath}
token := tokens[i-1]
var installCmd string
if srv.Role == "nameserver" {
installCmd = fmt.Sprintf("/opt/orama/bin/orama node install --join http://%s --token %s --vps-ip %s --domain %s --base-domain %s --nameserver --skip-checks",
genesisIP, token, srv.IP, cfg.Domain, cfg.Domain)
} else {
installCmd = fmt.Sprintf("/opt/orama/bin/orama node install --join http://%s --token %s --vps-ip %s --base-domain %s --skip-checks",
genesisIP, token, srv.IP, cfg.Domain)
}
fmt.Printf(" [%d/%d] Joining %s (%s, %s)...\n", i, len(state.Servers)-1, srv.Name, srv.IP, srv.Role)
if err := remotessh.RunSSHStreaming(node, installCmd, remotessh.WithNoHostKeyCheck()); err != nil {
return fmt.Errorf("join %s: %w", srv.Name, err)
}
// Wait for node health before proceeding
fmt.Printf(" Waiting for %s health...", srv.Name)
if err := waitForRQLiteHealth(node, 3*time.Minute); err != nil {
fmt.Printf(" WARN: %v\n", err)
} else {
fmt.Println(" OK")
}
}
return nil
}
// phase6Verify runs a basic cluster health check.
func phase6Verify(cfg *Config, state *SandboxState, sshKeyPath string) {
genesis := state.GenesisServer()
node := inspector.Node{User: "root", Host: genesis.IP, SSHKey: sshKeyPath}
// Check RQLite cluster
out, err := runSSHOutput(node, "curl -s http://localhost:5001/status | grep -o '\"state\":\"[^\"]*\"' | head -1")
if err == nil {
fmt.Printf(" RQLite: %s\n", strings.TrimSpace(out))
}
// Check DNS (if floating IPs configured, only with safe domain names)
if len(cfg.FloatingIPs) > 0 && isSafeDNSName(cfg.Domain) {
out, err = runSSHOutput(node, fmt.Sprintf("dig +short @%s test.%s 2>/dev/null || echo 'DNS not responding'",
cfg.FloatingIPs[0].IP, cfg.Domain))
if err == nil {
fmt.Printf(" DNS: %s\n", strings.TrimSpace(out))
}
}
}
// waitForRQLiteHealth polls RQLite until it reports Leader or Follower state.
func waitForRQLiteHealth(node inspector.Node, timeout time.Duration) error {
deadline := time.Now().Add(timeout)
for time.Now().Before(deadline) {
out, err := runSSHOutput(node, "curl -sf http://localhost:5001/status 2>/dev/null | grep -o '\"state\":\"[^\"]*\"'")
if err == nil {
result := strings.TrimSpace(out)
if strings.Contains(result, "Leader") || strings.Contains(result, "Follower") {
return nil
}
}
time.Sleep(5 * time.Second)
}
return fmt.Errorf("timeout waiting for RQLite health after %s", timeout)
}
// generateInviteToken runs `orama node invite` on the node and parses the token.
func generateInviteToken(node inspector.Node) (string, error) {
out, err := runSSHOutput(node, "/opt/orama/bin/orama node invite --expiry 1h 2>&1")
if err != nil {
return "", fmt.Errorf("invite command failed: %w", err)
}
// Parse token from output — the invite command outputs:
// "sudo orama install --join https://... --token <64-char-hex> --vps-ip ..."
// Look for the --token flag value first
fields := strings.Fields(out)
for i, field := range fields {
if field == "--token" && i+1 < len(fields) {
candidate := fields[i+1]
if len(candidate) == 64 && isHex(candidate) {
return candidate, nil
}
}
}
// Fallback: look for any standalone 64-char hex string
for _, word := range fields {
if len(word) == 64 && isHex(word) {
return word, nil
}
}
return "", fmt.Errorf("could not parse token from invite output:\n%s", out)
}
// isSafeDNSName returns true if the string is safe to use in shell commands.
func isSafeDNSName(s string) bool {
for _, c := range s {
if !((c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || (c >= '0' && c <= '9') || c == '.' || c == '-') {
return false
}
}
return len(s) > 0
}
// isHex returns true if s contains only hex characters.
func isHex(s string) bool {
for _, c := range s {
if !((c >= '0' && c <= '9') || (c >= 'a' && c <= 'f') || (c >= 'A' && c <= 'F')) {
return false
}
}
return true
}
// runSSHOutput runs a command via SSH and returns stdout as a string.
// Uses StrictHostKeyChecking=no because sandbox IPs are frequently recycled.
func runSSHOutput(node inspector.Node, command string) (string, error) {
args := []string{
"ssh", "-n",
"-o", "StrictHostKeyChecking=no",
"-o", "UserKnownHostsFile=/dev/null",
"-o", "ConnectTimeout=10",
"-o", "BatchMode=yes",
"-i", node.SSHKey,
fmt.Sprintf("%s@%s", node.User, node.Host),
command,
}
out, err := execCommand(args[0], args[1:]...)
return string(out), err
}
// execCommand runs a command and returns its output.
func execCommand(name string, args ...string) ([]byte, error) {
return exec.Command(name, args...).Output()
}
// findNewestArchive finds the newest binary archive in /tmp/.
func findNewestArchive() string {
entries, err := os.ReadDir("/tmp")
if err != nil {
return ""
}
var best string
var bestMod int64
for _, entry := range entries {
name := entry.Name()
if strings.HasPrefix(name, "orama-") && strings.Contains(name, "-linux-") && strings.HasSuffix(name, ".tar.gz") {
info, err := entry.Info()
if err != nil {
continue
}
if info.ModTime().Unix() > bestMod {
best = filepath.Join("/tmp", name)
bestMod = info.ModTime().Unix()
}
}
}
return best
}
// formatBytes formats a byte count as human-readable.
func formatBytes(b int64) string {
const unit = 1024
if b < unit {
return fmt.Sprintf("%d B", b)
}
div, exp := int64(unit), 0
for n := b / unit; n >= unit; n /= unit {
div *= unit
exp++
}
return fmt.Sprintf("%.1f %cB", float64(b)/float64(div), "KMGTPE"[exp])
}
// printCreateSummary prints the cluster summary after creation.
func printCreateSummary(cfg *Config, state *SandboxState) {
fmt.Printf("\nSandbox %q ready (%d nodes)\n", state.Name, len(state.Servers))
fmt.Println()
fmt.Println("Nameservers:")
for _, srv := range state.NameserverNodes() {
floating := ""
if srv.FloatingIP != "" {
floating = fmt.Sprintf(" (floating: %s)", srv.FloatingIP)
}
fmt.Printf(" %s: %s%s\n", srv.Name, srv.IP, floating)
}
fmt.Println("Nodes:")
for _, srv := range state.RegularNodes() {
fmt.Printf(" %s: %s\n", srv.Name, srv.IP)
}
fmt.Println()
fmt.Printf("Domain: %s\n", cfg.Domain)
fmt.Printf("Gateway: https://%s\n", cfg.Domain)
fmt.Println()
fmt.Println("SSH: orama sandbox ssh 1")
fmt.Println("Destroy: orama sandbox destroy")
}
// cleanupFailedCreate deletes any servers that were created during a failed provision.
func cleanupFailedCreate(client *HetznerClient, state *SandboxState) {
if len(state.Servers) == 0 {
return
}
fmt.Println("\nCleaning up failed creation...")
for _, srv := range state.Servers {
if srv.ID > 0 {
client.DeleteServer(srv.ID)
fmt.Printf(" Deleted %s\n", srv.Name)
}
}
DeleteState(state.Name)
}

122
pkg/cli/sandbox/destroy.go Normal file
View File

@ -0,0 +1,122 @@
package sandbox
import (
"bufio"
"fmt"
"os"
"strings"
"sync"
)
// Destroy tears down a sandbox cluster.
func Destroy(name string, force bool) error {
cfg, err := LoadConfig()
if err != nil {
return err
}
// Resolve sandbox name
state, err := resolveSandbox(name)
if err != nil {
return err
}
// Confirm destruction
if !force {
reader := bufio.NewReader(os.Stdin)
fmt.Printf("Destroy sandbox %q? This deletes %d servers. [y/N]: ", state.Name, len(state.Servers))
choice, _ := reader.ReadString('\n')
choice = strings.TrimSpace(strings.ToLower(choice))
if choice != "y" && choice != "yes" {
fmt.Println("Aborted.")
return nil
}
}
state.Status = StatusDestroying
SaveState(state) // best-effort status update
client := NewHetznerClient(cfg.HetznerAPIToken)
// Step 1: Unassign floating IPs from nameserver nodes
fmt.Println("Unassigning floating IPs...")
for _, srv := range state.NameserverNodes() {
if srv.FloatingIP == "" {
continue
}
// Find the floating IP ID from config
for _, fip := range cfg.FloatingIPs {
if fip.IP == srv.FloatingIP {
if err := client.UnassignFloatingIP(fip.ID); err != nil {
fmt.Fprintf(os.Stderr, " Warning: could not unassign floating IP %s: %v\n", fip.IP, err)
} else {
fmt.Printf(" Unassigned %s from %s\n", fip.IP, srv.Name)
}
break
}
}
}
// Step 2: Delete all servers in parallel
fmt.Printf("Deleting %d servers...\n", len(state.Servers))
var wg sync.WaitGroup
var mu sync.Mutex
var failed []string
for _, srv := range state.Servers {
wg.Add(1)
go func(srv ServerState) {
defer wg.Done()
if err := client.DeleteServer(srv.ID); err != nil {
// Treat 404 as already deleted (idempotent)
if strings.Contains(err.Error(), "404") || strings.Contains(err.Error(), "not found") {
fmt.Printf(" %s (ID %d): already deleted\n", srv.Name, srv.ID)
} else {
mu.Lock()
failed = append(failed, fmt.Sprintf("%s (ID %d): %v", srv.Name, srv.ID, err))
mu.Unlock()
fmt.Fprintf(os.Stderr, " Warning: failed to delete %s: %v\n", srv.Name, err)
}
} else {
fmt.Printf(" Deleted %s (ID %d)\n", srv.Name, srv.ID)
}
}(srv)
}
wg.Wait()
if len(failed) > 0 {
fmt.Fprintf(os.Stderr, "\nFailed to delete %d server(s):\n", len(failed))
for _, f := range failed {
fmt.Fprintf(os.Stderr, " %s\n", f)
}
fmt.Fprintf(os.Stderr, "\nManual cleanup: delete servers at https://console.hetzner.cloud\n")
state.Status = StatusError
SaveState(state)
return fmt.Errorf("failed to delete %d server(s)", len(failed))
}
// Step 3: Remove state file
if err := DeleteState(state.Name); err != nil {
return fmt.Errorf("delete state: %w", err)
}
fmt.Printf("\nSandbox %q destroyed (%d servers deleted)\n", state.Name, len(state.Servers))
return nil
}
// resolveSandbox finds a sandbox by name or returns the active one.
func resolveSandbox(name string) (*SandboxState, error) {
if name != "" {
return LoadState(name)
}
// Find the active sandbox
active, err := FindActiveSandbox()
if err != nil {
return nil, err
}
if active == nil {
return nil, fmt.Errorf("no active sandbox found, specify --name")
}
return active, nil
}

84
pkg/cli/sandbox/fanout.go Normal file
View File

@ -0,0 +1,84 @@
package sandbox
import (
"fmt"
"path/filepath"
"sync"
"github.com/DeBrosOfficial/network/pkg/cli/remotessh"
"github.com/DeBrosOfficial/network/pkg/inspector"
)
// fanoutArchive uploads a binary archive to the first server, then fans out
// server-to-server in parallel to all remaining servers. This is much faster
// than uploading from the local machine to each node individually.
// After distribution, the archive is extracted on all nodes.
func fanoutArchive(servers []ServerState, sshKeyPath, archivePath string) error {
remotePath := "/tmp/" + filepath.Base(archivePath)
extractCmd := fmt.Sprintf("mkdir -p /opt/orama && tar xzf %s -C /opt/orama && rm -f %s",
remotePath, remotePath)
// Step 1: Upload from local machine to first node
first := servers[0]
firstNode := inspector.Node{User: "root", Host: first.IP, SSHKey: sshKeyPath}
fmt.Printf(" Uploading to %s...\n", first.Name)
if err := remotessh.UploadFile(firstNode, archivePath, remotePath, remotessh.WithNoHostKeyCheck()); err != nil {
return fmt.Errorf("upload to %s: %w", first.Name, err)
}
// Step 2: Fan out from first node to remaining nodes in parallel (server-to-server)
if len(servers) > 1 {
fmt.Printf(" Fanning out from %s to %d nodes...\n", first.Name, len(servers)-1)
// Temporarily upload SSH key for server-to-server SCP
remoteKeyPath := "/tmp/.sandbox_key"
if err := remotessh.UploadFile(firstNode, sshKeyPath, remoteKeyPath, remotessh.WithNoHostKeyCheck()); err != nil {
return fmt.Errorf("upload SSH key to %s: %w", first.Name, err)
}
defer remotessh.RunSSHStreaming(firstNode, fmt.Sprintf("rm -f %s", remoteKeyPath), remotessh.WithNoHostKeyCheck())
if err := remotessh.RunSSHStreaming(firstNode, fmt.Sprintf("chmod 600 %s", remoteKeyPath), remotessh.WithNoHostKeyCheck()); err != nil {
return fmt.Errorf("chmod SSH key on %s: %w", first.Name, err)
}
var wg sync.WaitGroup
errs := make([]error, len(servers))
for i := 1; i < len(servers); i++ {
wg.Add(1)
go func(idx int, srv ServerState) {
defer wg.Done()
// SCP from first node to target
scpCmd := fmt.Sprintf("scp -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -i %s %s root@%s:%s",
remoteKeyPath, remotePath, srv.IP, remotePath)
if err := remotessh.RunSSHStreaming(firstNode, scpCmd, remotessh.WithNoHostKeyCheck()); err != nil {
errs[idx] = fmt.Errorf("fanout to %s: %w", srv.Name, err)
return
}
// Extract on target
targetNode := inspector.Node{User: "root", Host: srv.IP, SSHKey: sshKeyPath}
if err := remotessh.RunSSHStreaming(targetNode, extractCmd, remotessh.WithNoHostKeyCheck()); err != nil {
errs[idx] = fmt.Errorf("extract on %s: %w", srv.Name, err)
return
}
fmt.Printf(" Distributed to %s\n", srv.Name)
}(i, servers[i])
}
wg.Wait()
for _, err := range errs {
if err != nil {
return err
}
}
}
// Step 3: Extract on first node
fmt.Printf(" Extracting on %s...\n", first.Name)
if err := remotessh.RunSSHStreaming(firstNode, extractCmd, remotessh.WithNoHostKeyCheck()); err != nil {
return fmt.Errorf("extract on %s: %w", first.Name, err)
}
return nil
}

534
pkg/cli/sandbox/hetzner.go Normal file
View File

@ -0,0 +1,534 @@
package sandbox
import (
"bytes"
"encoding/json"
"fmt"
"io"
"net/http"
"strconv"
"time"
)
const hetznerBaseURL = "https://api.hetzner.cloud/v1"
// HetznerClient is a minimal Hetzner Cloud API client.
type HetznerClient struct {
token string
httpClient *http.Client
}
// NewHetznerClient creates a new Hetzner API client.
func NewHetznerClient(token string) *HetznerClient {
return &HetznerClient{
token: token,
httpClient: &http.Client{
Timeout: 30 * time.Second,
},
}
}
// --- Request helpers ---
func (c *HetznerClient) doRequest(method, path string, body interface{}) ([]byte, int, error) {
var bodyReader io.Reader
if body != nil {
data, err := json.Marshal(body)
if err != nil {
return nil, 0, fmt.Errorf("marshal request body: %w", err)
}
bodyReader = bytes.NewReader(data)
}
req, err := http.NewRequest(method, hetznerBaseURL+path, bodyReader)
if err != nil {
return nil, 0, fmt.Errorf("create request: %w", err)
}
req.Header.Set("Authorization", "Bearer "+c.token)
if body != nil {
req.Header.Set("Content-Type", "application/json")
}
resp, err := c.httpClient.Do(req)
if err != nil {
return nil, 0, fmt.Errorf("request %s %s: %w", method, path, err)
}
defer resp.Body.Close()
respBody, err := io.ReadAll(resp.Body)
if err != nil {
return nil, resp.StatusCode, fmt.Errorf("read response: %w", err)
}
return respBody, resp.StatusCode, nil
}
func (c *HetznerClient) get(path string) ([]byte, error) {
body, status, err := c.doRequest("GET", path, nil)
if err != nil {
return nil, err
}
if status < 200 || status >= 300 {
return nil, parseHetznerError(body, status)
}
return body, nil
}
func (c *HetznerClient) post(path string, payload interface{}) ([]byte, error) {
body, status, err := c.doRequest("POST", path, payload)
if err != nil {
return nil, err
}
if status < 200 || status >= 300 {
return nil, parseHetznerError(body, status)
}
return body, nil
}
func (c *HetznerClient) delete(path string) error {
_, status, err := c.doRequest("DELETE", path, nil)
if err != nil {
return err
}
if status < 200 || status >= 300 {
return fmt.Errorf("delete %s: HTTP %d", path, status)
}
return nil
}
// --- API types ---
// HetznerServer represents a Hetzner Cloud server.
type HetznerServer struct {
ID int64 `json:"id"`
Name string `json:"name"`
Status string `json:"status"` // initializing, running, off, ...
PublicNet HetznerPublicNet `json:"public_net"`
Labels map[string]string `json:"labels"`
ServerType struct {
Name string `json:"name"`
} `json:"server_type"`
}
// HetznerPublicNet holds public networking info for a server.
type HetznerPublicNet struct {
IPv4 struct {
IP string `json:"ip"`
} `json:"ipv4"`
}
// HetznerFloatingIP represents a Hetzner floating IP.
type HetznerFloatingIP struct {
ID int64 `json:"id"`
IP string `json:"ip"`
Server *int64 `json:"server"` // nil if unassigned
Labels map[string]string `json:"labels"`
Description string `json:"description"`
HomeLocation struct {
Name string `json:"name"`
} `json:"home_location"`
}
// HetznerSSHKey represents a Hetzner SSH key.
type HetznerSSHKey struct {
ID int64 `json:"id"`
Name string `json:"name"`
Fingerprint string `json:"fingerprint"`
PublicKey string `json:"public_key"`
}
// HetznerFirewall represents a Hetzner firewall.
type HetznerFirewall struct {
ID int64 `json:"id"`
Name string `json:"name"`
Rules []HetznerFWRule `json:"rules"`
Labels map[string]string `json:"labels"`
}
// HetznerFWRule represents a firewall rule.
type HetznerFWRule struct {
Direction string `json:"direction"`
Protocol string `json:"protocol"`
Port string `json:"port"`
SourceIPs []string `json:"source_ips"`
Description string `json:"description,omitempty"`
}
// HetznerError represents an API error response.
type HetznerError struct {
Error struct {
Code string `json:"code"`
Message string `json:"message"`
} `json:"error"`
}
func parseHetznerError(body []byte, status int) error {
var he HetznerError
if err := json.Unmarshal(body, &he); err == nil && he.Error.Message != "" {
return fmt.Errorf("hetzner API error (HTTP %d): %s — %s", status, he.Error.Code, he.Error.Message)
}
return fmt.Errorf("hetzner API error: HTTP %d", status)
}
// --- Server operations ---
// CreateServerRequest holds parameters for server creation.
type CreateServerRequest struct {
Name string `json:"name"`
ServerType string `json:"server_type"`
Image string `json:"image"`
Location string `json:"location"`
SSHKeys []int64 `json:"ssh_keys"`
Labels map[string]string `json:"labels"`
Firewalls []struct {
Firewall int64 `json:"firewall"`
} `json:"firewalls,omitempty"`
}
// CreateServer creates a new server and returns it.
func (c *HetznerClient) CreateServer(req CreateServerRequest) (*HetznerServer, error) {
body, err := c.post("/servers", req)
if err != nil {
return nil, fmt.Errorf("create server %q: %w", req.Name, err)
}
var resp struct {
Server HetznerServer `json:"server"`
}
if err := json.Unmarshal(body, &resp); err != nil {
return nil, fmt.Errorf("parse create server response: %w", err)
}
return &resp.Server, nil
}
// GetServer retrieves a server by ID.
func (c *HetznerClient) GetServer(id int64) (*HetznerServer, error) {
body, err := c.get("/servers/" + strconv.FormatInt(id, 10))
if err != nil {
return nil, fmt.Errorf("get server %d: %w", id, err)
}
var resp struct {
Server HetznerServer `json:"server"`
}
if err := json.Unmarshal(body, &resp); err != nil {
return nil, fmt.Errorf("parse server response: %w", err)
}
return &resp.Server, nil
}
// DeleteServer deletes a server by ID.
func (c *HetznerClient) DeleteServer(id int64) error {
return c.delete("/servers/" + strconv.FormatInt(id, 10))
}
// ListServersByLabel lists servers filtered by a label selector.
func (c *HetznerClient) ListServersByLabel(selector string) ([]HetznerServer, error) {
body, err := c.get("/servers?label_selector=" + selector)
if err != nil {
return nil, fmt.Errorf("list servers: %w", err)
}
var resp struct {
Servers []HetznerServer `json:"servers"`
}
if err := json.Unmarshal(body, &resp); err != nil {
return nil, fmt.Errorf("parse servers response: %w", err)
}
return resp.Servers, nil
}
// WaitForServer polls until the server reaches "running" status.
func (c *HetznerClient) WaitForServer(id int64, timeout time.Duration) (*HetznerServer, error) {
deadline := time.Now().Add(timeout)
for time.Now().Before(deadline) {
srv, err := c.GetServer(id)
if err != nil {
return nil, err
}
if srv.Status == "running" {
return srv, nil
}
time.Sleep(3 * time.Second)
}
return nil, fmt.Errorf("server %d did not reach running state within %s", id, timeout)
}
// --- Floating IP operations ---
// CreateFloatingIP creates a new floating IP.
func (c *HetznerClient) CreateFloatingIP(location, description string, labels map[string]string) (*HetznerFloatingIP, error) {
payload := map[string]interface{}{
"type": "ipv4",
"home_location": location,
"description": description,
"labels": labels,
}
body, err := c.post("/floating_ips", payload)
if err != nil {
return nil, fmt.Errorf("create floating IP: %w", err)
}
var resp struct {
FloatingIP HetznerFloatingIP `json:"floating_ip"`
}
if err := json.Unmarshal(body, &resp); err != nil {
return nil, fmt.Errorf("parse floating IP response: %w", err)
}
return &resp.FloatingIP, nil
}
// ListFloatingIPsByLabel lists floating IPs filtered by label.
func (c *HetznerClient) ListFloatingIPsByLabel(selector string) ([]HetznerFloatingIP, error) {
body, err := c.get("/floating_ips?label_selector=" + selector)
if err != nil {
return nil, fmt.Errorf("list floating IPs: %w", err)
}
var resp struct {
FloatingIPs []HetznerFloatingIP `json:"floating_ips"`
}
if err := json.Unmarshal(body, &resp); err != nil {
return nil, fmt.Errorf("parse floating IPs response: %w", err)
}
return resp.FloatingIPs, nil
}
// AssignFloatingIP assigns a floating IP to a server.
func (c *HetznerClient) AssignFloatingIP(floatingIPID, serverID int64) error {
payload := map[string]int64{"server": serverID}
_, err := c.post("/floating_ips/"+strconv.FormatInt(floatingIPID, 10)+"/actions/assign", payload)
if err != nil {
return fmt.Errorf("assign floating IP %d to server %d: %w", floatingIPID, serverID, err)
}
return nil
}
// UnassignFloatingIP removes a floating IP assignment.
func (c *HetznerClient) UnassignFloatingIP(floatingIPID int64) error {
_, err := c.post("/floating_ips/"+strconv.FormatInt(floatingIPID, 10)+"/actions/unassign", struct{}{})
if err != nil {
return fmt.Errorf("unassign floating IP %d: %w", floatingIPID, err)
}
return nil
}
// --- SSH Key operations ---
// UploadSSHKey uploads a public key to Hetzner.
func (c *HetznerClient) UploadSSHKey(name, publicKey string) (*HetznerSSHKey, error) {
payload := map[string]string{
"name": name,
"public_key": publicKey,
}
body, err := c.post("/ssh_keys", payload)
if err != nil {
return nil, fmt.Errorf("upload SSH key: %w", err)
}
var resp struct {
SSHKey HetznerSSHKey `json:"ssh_key"`
}
if err := json.Unmarshal(body, &resp); err != nil {
return nil, fmt.Errorf("parse SSH key response: %w", err)
}
return &resp.SSHKey, nil
}
// ListSSHKeysByFingerprint finds SSH keys matching a fingerprint.
func (c *HetznerClient) ListSSHKeysByFingerprint(fingerprint string) ([]HetznerSSHKey, error) {
body, err := c.get("/ssh_keys?fingerprint=" + fingerprint)
if err != nil {
return nil, fmt.Errorf("list SSH keys: %w", err)
}
var resp struct {
SSHKeys []HetznerSSHKey `json:"ssh_keys"`
}
if err := json.Unmarshal(body, &resp); err != nil {
return nil, fmt.Errorf("parse SSH keys response: %w", err)
}
return resp.SSHKeys, nil
}
// GetSSHKey retrieves an SSH key by ID.
func (c *HetznerClient) GetSSHKey(id int64) (*HetznerSSHKey, error) {
body, err := c.get("/ssh_keys/" + strconv.FormatInt(id, 10))
if err != nil {
return nil, fmt.Errorf("get SSH key %d: %w", id, err)
}
var resp struct {
SSHKey HetznerSSHKey `json:"ssh_key"`
}
if err := json.Unmarshal(body, &resp); err != nil {
return nil, fmt.Errorf("parse SSH key response: %w", err)
}
return &resp.SSHKey, nil
}
// --- Firewall operations ---
// CreateFirewall creates a firewall with the given rules.
func (c *HetznerClient) CreateFirewall(name string, rules []HetznerFWRule, labels map[string]string) (*HetznerFirewall, error) {
payload := map[string]interface{}{
"name": name,
"rules": rules,
"labels": labels,
}
body, err := c.post("/firewalls", payload)
if err != nil {
return nil, fmt.Errorf("create firewall: %w", err)
}
var resp struct {
Firewall HetznerFirewall `json:"firewall"`
}
if err := json.Unmarshal(body, &resp); err != nil {
return nil, fmt.Errorf("parse firewall response: %w", err)
}
return &resp.Firewall, nil
}
// ListFirewallsByLabel lists firewalls filtered by label.
func (c *HetznerClient) ListFirewallsByLabel(selector string) ([]HetznerFirewall, error) {
body, err := c.get("/firewalls?label_selector=" + selector)
if err != nil {
return nil, fmt.Errorf("list firewalls: %w", err)
}
var resp struct {
Firewalls []HetznerFirewall `json:"firewalls"`
}
if err := json.Unmarshal(body, &resp); err != nil {
return nil, fmt.Errorf("parse firewalls response: %w", err)
}
return resp.Firewalls, nil
}
// DeleteFirewall deletes a firewall by ID.
func (c *HetznerClient) DeleteFirewall(id int64) error {
return c.delete("/firewalls/" + strconv.FormatInt(id, 10))
}
// DeleteFloatingIP deletes a floating IP by ID.
func (c *HetznerClient) DeleteFloatingIP(id int64) error {
return c.delete("/floating_ips/" + strconv.FormatInt(id, 10))
}
// DeleteSSHKey deletes an SSH key by ID.
func (c *HetznerClient) DeleteSSHKey(id int64) error {
return c.delete("/ssh_keys/" + strconv.FormatInt(id, 10))
}
// --- Location & Server Type operations ---
// HetznerLocation represents a Hetzner datacenter location.
type HetznerLocation struct {
ID int64 `json:"id"`
Name string `json:"name"` // e.g., "fsn1", "nbg1", "hel1"
Description string `json:"description"` // e.g., "Falkenstein DC Park 1"
City string `json:"city"`
Country string `json:"country"` // ISO 3166-1 alpha-2
}
// HetznerServerType represents a Hetzner server type with pricing.
type HetznerServerType struct {
ID int64 `json:"id"`
Name string `json:"name"` // e.g., "cx22", "cx23"
Description string `json:"description"` // e.g., "CX23"
Cores int `json:"cores"`
Memory float64 `json:"memory"` // GB
Disk int `json:"disk"` // GB
Architecture string `json:"architecture"`
Deprecation *struct {
Announced string `json:"announced"`
UnavailableAfter string `json:"unavailable_after"`
} `json:"deprecation"` // nil = not deprecated
Prices []struct {
Location string `json:"location"`
Hourly struct {
Gross string `json:"gross"`
} `json:"price_hourly"`
Monthly struct {
Gross string `json:"gross"`
} `json:"price_monthly"`
} `json:"prices"`
}
// ListLocations returns all available Hetzner datacenter locations.
func (c *HetznerClient) ListLocations() ([]HetznerLocation, error) {
body, err := c.get("/locations")
if err != nil {
return nil, fmt.Errorf("list locations: %w", err)
}
var resp struct {
Locations []HetznerLocation `json:"locations"`
}
if err := json.Unmarshal(body, &resp); err != nil {
return nil, fmt.Errorf("parse locations response: %w", err)
}
return resp.Locations, nil
}
// ListServerTypes returns all available server types.
func (c *HetznerClient) ListServerTypes() ([]HetznerServerType, error) {
body, err := c.get("/server_types?per_page=50")
if err != nil {
return nil, fmt.Errorf("list server types: %w", err)
}
var resp struct {
ServerTypes []HetznerServerType `json:"server_types"`
}
if err := json.Unmarshal(body, &resp); err != nil {
return nil, fmt.Errorf("parse server types response: %w", err)
}
return resp.ServerTypes, nil
}
// --- Validation ---
// ValidateToken checks if the API token is valid by making a simple request.
func (c *HetznerClient) ValidateToken() error {
_, err := c.get("/servers?per_page=1")
if err != nil {
return fmt.Errorf("invalid Hetzner API token: %w", err)
}
return nil
}
// --- Sandbox firewall rules ---
// SandboxFirewallRules returns the standard firewall rules for sandbox nodes.
func SandboxFirewallRules() []HetznerFWRule {
allIPv4 := []string{"0.0.0.0/0"}
allIPv6 := []string{"::/0"}
allIPs := append(allIPv4, allIPv6...)
return []HetznerFWRule{
{Direction: "in", Protocol: "tcp", Port: "22", SourceIPs: allIPs, Description: "SSH"},
{Direction: "in", Protocol: "tcp", Port: "53", SourceIPs: allIPs, Description: "DNS TCP"},
{Direction: "in", Protocol: "udp", Port: "53", SourceIPs: allIPs, Description: "DNS UDP"},
{Direction: "in", Protocol: "tcp", Port: "80", SourceIPs: allIPs, Description: "HTTP"},
{Direction: "in", Protocol: "tcp", Port: "443", SourceIPs: allIPs, Description: "HTTPS"},
{Direction: "in", Protocol: "udp", Port: "51820", SourceIPs: allIPs, Description: "WireGuard"},
}
}

303
pkg/cli/sandbox/hetzner_test.go Normal file
View File

@ -0,0 +1,303 @@
package sandbox
import (
"encoding/json"
"net/http"
"net/http/httptest"
"testing"
"time"
)
func TestValidateToken_Success(t *testing.T) {
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.Header.Get("Authorization") != "Bearer test-token" {
t.Errorf("unexpected auth header: %s", r.Header.Get("Authorization"))
}
w.WriteHeader(200)
json.NewEncoder(w).Encode(map[string]interface{}{"servers": []interface{}{}})
}))
defer srv.Close()
client := newTestClient(srv, "test-token")
if err := client.ValidateToken(); err != nil {
t.Errorf("ValidateToken() error = %v, want nil", err)
}
}
func TestValidateToken_InvalidToken(t *testing.T) {
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.WriteHeader(401)
json.NewEncoder(w).Encode(map[string]interface{}{
"error": map[string]string{
"code": "unauthorized",
"message": "unable to authenticate",
},
})
}))
defer srv.Close()
client := newTestClient(srv, "bad-token")
if err := client.ValidateToken(); err == nil {
t.Error("ValidateToken() expected error for invalid token")
}
}
func TestCreateServer(t *testing.T) {
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.Method != "POST" || r.URL.Path != "/v1/servers" {
t.Errorf("unexpected request: %s %s", r.Method, r.URL.Path)
}
var req CreateServerRequest
json.NewDecoder(r.Body).Decode(&req)
if req.Name != "sbx-test-1" {
t.Errorf("unexpected server name: %s", req.Name)
}
if req.ServerType != "cx22" {
t.Errorf("unexpected server type: %s", req.ServerType)
}
w.WriteHeader(201)
json.NewEncoder(w).Encode(map[string]interface{}{
"server": map[string]interface{}{
"id": 12345,
"name": req.Name,
"status": "initializing",
"public_net": map[string]interface{}{
"ipv4": map[string]string{"ip": "1.2.3.4"},
},
"labels": req.Labels,
"server_type": map[string]string{"name": "cx22"},
},
})
}))
defer srv.Close()
client := newTestClient(srv, "test-token")
server, err := client.CreateServer(CreateServerRequest{
Name: "sbx-test-1",
ServerType: "cx22",
Image: "ubuntu-24.04",
Location: "fsn1",
SSHKeys: []int64{1},
Labels: map[string]string{"orama-sandbox": "test"},
})
if err != nil {
t.Fatalf("CreateServer() error = %v", err)
}
if server.ID != 12345 {
t.Errorf("server ID = %d, want 12345", server.ID)
}
if server.Name != "sbx-test-1" {
t.Errorf("server name = %s, want sbx-test-1", server.Name)
}
if server.PublicNet.IPv4.IP != "1.2.3.4" {
t.Errorf("server IP = %s, want 1.2.3.4", server.PublicNet.IPv4.IP)
}
}
func TestDeleteServer(t *testing.T) {
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.Method != "DELETE" || r.URL.Path != "/v1/servers/12345" {
t.Errorf("unexpected request: %s %s", r.Method, r.URL.Path)
}
w.WriteHeader(200)
}))
defer srv.Close()
client := newTestClient(srv, "test-token")
if err := client.DeleteServer(12345); err != nil {
t.Errorf("DeleteServer() error = %v", err)
}
}
func TestListServersByLabel(t *testing.T) {
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.URL.Query().Get("label_selector") != "orama-sandbox=test" {
t.Errorf("unexpected label_selector: %s", r.URL.Query().Get("label_selector"))
}
w.WriteHeader(200)
json.NewEncoder(w).Encode(map[string]interface{}{
"servers": []map[string]interface{}{
{"id": 1, "name": "sbx-test-1", "status": "running", "public_net": map[string]interface{}{"ipv4": map[string]string{"ip": "1.1.1.1"}}, "server_type": map[string]string{"name": "cx22"}},
{"id": 2, "name": "sbx-test-2", "status": "running", "public_net": map[string]interface{}{"ipv4": map[string]string{"ip": "2.2.2.2"}}, "server_type": map[string]string{"name": "cx22"}},
},
})
}))
defer srv.Close()
client := newTestClient(srv, "test-token")
servers, err := client.ListServersByLabel("orama-sandbox=test")
if err != nil {
t.Fatalf("ListServersByLabel() error = %v", err)
}
if len(servers) != 2 {
t.Errorf("got %d servers, want 2", len(servers))
}
}
func TestWaitForServer_AlreadyRunning(t *testing.T) {
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.WriteHeader(200)
json.NewEncoder(w).Encode(map[string]interface{}{
"server": map[string]interface{}{
"id": 1,
"name": "test",
"status": "running",
"public_net": map[string]interface{}{
"ipv4": map[string]string{"ip": "1.1.1.1"},
},
"server_type": map[string]string{"name": "cx22"},
},
})
}))
defer srv.Close()
client := newTestClient(srv, "test-token")
server, err := client.WaitForServer(1, 5*time.Second)
if err != nil {
t.Fatalf("WaitForServer() error = %v", err)
}
if server.Status != "running" {
t.Errorf("server status = %s, want running", server.Status)
}
}
func TestAssignFloatingIP(t *testing.T) {
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.Method != "POST" || r.URL.Path != "/v1/floating_ips/100/actions/assign" {
t.Errorf("unexpected request: %s %s", r.Method, r.URL.Path)
}
var body map[string]int64
json.NewDecoder(r.Body).Decode(&body)
if body["server"] != 200 {
t.Errorf("unexpected server ID: %d", body["server"])
}
w.WriteHeader(200)
json.NewEncoder(w).Encode(map[string]interface{}{"action": map[string]interface{}{"id": 1, "status": "running"}})
}))
defer srv.Close()
client := newTestClient(srv, "test-token")
if err := client.AssignFloatingIP(100, 200); err != nil {
t.Errorf("AssignFloatingIP() error = %v", err)
}
}
func TestUploadSSHKey(t *testing.T) {
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.Method != "POST" || r.URL.Path != "/v1/ssh_keys" {
t.Errorf("unexpected request: %s %s", r.Method, r.URL.Path)
}
w.WriteHeader(201)
json.NewEncoder(w).Encode(map[string]interface{}{
"ssh_key": map[string]interface{}{
"id": 42,
"name": "orama-sandbox",
"fingerprint": "aa:bb:cc:dd",
"public_key": "ssh-ed25519 AAAA...",
},
})
}))
defer srv.Close()
client := newTestClient(srv, "test-token")
key, err := client.UploadSSHKey("orama-sandbox", "ssh-ed25519 AAAA...")
if err != nil {
t.Fatalf("UploadSSHKey() error = %v", err)
}
if key.ID != 42 {
t.Errorf("key ID = %d, want 42", key.ID)
}
}
func TestCreateFirewall(t *testing.T) {
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.Method != "POST" || r.URL.Path != "/v1/firewalls" {
t.Errorf("unexpected request: %s %s", r.Method, r.URL.Path)
}
w.WriteHeader(201)
json.NewEncoder(w).Encode(map[string]interface{}{
"firewall": map[string]interface{}{
"id": 99,
"name": "orama-sandbox",
},
})
}))
defer srv.Close()
client := newTestClient(srv, "test-token")
fw, err := client.CreateFirewall("orama-sandbox", SandboxFirewallRules(), map[string]string{"orama-sandbox": "infra"})
if err != nil {
t.Fatalf("CreateFirewall() error = %v", err)
}
if fw.ID != 99 {
t.Errorf("firewall ID = %d, want 99", fw.ID)
}
}
func TestSandboxFirewallRules(t *testing.T) {
rules := SandboxFirewallRules()
if len(rules) != 6 {
t.Errorf("got %d rules, want 6", len(rules))
}
expectedPorts := map[string]bool{"22": false, "53": false, "80": false, "443": false, "51820": false}
for _, r := range rules {
expectedPorts[r.Port] = true
if r.Direction != "in" {
t.Errorf("rule %s direction = %s, want in", r.Port, r.Direction)
}
}
for port, seen := range expectedPorts {
if !seen {
t.Errorf("missing firewall rule for port %s", port)
}
}
}
func TestParseHetznerError(t *testing.T) {
body := `{"error":{"code":"uniqueness_error","message":"server name already used"}}`
err := parseHetznerError([]byte(body), 409)
if err == nil {
t.Fatal("expected error")
}
expected := "hetzner API error (HTTP 409): uniqueness_error — server name already used"
if err.Error() != expected {
t.Errorf("error = %q, want %q", err.Error(), expected)
}
}
// newTestClient creates a HetznerClient pointing at a test server.
func newTestClient(ts *httptest.Server, token string) *HetznerClient {
client := NewHetznerClient(token)
// Override the base URL by using a custom transport
client.httpClient = ts.Client()
// We need to override the base URL — wrap the transport
origTransport := client.httpClient.Transport
client.httpClient.Transport = &testTransport{
base: origTransport,
testURL: ts.URL,
}
return client
}
// testTransport rewrites requests to point at the test server.
type testTransport struct {
base http.RoundTripper
testURL string
}
func (t *testTransport) RoundTrip(req *http.Request) (*http.Response, error) {
// Rewrite the URL to point at the test server
req.URL.Scheme = "http"
req.URL.Host = t.testURL[len("http://"):]
if t.base != nil {
return t.base.RoundTrip(req)
}
return http.DefaultTransport.RoundTrip(req)
}

26
pkg/cli/sandbox/names.go Normal file
View File

@ -0,0 +1,26 @@
package sandbox
import (
"math/rand"
)
var adjectives = []string{
"swift", "bright", "calm", "dark", "eager",
"fair", "gold", "hazy", "iron", "jade",
"keen", "lush", "mild", "neat", "opal",
"pure", "raw", "sage", "teal", "warm",
}
var nouns = []string{
"falcon", "beacon", "cedar", "delta", "ember",
"frost", "grove", "haven", "ivory", "jewel",
"knot", "latch", "maple", "nexus", "orbit",
"prism", "reef", "spark", "tide", "vault",
}
// GenerateName produces a random adjective-noun name like "swift-falcon".
func GenerateName() string {
adj := adjectives[rand.Intn(len(adjectives))]
noun := nouns[rand.Intn(len(nouns))]
return adj + "-" + noun
}

119
pkg/cli/sandbox/reset.go Normal file
View File

@ -0,0 +1,119 @@
package sandbox
import (
"bufio"
"fmt"
"os"
"strings"
)
// Reset tears down all sandbox infrastructure (floating IPs, firewall, SSH key)
// and removes the config file so the user can rerun setup from scratch.
// This is useful when switching datacenter locations (floating IPs are location-bound).
func Reset() error {
fmt.Println("Sandbox Reset")
fmt.Println("=============")
fmt.Println()
cfg, err := LoadConfig()
if err != nil {
// Config doesn't exist — just clean up any local files
fmt.Println("No sandbox config found. Cleaning up local files...")
return resetLocalFiles()
}
// Check for active sandboxes — refuse to reset if clusters are still running
active, _ := FindActiveSandbox()
if active != nil {
return fmt.Errorf("active sandbox %q exists — run 'orama sandbox destroy' first", active.Name)
}
// Show what will be deleted
fmt.Println("This will delete the following Hetzner resources:")
for i, fip := range cfg.FloatingIPs {
fmt.Printf(" Floating IP %d: %s (ID: %d)\n", i+1, fip.IP, fip.ID)
}
if cfg.FirewallID != 0 {
fmt.Printf(" Firewall ID: %d\n", cfg.FirewallID)
}
if cfg.SSHKey.HetznerID != 0 {
fmt.Printf(" SSH Key ID: %d\n", cfg.SSHKey.HetznerID)
}
fmt.Println()
fmt.Println("Local files to remove:")
fmt.Println(" ~/.orama/sandbox.yaml")
fmt.Println()
reader := bufio.NewReader(os.Stdin)
fmt.Print("Delete all sandbox resources? [y/N]: ")
choice, _ := reader.ReadString('\n')
choice = strings.TrimSpace(strings.ToLower(choice))
if choice != "y" && choice != "yes" {
fmt.Println("Aborted.")
return nil
}
client := NewHetznerClient(cfg.HetznerAPIToken)
// Step 1: Delete floating IPs
fmt.Println()
fmt.Println("Deleting floating IPs...")
for _, fip := range cfg.FloatingIPs {
if err := client.DeleteFloatingIP(fip.ID); err != nil {
fmt.Fprintf(os.Stderr, " Warning: could not delete floating IP %s (ID %d): %v\n", fip.IP, fip.ID, err)
} else {
fmt.Printf(" Deleted %s (ID %d)\n", fip.IP, fip.ID)
}
}
// Step 2: Delete firewall
if cfg.FirewallID != 0 {
fmt.Println("Deleting firewall...")
if err := client.DeleteFirewall(cfg.FirewallID); err != nil {
fmt.Fprintf(os.Stderr, " Warning: could not delete firewall (ID %d): %v\n", cfg.FirewallID, err)
} else {
fmt.Printf(" Deleted firewall (ID %d)\n", cfg.FirewallID)
}
}
// Step 3: Delete SSH key from Hetzner
if cfg.SSHKey.HetznerID != 0 {
fmt.Println("Deleting SSH key from Hetzner...")
if err := client.DeleteSSHKey(cfg.SSHKey.HetznerID); err != nil {
fmt.Fprintf(os.Stderr, " Warning: could not delete SSH key (ID %d): %v\n", cfg.SSHKey.HetznerID, err)
} else {
fmt.Printf(" Deleted SSH key (ID %d)\n", cfg.SSHKey.HetznerID)
}
}
// Step 4: Remove local files
if err := resetLocalFiles(); err != nil {
return err
}
fmt.Println()
fmt.Println("Reset complete. All sandbox resources deleted.")
fmt.Println()
fmt.Println("Next: orama sandbox setup")
return nil
}
// resetLocalFiles removes the sandbox config file.
func resetLocalFiles() error {
dir, err := configDir()
if err != nil {
return err
}
configFile := dir + "/sandbox.yaml"
fmt.Println("Removing local files...")
if err := os.Remove(configFile); err != nil {
if !os.IsNotExist(err) {
fmt.Fprintf(os.Stderr, " Warning: could not remove %s: %v\n", configFile, err)
}
} else {
fmt.Printf(" Removed %s\n", configFile)
}
return nil
}

162
pkg/cli/sandbox/rollout.go Normal file
View File

@ -0,0 +1,162 @@
package sandbox
import (
"fmt"
"os"
"path/filepath"
"strings"
"time"
"github.com/DeBrosOfficial/network/pkg/cli/remotessh"
"github.com/DeBrosOfficial/network/pkg/inspector"
)
// RolloutFlags holds optional flags passed through to `orama node upgrade`.
type RolloutFlags struct {
AnyoneClient bool
}
// Rollout builds, pushes, and performs a rolling upgrade on a sandbox cluster.
func Rollout(name string, flags RolloutFlags) error {
cfg, err := LoadConfig()
if err != nil {
return err
}
state, err := resolveSandbox(name)
if err != nil {
return err
}
sshKeyPath, cleanup, err := resolveVaultKeyOnce(cfg.SSHKey.VaultTarget)
if err != nil {
return fmt.Errorf("prepare SSH key: %w", err)
}
defer cleanup()
fmt.Printf("Rolling out to sandbox %q (%d nodes)\n\n", state.Name, len(state.Servers))
// Step 1: Find or require binary archive
archivePath := findNewestArchive()
if archivePath == "" {
return fmt.Errorf("no binary archive found in /tmp/ (run `orama build` first)")
}
info, _ := os.Stat(archivePath)
fmt.Printf("Archive: %s (%s)\n\n", filepath.Base(archivePath), formatBytes(info.Size()))
// Build extra flags string for upgrade command
extraFlags := flags.upgradeFlags()
// Step 2: Push archive to all nodes (upload to first, fan out server-to-server)
fmt.Println("Pushing archive to all nodes...")
if err := fanoutArchive(state.Servers, sshKeyPath, archivePath); err != nil {
return err
}
// Step 3: Rolling upgrade — followers first, leader last
fmt.Println("\nRolling upgrade (followers first, leader last)...")
// Find the leader
leaderIdx := findLeaderIndex(state, sshKeyPath)
if leaderIdx < 0 {
fmt.Fprintf(os.Stderr, " Warning: could not detect RQLite leader, upgrading in order\n")
}
// Upgrade non-leaders first
for i, srv := range state.Servers {
if i == leaderIdx {
continue // skip leader, do it last
}
if err := upgradeNode(srv, sshKeyPath, i+1, len(state.Servers), extraFlags); err != nil {
return err
}
// Wait between nodes
if i < len(state.Servers)-1 {
fmt.Printf(" Waiting 15s before next node...\n")
time.Sleep(15 * time.Second)
}
}
// Upgrade leader last
if leaderIdx >= 0 {
srv := state.Servers[leaderIdx]
if err := upgradeNode(srv, sshKeyPath, len(state.Servers), len(state.Servers), extraFlags); err != nil {
return err
}
}
fmt.Printf("\nRollout complete for sandbox %q\n", state.Name)
return nil
}
// upgradeFlags builds the extra CLI flags string for `orama node upgrade`.
func (f RolloutFlags) upgradeFlags() string {
var parts []string
if f.AnyoneClient {
parts = append(parts, "--anyone-client")
}
return strings.Join(parts, " ")
}
// findLeaderIndex returns the index of the RQLite leader node, or -1 if unknown.
func findLeaderIndex(state *SandboxState, sshKeyPath string) int {
for i, srv := range state.Servers {
node := inspector.Node{User: "root", Host: srv.IP, SSHKey: sshKeyPath}
out, err := runSSHOutput(node, "curl -sf http://localhost:5001/status 2>/dev/null | grep -o '\"state\":\"[^\"]*\"'")
if err == nil && contains(out, "Leader") {
return i
}
}
return -1
}
// upgradeNode performs `orama node upgrade --restart` on a single node.
// It pre-replaces the orama CLI binary before running the upgrade command
// to avoid ETXTBSY ("text file busy") errors when the old binary doesn't
// have the os.Remove fix in copyBinary().
func upgradeNode(srv ServerState, sshKeyPath string, current, total int, extraFlags string) error {
node := inspector.Node{User: "root", Host: srv.IP, SSHKey: sshKeyPath}
fmt.Printf(" [%d/%d] Upgrading %s (%s)...\n", current, total, srv.Name, srv.IP)
// Pre-replace the orama CLI so the upgrade runs the NEW binary (with ETXTBSY fix).
// rm unlinks the old inode (kernel keeps it alive for the running process),
// cp creates a fresh inode at the same path.
preReplace := "rm -f /usr/local/bin/orama && cp /opt/orama/bin/orama /usr/local/bin/orama"
if err := remotessh.RunSSHStreaming(node, preReplace, remotessh.WithNoHostKeyCheck()); err != nil {
return fmt.Errorf("pre-replace orama binary on %s: %w", srv.Name, err)
}
upgradeCmd := "orama node upgrade --restart"
if extraFlags != "" {
upgradeCmd += " " + extraFlags
}
if err := remotessh.RunSSHStreaming(node, upgradeCmd, remotessh.WithNoHostKeyCheck()); err != nil {
return fmt.Errorf("upgrade %s: %w", srv.Name, err)
}
// Wait for health
fmt.Printf(" Checking health...")
if err := waitForRQLiteHealth(node, 2*time.Minute); err != nil {
fmt.Printf(" WARN: %v\n", err)
} else {
fmt.Println(" OK")
}
return nil
}
// contains checks if s contains substr.
func contains(s, substr string) bool {
return len(s) >= len(substr) && findSubstring(s, substr)
}
func findSubstring(s, substr string) bool {
for i := 0; i <= len(s)-len(substr); i++ {
if s[i:i+len(substr)] == substr {
return true
}
}
return false
}

550
pkg/cli/sandbox/setup.go Normal file
View File

@ -0,0 +1,550 @@
package sandbox
import (
"bufio"
"fmt"
"os"
"os/exec"
"sort"
"strconv"
"strings"
"time"
"github.com/DeBrosOfficial/network/pkg/cli/remotessh"
)
// Setup runs the interactive sandbox setup wizard.
func Setup() error {
fmt.Println("Orama Sandbox Setup")
fmt.Println("====================")
fmt.Println()
reader := bufio.NewReader(os.Stdin)
// Step 1: Hetzner API token
fmt.Print("Hetzner Cloud API token: ")
token, err := reader.ReadString('\n')
if err != nil {
return fmt.Errorf("read token: %w", err)
}
token = strings.TrimSpace(token)
if token == "" {
return fmt.Errorf("API token is required")
}
fmt.Print(" Validating token... ")
client := NewHetznerClient(token)
if err := client.ValidateToken(); err != nil {
fmt.Println("FAILED")
return fmt.Errorf("invalid token: %w", err)
}
fmt.Println("OK")
fmt.Println()
// Step 2: Domain
fmt.Print("Sandbox domain (e.g., sbx.dbrs.space): ")
domain, err := reader.ReadString('\n')
if err != nil {
return fmt.Errorf("read domain: %w", err)
}
domain = strings.TrimSpace(domain)
if domain == "" {
return fmt.Errorf("domain is required")
}
cfg := &Config{
HetznerAPIToken: token,
Domain: domain,
}
// Step 3: Location selection
fmt.Println()
location, err := selectLocation(client, reader)
if err != nil {
return err
}
cfg.Location = location
// Step 4: Server type selection
fmt.Println()
serverType, err := selectServerType(client, reader, location)
if err != nil {
return err
}
cfg.ServerType = serverType
// Step 5: Floating IPs
fmt.Println()
fmt.Println("Checking floating IPs...")
floatingIPs, err := setupFloatingIPs(client, cfg.Location)
if err != nil {
return err
}
cfg.FloatingIPs = floatingIPs
// Step 6: Firewall
fmt.Println()
fmt.Println("Checking firewall...")
fwID, err := setupFirewall(client)
if err != nil {
return err
}
cfg.FirewallID = fwID
// Step 7: SSH key
fmt.Println()
fmt.Println("Setting up SSH key...")
sshKeyConfig, err := setupSSHKey(client)
if err != nil {
return err
}
cfg.SSHKey = sshKeyConfig
// Step 8: Display DNS instructions
fmt.Println()
fmt.Println("DNS Configuration")
fmt.Println("-----------------")
fmt.Println("Configure the following at your domain registrar:")
fmt.Println()
fmt.Printf(" 1. Add glue records (Personal DNS Servers):\n")
fmt.Printf(" ns1.%s -> %s\n", domain, cfg.FloatingIPs[0].IP)
fmt.Printf(" ns2.%s -> %s\n", domain, cfg.FloatingIPs[1].IP)
fmt.Println()
fmt.Printf(" 2. Set custom nameservers for %s:\n", domain)
fmt.Printf(" ns1.%s\n", domain)
fmt.Printf(" ns2.%s\n", domain)
fmt.Println()
// Step 9: Verify DNS (optional)
fmt.Print("Verify DNS now? [y/N]: ")
verifyChoice, _ := reader.ReadString('\n')
verifyChoice = strings.TrimSpace(strings.ToLower(verifyChoice))
if verifyChoice == "y" || verifyChoice == "yes" {
verifyDNS(domain, cfg.FloatingIPs, reader)
}
// Save config
if err := SaveConfig(cfg); err != nil {
return fmt.Errorf("save config: %w", err)
}
fmt.Println()
fmt.Println("Setup complete! Config saved to ~/.orama/sandbox.yaml")
fmt.Println()
fmt.Println("Next: orama sandbox create")
return nil
}
// selectLocation fetches available Hetzner locations and lets the user pick one.
func selectLocation(client *HetznerClient, reader *bufio.Reader) (string, error) {
fmt.Println("Fetching available locations...")
locations, err := client.ListLocations()
if err != nil {
return "", fmt.Errorf("list locations: %w", err)
}
sort.Slice(locations, func(i, j int) bool {
return locations[i].Name < locations[j].Name
})
defaultLoc := "nbg1"
fmt.Println(" Available datacenter locations:")
for i, loc := range locations {
def := ""
if loc.Name == defaultLoc {
def = " (default)"
}
fmt.Printf(" %d) %s — %s, %s%s\n", i+1, loc.Name, loc.City, loc.Country, def)
}
fmt.Printf("\n Select location [%s]: ", defaultLoc)
choice, _ := reader.ReadString('\n')
choice = strings.TrimSpace(choice)
if choice == "" {
fmt.Printf(" Using %s\n", defaultLoc)
return defaultLoc, nil
}
// Try as number first
if num, err := strconv.Atoi(choice); err == nil && num >= 1 && num <= len(locations) {
loc := locations[num-1].Name
fmt.Printf(" Using %s\n", loc)
return loc, nil
}
// Try as location name
for _, loc := range locations {
if strings.EqualFold(loc.Name, choice) {
fmt.Printf(" Using %s\n", loc.Name)
return loc.Name, nil
}
}
return "", fmt.Errorf("unknown location %q", choice)
}
// selectServerType fetches available server types for a location and lets the user pick one.
func selectServerType(client *HetznerClient, reader *bufio.Reader, location string) (string, error) {
fmt.Println("Fetching available server types...")
serverTypes, err := client.ListServerTypes()
if err != nil {
return "", fmt.Errorf("list server types: %w", err)
}
// Filter to x86 shared-vCPU types available at the selected location, skip deprecated
type option struct {
name string
cores int
memory float64
disk int
hourly string
monthly string
}
var options []option
for _, st := range serverTypes {
if st.Architecture != "x86" {
continue
}
if st.Deprecation != nil {
continue
}
// Only show shared-vCPU types (cx/cpx prefixes) — skip dedicated (ccx/cx5x)
if !strings.HasPrefix(st.Name, "cx") && !strings.HasPrefix(st.Name, "cpx") {
continue
}
// Find pricing for the selected location
hourly, monthly := "", ""
for _, p := range st.Prices {
if p.Location == location {
hourly = p.Hourly.Gross
monthly = p.Monthly.Gross
break
}
}
if hourly == "" {
continue // Not available in this location
}
options = append(options, option{
name: st.Name,
cores: st.Cores,
memory: st.Memory,
disk: st.Disk,
hourly: hourly,
monthly: monthly,
})
}
if len(options) == 0 {
return "", fmt.Errorf("no server types available in %s", location)
}
// Sort by hourly price (cheapest first)
sort.Slice(options, func(i, j int) bool {
pi, _ := strconv.ParseFloat(options[i].hourly, 64)
pj, _ := strconv.ParseFloat(options[j].hourly, 64)
return pi < pj
})
defaultType := options[0].name // cheapest
fmt.Printf(" Available server types in %s:\n", location)
for i, opt := range options {
def := ""
if opt.name == defaultType {
def = " (default)"
}
fmt.Printf(" %d) %-8s %d vCPU / %4.0f GB RAM / %3d GB disk — €%s/hr (€%s/mo)%s\n",
i+1, opt.name, opt.cores, opt.memory, opt.disk, formatPrice(opt.hourly), formatPrice(opt.monthly), def)
}
fmt.Printf("\n Select server type [%s]: ", defaultType)
choice, _ := reader.ReadString('\n')
choice = strings.TrimSpace(choice)
if choice == "" {
fmt.Printf(" Using %s (×5 nodes ≈ €%s/hr)\n", defaultType, multiplyPrice(options[0].hourly, 5))
return defaultType, nil
}
// Try as number
if num, err := strconv.Atoi(choice); err == nil && num >= 1 && num <= len(options) {
opt := options[num-1]
fmt.Printf(" Using %s (×5 nodes ≈ €%s/hr)\n", opt.name, multiplyPrice(opt.hourly, 5))
return opt.name, nil
}
// Try as name
for _, opt := range options {
if strings.EqualFold(opt.name, choice) {
fmt.Printf(" Using %s (×5 nodes ≈ €%s/hr)\n", opt.name, multiplyPrice(opt.hourly, 5))
return opt.name, nil
}
}
return "", fmt.Errorf("unknown server type %q", choice)
}
// formatPrice trims trailing zeros from a price string like "0.0063000000000000" → "0.0063".
func formatPrice(price string) string {
f, err := strconv.ParseFloat(price, 64)
if err != nil {
return price
}
// Use enough precision then trim trailing zeros
s := fmt.Sprintf("%.4f", f)
s = strings.TrimRight(s, "0")
s = strings.TrimRight(s, ".")
return s
}
// multiplyPrice multiplies a price string by n and returns formatted.
func multiplyPrice(price string, n int) string {
f, err := strconv.ParseFloat(price, 64)
if err != nil {
return "?"
}
return formatPrice(fmt.Sprintf("%.10f", f*float64(n)))
}
// setupFloatingIPs checks for existing floating IPs or creates new ones.
func setupFloatingIPs(client *HetznerClient, location string) ([]FloatIP, error) {
existing, err := client.ListFloatingIPsByLabel("orama-sandbox-dns=true")
if err != nil {
return nil, fmt.Errorf("list floating IPs: %w", err)
}
if len(existing) >= 2 {
fmt.Printf(" Found %d existing floating IPs:\n", len(existing))
result := make([]FloatIP, 2)
for i := 0; i < 2; i++ {
fmt.Printf(" ns%d: %s (ID: %d)\n", i+1, existing[i].IP, existing[i].ID)
result[i] = FloatIP{ID: existing[i].ID, IP: existing[i].IP}
}
return result, nil
}
// Need to create missing floating IPs
needed := 2 - len(existing)
fmt.Printf(" Need to create %d floating IP(s)...\n", needed)
reader := bufio.NewReader(os.Stdin)
fmt.Printf(" Create %d floating IP(s) in %s? (~$0.005/hr each) [Y/n]: ", needed, location)
choice, _ := reader.ReadString('\n')
choice = strings.TrimSpace(strings.ToLower(choice))
if choice == "n" || choice == "no" {
return nil, fmt.Errorf("floating IPs required, aborting setup")
}
result := make([]FloatIP, 0, 2)
for _, fip := range existing {
result = append(result, FloatIP{ID: fip.ID, IP: fip.IP})
}
for i := len(existing); i < 2; i++ {
desc := fmt.Sprintf("orama-sandbox-ns%d", i+1)
labels := map[string]string{"orama-sandbox-dns": "true"}
fip, err := client.CreateFloatingIP(location, desc, labels)
if err != nil {
return nil, fmt.Errorf("create floating IP %d: %w", i+1, err)
}
fmt.Printf(" Created ns%d: %s (ID: %d)\n", i+1, fip.IP, fip.ID)
result = append(result, FloatIP{ID: fip.ID, IP: fip.IP})
}
return result, nil
}
// setupFirewall ensures a sandbox firewall exists.
func setupFirewall(client *HetznerClient) (int64, error) {
existing, err := client.ListFirewallsByLabel("orama-sandbox=infra")
if err != nil {
return 0, fmt.Errorf("list firewalls: %w", err)
}
if len(existing) > 0 {
fmt.Printf(" Found existing firewall: %s (ID: %d)\n", existing[0].Name, existing[0].ID)
return existing[0].ID, nil
}
fmt.Print(" Creating sandbox firewall... ")
fw, err := client.CreateFirewall(
"orama-sandbox",
SandboxFirewallRules(),
map[string]string{"orama-sandbox": "infra"},
)
if err != nil {
fmt.Println("FAILED")
return 0, fmt.Errorf("create firewall: %w", err)
}
fmt.Printf("OK (ID: %d)\n", fw.ID)
return fw.ID, nil
}
// setupSSHKey ensures a wallet SSH entry exists and uploads its public key to Hetzner.
func setupSSHKey(client *HetznerClient) (SSHKeyConfig, error) {
const vaultTarget = "sandbox/root"
// Ensure wallet entry exists (creates if missing)
fmt.Print(" Ensuring wallet SSH entry... ")
if err := remotessh.EnsureVaultEntry(vaultTarget); err != nil {
fmt.Println("FAILED")
return SSHKeyConfig{}, fmt.Errorf("ensure vault entry: %w", err)
}
fmt.Println("OK")
// Get public key from wallet
fmt.Print(" Resolving public key from wallet... ")
pubStr, err := remotessh.ResolveVaultPublicKey(vaultTarget)
if err != nil {
fmt.Println("FAILED")
return SSHKeyConfig{}, fmt.Errorf("resolve public key: %w", err)
}
fmt.Println("OK")
// Upload to Hetzner (will fail with uniqueness error if already exists)
fmt.Print(" Uploading to Hetzner... ")
key, err := client.UploadSSHKey("orama-sandbox", pubStr)
if err != nil {
// Key may already exist on Hetzner — try to find by fingerprint
existing, listErr := client.ListSSHKeysByFingerprint("") // empty = list all
if listErr == nil {
for _, k := range existing {
if strings.TrimSpace(k.PublicKey) == pubStr {
fmt.Printf("already exists (ID: %d)\n", k.ID)
return SSHKeyConfig{
HetznerID: k.ID,
VaultTarget: vaultTarget,
}, nil
}
}
}
fmt.Println("FAILED")
return SSHKeyConfig{}, fmt.Errorf("upload SSH key: %w", err)
}
fmt.Printf("OK (ID: %d)\n", key.ID)
return SSHKeyConfig{
HetznerID: key.ID,
VaultTarget: vaultTarget,
}, nil
}
// verifyDNS checks if glue records for the sandbox domain are configured.
//
// There's a chicken-and-egg problem: NS records can't fully resolve until
// CoreDNS is running on the floating IPs (which requires a sandbox cluster).
// So instead of resolving NS → A records, we check for glue records at the
// TLD level, which proves the registrar configuration is correct.
func verifyDNS(domain string, floatingIPs []FloatIP, reader *bufio.Reader) {
expectedIPs := make(map[string]bool)
for _, fip := range floatingIPs {
expectedIPs[fip.IP] = true
}
// Find the TLD nameserver to query for glue records
findTLDServer := func() string {
// For "dbrs.space", the TLD is "space." — ask the root for its NS
parts := strings.Split(domain, ".")
if len(parts) < 2 {
return ""
}
tld := parts[len(parts)-1]
out, err := exec.Command("dig", "+short", "NS", tld+".", "@8.8.8.8").Output()
if err != nil {
return ""
}
lines := strings.Split(strings.TrimSpace(string(out)), "\n")
if len(lines) > 0 && lines[0] != "" {
return strings.TrimSpace(lines[0])
}
return ""
}
check := func() (glueFound bool, foundIPs []string) {
tldNS := findTLDServer()
if tldNS == "" {
return false, nil
}
// Query the TLD nameserver for NS + glue of our domain
// dig NS domain @tld-server will include glue in ADDITIONAL section
out, err := exec.Command("dig", "NS", domain, "@"+tldNS, "+norecurse", "+additional").Output()
if err != nil {
return false, nil
}
output := string(out)
remaining := make(map[string]bool)
for k, v := range expectedIPs {
remaining[k] = v
}
// Look for our floating IPs in the ADDITIONAL section (glue records)
// or anywhere in the response
for _, fip := range floatingIPs {
if strings.Contains(output, fip.IP) {
foundIPs = append(foundIPs, fip.IP)
delete(remaining, fip.IP)
}
}
return len(remaining) == 0, foundIPs
}
fmt.Printf(" Checking glue records for %s at TLD nameserver...\n", domain)
matched, foundIPs := check()
if matched {
fmt.Println(" ✓ Glue records configured correctly:")
for i, ip := range foundIPs {
fmt.Printf(" ns%d.%s → %s\n", i+1, domain, ip)
}
fmt.Println()
fmt.Println(" Note: Full DNS resolution will work once a sandbox is running")
fmt.Println(" (CoreDNS on the floating IPs needs to be up to answer queries).")
return
}
if len(foundIPs) > 0 {
fmt.Println(" ⚠ Partial glue records found:")
for _, ip := range foundIPs {
fmt.Printf(" %s\n", ip)
}
fmt.Println(" Missing floating IPs in glue:")
for _, fip := range floatingIPs {
if expectedIPs[fip.IP] {
fmt.Printf(" %s\n", fip.IP)
}
}
} else {
fmt.Println(" ✗ No glue records found yet.")
fmt.Println(" Make sure you configured at your registrar:")
fmt.Printf(" ns1.%s → %s\n", domain, floatingIPs[0].IP)
fmt.Printf(" ns2.%s → %s\n", domain, floatingIPs[1].IP)
}
fmt.Println()
fmt.Print(" Wait for glue propagation? (polls every 30s, Ctrl+C to stop) [y/N]: ")
choice, _ := reader.ReadString('\n')
choice = strings.TrimSpace(strings.ToLower(choice))
if choice != "y" && choice != "yes" {
fmt.Println(" Skipping. You can create the sandbox now — DNS will work once glue propagates.")
return
}
fmt.Println(" Waiting for glue record propagation...")
for i := 1; ; i++ {
time.Sleep(30 * time.Second)
matched, _ = check()
if matched {
fmt.Printf("\n ✓ Glue records propagated after %d checks\n", i)
fmt.Println(" You can now create a sandbox: orama sandbox create")
return
}
fmt.Printf(" [%d] Not yet... checking again in 30s\n", i)
}
}

66
pkg/cli/sandbox/ssh_cmd.go Normal file
View File

@ -0,0 +1,66 @@
package sandbox
import (
"fmt"
"os"
"os/exec"
)
// SSHInto opens an interactive SSH session to a sandbox node.
func SSHInto(name string, nodeNum int) error {
cfg, err := LoadConfig()
if err != nil {
return err
}
state, err := resolveSandbox(name)
if err != nil {
return err
}
if nodeNum < 1 || nodeNum > len(state.Servers) {
return fmt.Errorf("node number must be between 1 and %d", len(state.Servers))
}
srv := state.Servers[nodeNum-1]
sshKeyPath, cleanup, err := resolveVaultKeyOnce(cfg.SSHKey.VaultTarget)
if err != nil {
return fmt.Errorf("prepare SSH key: %w", err)
}
fmt.Printf("Connecting to %s (%s, %s)...\n", srv.Name, srv.IP, srv.Role)
// Find ssh binary
sshBin, err := findSSHBinary()
if err != nil {
cleanup()
return err
}
// Run SSH as a child process so cleanup runs after the session ends
cmd := exec.Command(sshBin,
"-o", "StrictHostKeyChecking=no",
"-o", "UserKnownHostsFile=/dev/null",
"-i", sshKeyPath,
fmt.Sprintf("root@%s", srv.IP),
)
cmd.Stdin = os.Stdin
cmd.Stdout = os.Stdout
cmd.Stderr = os.Stderr
err = cmd.Run()
cleanup()
return err
}
// findSSHBinary locates the ssh binary in PATH.
func findSSHBinary() (string, error) {
paths := []string{"/usr/bin/ssh", "/usr/local/bin/ssh", "/opt/homebrew/bin/ssh"}
for _, p := range paths {
if _, err := os.Stat(p); err == nil {
return p, nil
}
}
return "", fmt.Errorf("ssh binary not found")
}

211
pkg/cli/sandbox/state.go Normal file
View File

@ -0,0 +1,211 @@
package sandbox
import (
"fmt"
"os"
"path/filepath"
"strings"
"time"
"github.com/DeBrosOfficial/network/pkg/inspector"
"gopkg.in/yaml.v3"
)
// SandboxStatus represents the lifecycle state of a sandbox.
type SandboxStatus string
const (
StatusCreating SandboxStatus = "creating"
StatusRunning SandboxStatus = "running"
StatusDestroying SandboxStatus = "destroying"
StatusError SandboxStatus = "error"
)
// SandboxState holds the full state of an active sandbox cluster.
type SandboxState struct {
Name string `yaml:"name"`
CreatedAt time.Time `yaml:"created_at"`
Domain string `yaml:"domain"`
Status SandboxStatus `yaml:"status"`
Servers []ServerState `yaml:"servers"`
}
// ServerState holds the state of a single server in the sandbox.
type ServerState struct {
ID int64 `yaml:"id"` // Hetzner server ID
Name string `yaml:"name"` // e.g., sbx-feature-webrtc-1
IP string `yaml:"ip"` // Public IPv4
Role string `yaml:"role"` // "nameserver" or "node"
FloatingIP string `yaml:"floating_ip,omitempty"` // Only for nameserver nodes
WgIP string `yaml:"wg_ip,omitempty"` // WireGuard IP (populated after install)
}
// sandboxesDir returns ~/.orama/sandboxes/, creating it if needed.
func sandboxesDir() (string, error) {
dir, err := configDir()
if err != nil {
return "", err
}
sbxDir := filepath.Join(dir, "sandboxes")
if err := os.MkdirAll(sbxDir, 0700); err != nil {
return "", fmt.Errorf("create sandboxes directory: %w", err)
}
return sbxDir, nil
}
// statePath returns the path for a sandbox's state file.
func statePath(name string) (string, error) {
dir, err := sandboxesDir()
if err != nil {
return "", err
}
return filepath.Join(dir, name+".yaml"), nil
}
// SaveState persists the sandbox state to disk.
func SaveState(state *SandboxState) error {
path, err := statePath(state.Name)
if err != nil {
return err
}
data, err := yaml.Marshal(state)
if err != nil {
return fmt.Errorf("marshal state: %w", err)
}
if err := os.WriteFile(path, data, 0600); err != nil {
return fmt.Errorf("write state: %w", err)
}
return nil
}
// LoadState reads a sandbox state from disk.
func LoadState(name string) (*SandboxState, error) {
path, err := statePath(name)
if err != nil {
return nil, err
}
data, err := os.ReadFile(path)
if err != nil {
if os.IsNotExist(err) {
return nil, fmt.Errorf("sandbox %q not found", name)
}
return nil, fmt.Errorf("read state: %w", err)
}
var state SandboxState
if err := yaml.Unmarshal(data, &state); err != nil {
return nil, fmt.Errorf("parse state: %w", err)
}
return &state, nil
}
// DeleteState removes the sandbox state file.
func DeleteState(name string) error {
path, err := statePath(name)
if err != nil {
return err
}
if err := os.Remove(path); err != nil && !os.IsNotExist(err) {
return fmt.Errorf("delete state: %w", err)
}
return nil
}
// ListStates returns all sandbox states from disk.
func ListStates() ([]*SandboxState, error) {
dir, err := sandboxesDir()
if err != nil {
return nil, err
}
entries, err := os.ReadDir(dir)
if err != nil {
return nil, fmt.Errorf("read sandboxes directory: %w", err)
}
var states []*SandboxState
for _, entry := range entries {
if entry.IsDir() || !strings.HasSuffix(entry.Name(), ".yaml") {
continue
}
name := strings.TrimSuffix(entry.Name(), ".yaml")
state, err := LoadState(name)
if err != nil {
fmt.Fprintf(os.Stderr, "Warning: could not load sandbox %q: %v\n", name, err)
continue
}
states = append(states, state)
}
return states, nil
}
// FindActiveSandbox returns the first sandbox in running or creating state.
// Returns nil if no active sandbox exists.
func FindActiveSandbox() (*SandboxState, error) {
states, err := ListStates()
if err != nil {
return nil, err
}
for _, s := range states {
if s.Status == StatusRunning || s.Status == StatusCreating {
return s, nil
}
}
return nil, nil
}
// ToNodes converts sandbox servers to inspector.Node structs for SSH operations.
// Sets VaultTarget on each node so PrepareNodeKeys resolves from the wallet.
func (s *SandboxState) ToNodes(vaultTarget string) []inspector.Node {
nodes := make([]inspector.Node, len(s.Servers))
for i, srv := range s.Servers {
nodes[i] = inspector.Node{
Environment: "sandbox",
User: "root",
Host: srv.IP,
Role: srv.Role,
VaultTarget: vaultTarget,
}
}
return nodes
}
// NameserverNodes returns only the nameserver nodes.
func (s *SandboxState) NameserverNodes() []ServerState {
var ns []ServerState
for _, srv := range s.Servers {
if srv.Role == "nameserver" {
ns = append(ns, srv)
}
}
return ns
}
// RegularNodes returns only the non-nameserver nodes.
func (s *SandboxState) RegularNodes() []ServerState {
var nodes []ServerState
for _, srv := range s.Servers {
if srv.Role == "node" {
nodes = append(nodes, srv)
}
}
return nodes
}
// GenesisServer returns the first server (genesis node).
func (s *SandboxState) GenesisServer() ServerState {
if len(s.Servers) == 0 {
return ServerState{}
}
return s.Servers[0]
}

217
pkg/cli/sandbox/state_test.go Normal file
View File

@ -0,0 +1,217 @@
package sandbox
import (
"os"
"path/filepath"
"testing"
"time"
)
func TestSaveAndLoadState(t *testing.T) {
// Use temp dir for test
tmpDir := t.TempDir()
origHome := os.Getenv("HOME")
os.Setenv("HOME", tmpDir)
defer os.Setenv("HOME", origHome)
state := &SandboxState{
Name: "test-sandbox",
CreatedAt: time.Date(2026, 2, 25, 10, 0, 0, 0, time.UTC),
Domain: "test.example.com",
Status: StatusRunning,
Servers: []ServerState{
{ID: 1, Name: "sbx-test-1", IP: "1.1.1.1", Role: "nameserver", FloatingIP: "10.0.0.1", WgIP: "10.0.0.1"},
{ID: 2, Name: "sbx-test-2", IP: "2.2.2.2", Role: "nameserver", FloatingIP: "10.0.0.2", WgIP: "10.0.0.2"},
{ID: 3, Name: "sbx-test-3", IP: "3.3.3.3", Role: "node", WgIP: "10.0.0.3"},
{ID: 4, Name: "sbx-test-4", IP: "4.4.4.4", Role: "node", WgIP: "10.0.0.4"},
{ID: 5, Name: "sbx-test-5", IP: "5.5.5.5", Role: "node", WgIP: "10.0.0.5"},
},
}
if err := SaveState(state); err != nil {
t.Fatalf("SaveState() error = %v", err)
}
// Verify file exists
expected := filepath.Join(tmpDir, ".orama", "sandboxes", "test-sandbox.yaml")
if _, err := os.Stat(expected); err != nil {
t.Fatalf("state file not created at %s: %v", expected, err)
}
// Load back
loaded, err := LoadState("test-sandbox")
if err != nil {
t.Fatalf("LoadState() error = %v", err)
}
if loaded.Name != "test-sandbox" {
t.Errorf("name = %s, want test-sandbox", loaded.Name)
}
if loaded.Domain != "test.example.com" {
t.Errorf("domain = %s, want test.example.com", loaded.Domain)
}
if loaded.Status != StatusRunning {
t.Errorf("status = %s, want running", loaded.Status)
}
if len(loaded.Servers) != 5 {
t.Errorf("servers = %d, want 5", len(loaded.Servers))
}
}
func TestLoadState_NotFound(t *testing.T) {
tmpDir := t.TempDir()
origHome := os.Getenv("HOME")
os.Setenv("HOME", tmpDir)
defer os.Setenv("HOME", origHome)
_, err := LoadState("nonexistent")
if err == nil {
t.Error("LoadState() expected error for nonexistent sandbox")
}
}
func TestDeleteState(t *testing.T) {
tmpDir := t.TempDir()
origHome := os.Getenv("HOME")
os.Setenv("HOME", tmpDir)
defer os.Setenv("HOME", origHome)
state := &SandboxState{
Name: "to-delete",
Status: StatusRunning,
}
if err := SaveState(state); err != nil {
t.Fatalf("SaveState() error = %v", err)
}
if err := DeleteState("to-delete"); err != nil {
t.Fatalf("DeleteState() error = %v", err)
}
_, err := LoadState("to-delete")
if err == nil {
t.Error("LoadState() should fail after DeleteState()")
}
}
func TestListStates(t *testing.T) {
tmpDir := t.TempDir()
origHome := os.Getenv("HOME")
os.Setenv("HOME", tmpDir)
defer os.Setenv("HOME", origHome)
// Create 2 sandboxes
for _, name := range []string{"sandbox-a", "sandbox-b"} {
if err := SaveState(&SandboxState{Name: name, Status: StatusRunning}); err != nil {
t.Fatalf("SaveState(%s) error = %v", name, err)
}
}
states, err := ListStates()
if err != nil {
t.Fatalf("ListStates() error = %v", err)
}
if len(states) != 2 {
t.Errorf("ListStates() returned %d, want 2", len(states))
}
}
func TestFindActiveSandbox(t *testing.T) {
tmpDir := t.TempDir()
origHome := os.Getenv("HOME")
os.Setenv("HOME", tmpDir)
defer os.Setenv("HOME", origHome)
// No sandboxes
active, err := FindActiveSandbox()
if err != nil {
t.Fatalf("FindActiveSandbox() error = %v", err)
}
if active != nil {
t.Error("expected nil when no sandboxes exist")
}
// Add one running sandbox
if err := SaveState(&SandboxState{Name: "active-one", Status: StatusRunning}); err != nil {
t.Fatal(err)
}
if err := SaveState(&SandboxState{Name: "errored-one", Status: StatusError}); err != nil {
t.Fatal(err)
}
active, err = FindActiveSandbox()
if err != nil {
t.Fatalf("FindActiveSandbox() error = %v", err)
}
if active == nil || active.Name != "active-one" {
t.Errorf("FindActiveSandbox() = %v, want active-one", active)
}
}
func TestToNodes(t *testing.T) {
state := &SandboxState{
Servers: []ServerState{
{IP: "1.1.1.1", Role: "nameserver"},
{IP: "2.2.2.2", Role: "node"},
},
}
nodes := state.ToNodes("sandbox/root")
if len(nodes) != 2 {
t.Fatalf("ToNodes() returned %d nodes, want 2", len(nodes))
}
if nodes[0].Host != "1.1.1.1" {
t.Errorf("node[0].Host = %s, want 1.1.1.1", nodes[0].Host)
}
if nodes[0].User != "root" {
t.Errorf("node[0].User = %s, want root", nodes[0].User)
}
if nodes[0].VaultTarget != "sandbox/root" {
t.Errorf("node[0].VaultTarget = %s, want sandbox/root", nodes[0].VaultTarget)
}
if nodes[0].SSHKey != "" {
t.Errorf("node[0].SSHKey = %s, want empty (set by PrepareNodeKeys)", nodes[0].SSHKey)
}
if nodes[0].Environment != "sandbox" {
t.Errorf("node[0].Environment = %s, want sandbox", nodes[0].Environment)
}
}
func TestNameserverAndRegularNodes(t *testing.T) {
state := &SandboxState{
Servers: []ServerState{
{Role: "nameserver"},
{Role: "nameserver"},
{Role: "node"},
{Role: "node"},
{Role: "node"},
},
}
ns := state.NameserverNodes()
if len(ns) != 2 {
t.Errorf("NameserverNodes() = %d, want 2", len(ns))
}
regular := state.RegularNodes()
if len(regular) != 3 {
t.Errorf("RegularNodes() = %d, want 3", len(regular))
}
}
func TestGenesisServer(t *testing.T) {
state := &SandboxState{
Servers: []ServerState{
{Name: "first"},
{Name: "second"},
},
}
if state.GenesisServer().Name != "first" {
t.Errorf("GenesisServer().Name = %s, want first", state.GenesisServer().Name)
}
empty := &SandboxState{}
if empty.GenesisServer().Name != "" {
t.Error("GenesisServer() on empty state should return zero value")
}
}

165
pkg/cli/sandbox/status.go Normal file
View File

@ -0,0 +1,165 @@
package sandbox
import (
"encoding/json"
"fmt"
"strings"
"github.com/DeBrosOfficial/network/pkg/inspector"
)
// List prints all sandbox clusters.
func List() error {
states, err := ListStates()
if err != nil {
return err
}
if len(states) == 0 {
fmt.Println("No sandboxes found.")
fmt.Println("Create one: orama sandbox create")
return nil
}
fmt.Printf("%-20s %-10s %-5s %-25s %s\n", "NAME", "STATUS", "NODES", "CREATED", "DOMAIN")
for _, s := range states {
fmt.Printf("%-20s %-10s %-5d %-25s %s\n",
s.Name, s.Status, len(s.Servers), s.CreatedAt.Format("2006-01-02 15:04"), s.Domain)
}
// Check for orphaned servers on Hetzner
cfg, err := LoadConfig()
if err != nil {
return nil // Config not set up, skip orphan check
}
client := NewHetznerClient(cfg.HetznerAPIToken)
hetznerServers, err := client.ListServersByLabel("orama-sandbox")
if err != nil {
return nil // API error, skip orphan check
}
// Build set of known server IDs
known := make(map[int64]bool)
for _, s := range states {
for _, srv := range s.Servers {
known[srv.ID] = true
}
}
var orphans []string
for _, srv := range hetznerServers {
if !known[srv.ID] {
orphans = append(orphans, fmt.Sprintf("%s (ID: %d, IP: %s)", srv.Name, srv.ID, srv.PublicNet.IPv4.IP))
}
}
if len(orphans) > 0 {
fmt.Printf("\nWarning: %d orphaned server(s) on Hetzner (no state file):\n", len(orphans))
for _, o := range orphans {
fmt.Printf(" %s\n", o)
}
fmt.Println("Delete manually at https://console.hetzner.cloud")
}
return nil
}
// Status prints the health report for a sandbox cluster.
func Status(name string) error {
cfg, err := LoadConfig()
if err != nil {
return err
}
state, err := resolveSandbox(name)
if err != nil {
return err
}
sshKeyPath, cleanup, err := resolveVaultKeyOnce(cfg.SSHKey.VaultTarget)
if err != nil {
return fmt.Errorf("prepare SSH key: %w", err)
}
defer cleanup()
fmt.Printf("Sandbox: %s (status: %s)\n\n", state.Name, state.Status)
for _, srv := range state.Servers {
node := inspector.Node{User: "root", Host: srv.IP, SSHKey: sshKeyPath}
fmt.Printf("%s (%s) — %s\n", srv.Name, srv.IP, srv.Role)
// Get node report
out, err := runSSHOutput(node, "orama node report --json 2>/dev/null")
if err != nil {
fmt.Printf(" Status: UNREACHABLE (%v)\n", err)
fmt.Println()
continue
}
printNodeReport(out)
fmt.Println()
}
// Cluster summary
fmt.Println("Cluster Summary")
fmt.Println("---------------")
genesis := state.GenesisServer()
genesisNode := inspector.Node{User: "root", Host: genesis.IP, SSHKey: sshKeyPath}
out, err := runSSHOutput(genesisNode, "curl -sf http://localhost:5001/status 2>/dev/null")
if err != nil {
fmt.Println(" RQLite: UNREACHABLE")
} else {
var status map[string]interface{}
if err := json.Unmarshal([]byte(out), &status); err == nil {
if store, ok := status["store"].(map[string]interface{}); ok {
if raft, ok := store["raft"].(map[string]interface{}); ok {
fmt.Printf(" RQLite state: %v\n", raft["state"])
fmt.Printf(" Commit index: %v\n", raft["commit_index"])
if nodes, ok := raft["nodes"].([]interface{}); ok {
fmt.Printf(" Nodes: %d\n", len(nodes))
}
}
}
}
}
return nil
}
// printNodeReport parses and prints a node report JSON.
func printNodeReport(jsonStr string) {
var report map[string]interface{}
if err := json.Unmarshal([]byte(jsonStr), &report); err != nil {
fmt.Printf(" Report: (parse error)\n")
return
}
// Print key fields
if services, ok := report["services"].(map[string]interface{}); ok {
var active, inactive []string
for name, info := range services {
if svc, ok := info.(map[string]interface{}); ok {
if state, ok := svc["active"].(bool); ok && state {
active = append(active, name)
} else {
inactive = append(inactive, name)
}
}
}
if len(active) > 0 {
fmt.Printf(" Active: %s\n", strings.Join(active, ", "))
}
if len(inactive) > 0 {
fmt.Printf(" Inactive: %s\n", strings.Join(inactive, ", "))
}
}
if rqlite, ok := report["rqlite"].(map[string]interface{}); ok {
if state, ok := rqlite["state"].(string); ok {
fmt.Printf(" RQLite: %s\n", state)
}
}
}

31
pkg/client/database_client.go
View File

@ -9,26 +9,29 @@ import (
"github.com/rqlite/gorqlite"
)
// safeWriteOneParameterized wraps conn.WriteOneParameterized with panic recovery.
// gorqlite panics with "index out of range" when RQLite returns empty results
// during temporary unavailability. This converts the panic to a normal error.
func safeWriteOneParameterized(conn *gorqlite.Connection, stmt gorqlite.ParameterizedStatement) (result gorqlite.WriteResult, err error) {
// safeWriteOne wraps gorqlite's WriteOneParameterized to recover from panics.
// gorqlite's WriteOne* functions access wra[0] without checking if the slice
// is empty, which panics when the server returns an error (e.g. "leader not found")
// with no result rows.
func safeWriteOne(conn *gorqlite.Connection, stmt gorqlite.ParameterizedStatement) (wr gorqlite.WriteResult, err error) {
defer func() {
if r := recover(); r != nil {
err = fmt.Errorf("gorqlite panic (WriteOneParameterized): %v", r)
err = fmt.Errorf("rqlite write failed (recovered panic): %v", r)
}
}()
return conn.WriteOneParameterized(stmt)
wr, err = conn.WriteOneParameterized(stmt)
return
}
// safeWriteOne wraps conn.WriteOne with panic recovery.
func safeWriteOne(conn *gorqlite.Connection, query string) (result gorqlite.WriteResult, err error) {
// safeWriteOneRaw wraps gorqlite's WriteOne to recover from panics.
func safeWriteOneRaw(conn *gorqlite.Connection, sql string) (wr gorqlite.WriteResult, err error) {
defer func() {
if r := recover(); r != nil {
err = fmt.Errorf("gorqlite panic (WriteOne): %v", r)
err = fmt.Errorf("rqlite write failed (recovered panic): %v", r)
}
}()
return conn.WriteOne(query)
wr, err = conn.WriteOne(sql)
return
}
// DatabaseClientImpl implements DatabaseClient
@ -101,7 +104,7 @@ func (d *DatabaseClientImpl) Query(ctx context.Context, sql string, args ...inte
if isWriteOperation {
// Execute write operation with parameters
_, err := safeWriteOneParameterized(conn, gorqlite.ParameterizedStatement{
_, err := safeWriteOne(conn, gorqlite.ParameterizedStatement{
Query: sql,
Arguments: args,
})
@ -315,7 +318,7 @@ func (d *DatabaseClientImpl) Transaction(ctx context.Context, queries []string)
// Execute all queries in the transaction
success := true
for _, query := range queries {
_, err := safeWriteOne(conn, query)
_, err := safeWriteOneRaw(conn, query)
if err != nil {
lastErr = err
success = false
@ -343,7 +346,7 @@ func (d *DatabaseClientImpl) CreateTable(ctx context.Context, schema string) err
}
return d.withRetry(func(conn *gorqlite.Connection) error {
_, err := safeWriteOne(conn, schema)
_, err := safeWriteOneRaw(conn, schema)
return err
})
}
@ -356,7 +359,7 @@ func (d *DatabaseClientImpl) DropTable(ctx context.Context, tableName string) er
return d.withRetry(func(conn *gorqlite.Connection) error {
dropSQL := fmt.Sprintf("DROP TABLE IF EXISTS %s", tableName)
_, err := safeWriteOne(conn, dropSQL)
_, err := safeWriteOneRaw(conn, dropSQL)
return err
})
}

82
pkg/client/database_client_test.go Normal file
View File

@ -0,0 +1,82 @@
package client
import (
"fmt"
"testing"
"github.com/rqlite/gorqlite"
)
// mockPanicConnection simulates what gorqlite does when WriteParameterized
// returns an empty slice: accessing [0] panics.
func simulateGorqlitePanic() (gorqlite.WriteResult, error) {
var empty []gorqlite.WriteResult
return empty[0], fmt.Errorf("leader not found") // panics
}
func TestSafeWriteOne_recoversPanic(t *testing.T) {
// We can't easily create a real gorqlite.Connection that panics,
// but we can verify our recovery wrapper works by testing the
// recovery pattern directly.
var recovered bool
func() {
defer func() {
if r := recover(); r != nil {
recovered = true
}
}()
simulateGorqlitePanic()
}()
if !recovered {
t.Fatal("expected simulateGorqlitePanic to panic, but it didn't")
}
}
func TestSafeWriteOne_nilConnection(t *testing.T) {
// safeWriteOne with nil connection should recover from panic, not crash.
_, err := safeWriteOne(nil, gorqlite.ParameterizedStatement{
Query: "INSERT INTO test (a) VALUES (?)",
Arguments: []interface{}{"x"},
})
if err == nil {
t.Fatal("expected error from nil connection, got nil")
}
}
func TestSafeWriteOneRaw_nilConnection(t *testing.T) {
// safeWriteOneRaw with nil connection should recover from panic, not crash.
_, err := safeWriteOneRaw(nil, "INSERT INTO test (a) VALUES ('x')")
if err == nil {
t.Fatal("expected error from nil connection, got nil")
}
}
func TestIsWriteOperation(t *testing.T) {
d := &DatabaseClientImpl{}
tests := []struct {
sql string
isWrite bool
}{
{"INSERT INTO foo VALUES (1)", true},
{" INSERT INTO foo VALUES (1)", true},
{"UPDATE foo SET a = 1", true},
{"DELETE FROM foo", true},
{"CREATE TABLE foo (a TEXT)", true},
{"DROP TABLE foo", true},
{"ALTER TABLE foo ADD COLUMN b TEXT", true},
{"SELECT * FROM foo", false},
{" SELECT * FROM foo", false},
{"EXPLAIN SELECT * FROM foo", false},
}
for _, tt := range tests {
t.Run(tt.sql, func(t *testing.T) {
got := d.isWriteOperation(tt.sql)
if got != tt.isWrite {
t.Errorf("isWriteOperation(%q) = %v, want %v", tt.sql, got, tt.isWrite)
}
})
}
}

7
pkg/config/database_config.go
View File

@ -22,6 +22,13 @@ type DatabaseConfig struct {
NodeCACert string `yaml:"node_ca_cert"` // Path to CA certificate (optional, uses system CA if not set)
NodeNoVerify bool `yaml:"node_no_verify"` // Skip certificate verification (for testing/self-signed certs)
// RQLite HTTP Basic Auth credentials.
// When RQLiteAuthFile is set, rqlited is launched with `-auth <file>`.
// Username/password are embedded in all client DSNs (harmless when auth not enforced).
RQLiteUsername string `yaml:"rqlite_username"`
RQLitePassword string `yaml:"rqlite_password"`
RQLiteAuthFile string `yaml:"rqlite_auth_file"` // Path to RQLite auth JSON file. Empty = auth not enforced.
// Raft tuning (passed through to rqlited CLI flags).
// Higher defaults than rqlited's 1s suit WireGuard latency.
RaftElectionTimeout time.Duration `yaml:"raft_election_timeout"` // default: 5s

13
pkg/constants/versions.go Normal file
View File

@ -0,0 +1,13 @@
package constants
// External dependency versions used across the network.
// Single source of truth — all installer files and build scripts import from here.
const (
GoVersion = "1.24.6"
OlricVersion = "v0.7.0"
IPFSKuboVersion = "v0.38.2"
IPFSClusterVersion = "v1.1.2"
RQLiteVersion = "8.43.0"
CoreDNSVersion = "1.12.0"
CaddyVersion = "2.10.2"
)

7
pkg/coredns/rqlite/backend.go
View File

@ -31,9 +31,10 @@ type Backend struct {
healthy bool
}
// NewBackend creates a new RQLite backend
func NewBackend(dsn string, refreshRate time.Duration, logger *zap.Logger) (*Backend, error) {
client, err := NewRQLiteClient(dsn, logger)
// NewBackend creates a new RQLite backend.
// Optional username/password enable HTTP basic auth for RQLite connections.
func NewBackend(dsn string, refreshRate time.Duration, logger *zap.Logger, username, password string) (*Backend, error) {
client, err := NewRQLiteClient(dsn, logger, username, password)
if err != nil {
return nil, fmt.Errorf("failed to create RQLite client: %w", err)
}

14
pkg/coredns/rqlite/client.go
View File

@ -15,6 +15,8 @@ import (
// RQLiteClient is a simple HTTP client for RQLite
type RQLiteClient struct {
baseURL string
username string // HTTP basic auth username (empty = no auth)
password string // HTTP basic auth password
httpClient *http.Client
logger *zap.Logger
}
@ -32,10 +34,13 @@ type QueryResult struct {
Error string `json:"error"`
}
// NewRQLiteClient creates a new RQLite HTTP client
func NewRQLiteClient(dsn string, logger *zap.Logger) (*RQLiteClient, error) {
// NewRQLiteClient creates a new RQLite HTTP client.
// Optional username/password enable HTTP basic auth on all requests.
func NewRQLiteClient(dsn string, logger *zap.Logger, username, password string) (*RQLiteClient, error) {
return &RQLiteClient{
baseURL: dsn,
baseURL: dsn,
username: username,
password: password,
httpClient: &http.Client{
Timeout: 10 * time.Second,
Transport: &http.Transport{
@ -65,6 +70,9 @@ func (c *RQLiteClient) Query(ctx context.Context, query string, args ...interfac
}
req.Header.Set("Content-Type", "application/json")
if c.username != "" && c.password != "" {
req.SetBasicAuth(c.username, c.password)
}
resp, err := c.httpClient.Do(req)
if err != nil {

26
pkg/coredns/rqlite/setup.go
View File

@ -38,11 +38,13 @@ func parseConfig(c *caddy.Controller) (*RQLitePlugin, error) {
}
var (
dsn = "http://localhost:5001"
refreshRate = 10 * time.Second
cacheTTL = 30 * time.Second
cacheSize = 10000
zones []string
dsn = "http://localhost:5001"
refreshRate = 10 * time.Second
cacheTTL = 30 * time.Second
cacheSize = 10000
rqliteUsername string
rqlitePassword string
zones []string
)
// Parse zone arguments
@ -90,6 +92,18 @@ func parseConfig(c *caddy.Controller) (*RQLitePlugin, error) {
}
cacheSize = size
case "username":
if !c.NextArg() {
return nil, c.ArgErr()
}
rqliteUsername = c.Val()
case "password":
if !c.NextArg() {
return nil, c.ArgErr()
}
rqlitePassword = c.Val()
default:
return nil, c.Errf("unknown property '%s'", c.Val())
}
@ -101,7 +115,7 @@ func parseConfig(c *caddy.Controller) (*RQLitePlugin, error) {
}
// Create backend
backend, err := NewBackend(dsn, refreshRate, logger)
backend, err := NewBackend(dsn, refreshRate, logger, rqliteUsername, rqlitePassword)
if err != nil {
return nil, fmt.Errorf("failed to create backend: %w", err)
}

194
pkg/encryption/wallet_keygen.go
View File

@ -1,194 +0,0 @@
package encryption
import (
"crypto/ed25519"
"crypto/sha256"
"fmt"
"io"
"os"
"os/exec"
"strings"
"golang.org/x/crypto/curve25519"
"golang.org/x/crypto/hkdf"
)
// NodeKeys holds all cryptographic keys derived from a wallet's master key.
type NodeKeys struct {
LibP2PPrivateKey ed25519.PrivateKey // Ed25519 for LibP2P identity
LibP2PPublicKey ed25519.PublicKey
WireGuardKey [32]byte // Curve25519 private key (clamped)
WireGuardPubKey [32]byte // Curve25519 public key
IPFSPrivateKey ed25519.PrivateKey
IPFSPublicKey ed25519.PublicKey
ClusterPrivateKey ed25519.PrivateKey // IPFS Cluster identity
ClusterPublicKey ed25519.PublicKey
JWTPrivateKey ed25519.PrivateKey // EdDSA JWT signing key
JWTPublicKey ed25519.PublicKey
}
// DeriveNodeKeysFromWallet calls `rw derive` to get a master key from the user's
// Root Wallet, then expands it into all node keys. The wallet's private key never
// leaves the `rw` process.
//
// vpsIP is used as the HKDF info parameter, so each VPS gets unique keys from the
// same wallet. Stdin is passed through so rw can prompt for the wallet password.
func DeriveNodeKeysFromWallet(vpsIP string) (*NodeKeys, error) {
if vpsIP == "" {
return nil, fmt.Errorf("VPS IP is required for key derivation")
}
// Check rw is installed
if _, err := exec.LookPath("rw"); err != nil {
return nil, fmt.Errorf("Root Wallet (rw) not found in PATH — install it first")
}
// Call rw derive to get master key bytes
cmd := exec.Command("rw", "derive", "--salt", "orama-node", "--info", vpsIP)
cmd.Stdin = os.Stdin // pass through for password prompts
cmd.Stderr = os.Stderr // rw UI messages go to terminal
out, err := cmd.Output()
if err != nil {
return nil, fmt.Errorf("rw derive failed: %w", err)
}
masterHex := strings.TrimSpace(string(out))
if len(masterHex) != 64 { // 32 bytes = 64 hex chars
return nil, fmt.Errorf("rw derive returned unexpected output length: %d (expected 64 hex chars)", len(masterHex))
}
masterKey, err := hexToBytes(masterHex)
if err != nil {
return nil, fmt.Errorf("rw derive returned invalid hex: %w", err)
}
defer zeroBytes(masterKey)
return ExpandNodeKeys(masterKey)
}
// ExpandNodeKeys expands a 32-byte master key into all node keys using HKDF-SHA256.
// The master key should come from `rw derive --salt "orama-node" --info "<IP>"`.
//
// Each key type uses a different HKDF info string under the salt "orama-expand",
// ensuring cryptographic independence between key types.
func ExpandNodeKeys(masterKey []byte) (*NodeKeys, error) {
if len(masterKey) != 32 {
return nil, fmt.Errorf("master key must be 32 bytes, got %d", len(masterKey))
}
salt := []byte("orama-expand")
keys := &NodeKeys{}
// Derive LibP2P Ed25519 key
seed, err := deriveBytes(masterKey, salt, []byte("libp2p-identity"), ed25519.SeedSize)
if err != nil {
return nil, fmt.Errorf("failed to derive libp2p key: %w", err)
}
priv := ed25519.NewKeyFromSeed(seed)
zeroBytes(seed)
keys.LibP2PPrivateKey = priv
keys.LibP2PPublicKey = priv.Public().(ed25519.PublicKey)
// Derive WireGuard Curve25519 key
wgSeed, err := deriveBytes(masterKey, salt, []byte("wireguard-key"), 32)
if err != nil {
return nil, fmt.Errorf("failed to derive wireguard key: %w", err)
}
copy(keys.WireGuardKey[:], wgSeed)
zeroBytes(wgSeed)
clampCurve25519Key(&keys.WireGuardKey)
pubKey, err := curve25519.X25519(keys.WireGuardKey[:], curve25519.Basepoint)
if err != nil {
return nil, fmt.Errorf("failed to compute wireguard public key: %w", err)
}
copy(keys.WireGuardPubKey[:], pubKey)
// Derive IPFS Ed25519 key
seed, err = deriveBytes(masterKey, salt, []byte("ipfs-identity"), ed25519.SeedSize)
if err != nil {
return nil, fmt.Errorf("failed to derive ipfs key: %w", err)
}
priv = ed25519.NewKeyFromSeed(seed)
zeroBytes(seed)
keys.IPFSPrivateKey = priv
keys.IPFSPublicKey = priv.Public().(ed25519.PublicKey)
// Derive IPFS Cluster Ed25519 key
seed, err = deriveBytes(masterKey, salt, []byte("ipfs-cluster"), ed25519.SeedSize)
if err != nil {
return nil, fmt.Errorf("failed to derive cluster key: %w", err)
}
priv = ed25519.NewKeyFromSeed(seed)
zeroBytes(seed)
keys.ClusterPrivateKey = priv
keys.ClusterPublicKey = priv.Public().(ed25519.PublicKey)
// Derive JWT EdDSA signing key
seed, err = deriveBytes(masterKey, salt, []byte("jwt-signing"), ed25519.SeedSize)
if err != nil {
return nil, fmt.Errorf("failed to derive jwt key: %w", err)
}
priv = ed25519.NewKeyFromSeed(seed)
zeroBytes(seed)
keys.JWTPrivateKey = priv
keys.JWTPublicKey = priv.Public().(ed25519.PublicKey)
return keys, nil
}
// deriveBytes uses HKDF-SHA256 to derive n bytes from the given IKM, salt, and info.
func deriveBytes(ikm, salt, info []byte, n int) ([]byte, error) {
hkdfReader := hkdf.New(sha256.New, ikm, salt, info)
out := make([]byte, n)
if _, err := io.ReadFull(hkdfReader, out); err != nil {
return nil, err
}
return out, nil
}
// clampCurve25519Key applies the standard Curve25519 clamping to a private key.
func clampCurve25519Key(key *[32]byte) {
key[0] &= 248
key[31] &= 127
key[31] |= 64
}
// hexToBytes decodes a hex string to bytes.
func hexToBytes(hex string) ([]byte, error) {
if len(hex)%2 != 0 {
return nil, fmt.Errorf("odd-length hex string")
}
b := make([]byte, len(hex)/2)
for i := 0; i < len(hex); i += 2 {
var hi, lo byte
var err error
if hi, err = hexCharToByte(hex[i]); err != nil {
return nil, err
}
if lo, err = hexCharToByte(hex[i+1]); err != nil {
return nil, err
}
b[i/2] = hi<<4 | lo
}
return b, nil
}
func hexCharToByte(c byte) (byte, error) {
switch {
case c >= '0' && c <= '9':
return c - '0', nil
case c >= 'a' && c <= 'f':
return c - 'a' + 10, nil
case c >= 'A' && c <= 'F':
return c - 'A' + 10, nil
default:
return 0, fmt.Errorf("invalid hex character: %c", c)
}
}
// zeroBytes zeroes a byte slice to clear sensitive data from memory.
func zeroBytes(b []byte) {
for i := range b {
b[i] = 0
}
}

202
pkg/encryption/wallet_keygen_test.go
View File

@ -1,202 +0,0 @@
package encryption
import (
"bytes"
"crypto/ed25519"
"testing"
)
// testMasterKey is a deterministic 32-byte key for testing ExpandNodeKeys.
// In production, this comes from `rw derive --salt "orama-node" --info "<IP>"`.
var testMasterKey = bytes.Repeat([]byte{0xab}, 32)
var testMasterKey2 = bytes.Repeat([]byte{0xcd}, 32)
func TestExpandNodeKeys_Determinism(t *testing.T) {
keys1, err := ExpandNodeKeys(testMasterKey)
if err != nil {
t.Fatalf("ExpandNodeKeys: %v", err)
}
keys2, err := ExpandNodeKeys(testMasterKey)
if err != nil {
t.Fatalf("ExpandNodeKeys (second): %v", err)
}
if !bytes.Equal(keys1.LibP2PPrivateKey, keys2.LibP2PPrivateKey) {
t.Error("LibP2P private keys differ for same input")
}
if !bytes.Equal(keys1.WireGuardKey[:], keys2.WireGuardKey[:]) {
t.Error("WireGuard keys differ for same input")
}
if !bytes.Equal(keys1.IPFSPrivateKey, keys2.IPFSPrivateKey) {
t.Error("IPFS private keys differ for same input")
}
if !bytes.Equal(keys1.ClusterPrivateKey, keys2.ClusterPrivateKey) {
t.Error("Cluster private keys differ for same input")
}
if !bytes.Equal(keys1.JWTPrivateKey, keys2.JWTPrivateKey) {
t.Error("JWT private keys differ for same input")
}
}
func TestExpandNodeKeys_Uniqueness(t *testing.T) {
keys1, err := ExpandNodeKeys(testMasterKey)
if err != nil {
t.Fatalf("ExpandNodeKeys(master1): %v", err)
}
keys2, err := ExpandNodeKeys(testMasterKey2)
if err != nil {
t.Fatalf("ExpandNodeKeys(master2): %v", err)
}
if bytes.Equal(keys1.LibP2PPrivateKey, keys2.LibP2PPrivateKey) {
t.Error("LibP2P keys should differ for different master keys")
}
if bytes.Equal(keys1.WireGuardKey[:], keys2.WireGuardKey[:]) {
t.Error("WireGuard keys should differ for different master keys")
}
if bytes.Equal(keys1.IPFSPrivateKey, keys2.IPFSPrivateKey) {
t.Error("IPFS keys should differ for different master keys")
}
if bytes.Equal(keys1.ClusterPrivateKey, keys2.ClusterPrivateKey) {
t.Error("Cluster keys should differ for different master keys")
}
if bytes.Equal(keys1.JWTPrivateKey, keys2.JWTPrivateKey) {
t.Error("JWT keys should differ for different master keys")
}
}
func TestExpandNodeKeys_KeysAreMutuallyUnique(t *testing.T) {
keys, err := ExpandNodeKeys(testMasterKey)
if err != nil {
t.Fatalf("ExpandNodeKeys: %v", err)
}
privKeys := [][]byte{
keys.LibP2PPrivateKey.Seed(),
keys.IPFSPrivateKey.Seed(),
keys.ClusterPrivateKey.Seed(),
keys.JWTPrivateKey.Seed(),
keys.WireGuardKey[:],
}
labels := []string{"LibP2P", "IPFS", "Cluster", "JWT", "WireGuard"}
for i := 0; i < len(privKeys); i++ {
for j := i + 1; j < len(privKeys); j++ {
if bytes.Equal(privKeys[i], privKeys[j]) {
t.Errorf("%s and %s keys should differ", labels[i], labels[j])
}
}
}
}
func TestExpandNodeKeys_Ed25519Validity(t *testing.T) {
keys, err := ExpandNodeKeys(testMasterKey)
if err != nil {
t.Fatalf("ExpandNodeKeys: %v", err)
}
msg := []byte("test message for verification")
pairs := []struct {
name string
priv ed25519.PrivateKey
pub ed25519.PublicKey
}{
{"LibP2P", keys.LibP2PPrivateKey, keys.LibP2PPublicKey},
{"IPFS", keys.IPFSPrivateKey, keys.IPFSPublicKey},
{"Cluster", keys.ClusterPrivateKey, keys.ClusterPublicKey},
{"JWT", keys.JWTPrivateKey, keys.JWTPublicKey},
}
for _, p := range pairs {
signature := ed25519.Sign(p.priv, msg)
if !ed25519.Verify(p.pub, msg, signature) {
t.Errorf("%s key pair: signature verification failed", p.name)
}
}
}
func TestExpandNodeKeys_WireGuardClamping(t *testing.T) {
keys, err := ExpandNodeKeys(testMasterKey)
if err != nil {
t.Fatalf("ExpandNodeKeys: %v", err)
}
if keys.WireGuardKey[0]&7 != 0 {
t.Errorf("WireGuard key not properly clamped: low 3 bits of first byte should be 0, got %08b", keys.WireGuardKey[0])
}
if keys.WireGuardKey[31]&128 != 0 {
t.Errorf("WireGuard key not properly clamped: high bit of last byte should be 0, got %08b", keys.WireGuardKey[31])
}
if keys.WireGuardKey[31]&64 != 64 {
t.Errorf("WireGuard key not properly clamped: second-high bit of last byte should be 1, got %08b", keys.WireGuardKey[31])
}
var zero [32]byte
if keys.WireGuardPubKey == zero {
t.Error("WireGuard public key is all zeros")
}
}
func TestExpandNodeKeys_InvalidMasterKeyLength(t *testing.T) {
_, err := ExpandNodeKeys(nil)
if err == nil {
t.Error("expected error for nil master key")
}
_, err = ExpandNodeKeys([]byte{})
if err == nil {
t.Error("expected error for empty master key")
}
_, err = ExpandNodeKeys(make([]byte, 16))
if err == nil {
t.Error("expected error for 16-byte master key")
}
_, err = ExpandNodeKeys(make([]byte, 64))
if err == nil {
t.Error("expected error for 64-byte master key")
}
}
func TestHexToBytes(t *testing.T) {
tests := []struct {
input string
expected []byte
wantErr bool
}{
{"", []byte{}, false},
{"00", []byte{0}, false},
{"ff", []byte{255}, false},
{"FF", []byte{255}, false},
{"0a1b2c", []byte{10, 27, 44}, false},
{"0", nil, true}, // odd length
{"zz", nil, true}, // invalid chars
{"gg", nil, true}, // invalid chars
}
for _, tt := range tests {
got, err := hexToBytes(tt.input)
if tt.wantErr {
if err == nil {
t.Errorf("hexToBytes(%q): expected error", tt.input)
}
continue
}
if err != nil {
t.Errorf("hexToBytes(%q): unexpected error: %v", tt.input, err)
continue
}
if !bytes.Equal(got, tt.expected) {
t.Errorf("hexToBytes(%q) = %v, want %v", tt.input, got, tt.expected)
}
}
}
func TestDeriveNodeKeysFromWallet_EmptyIP(t *testing.T) {
_, err := DeriveNodeKeysFromWallet("")
if err == nil {
t.Error("expected error for empty VPS IP")
}
}

166
pkg/environments/production/config.go
View File

@ -2,6 +2,7 @@ package production
import (
"crypto/rand"
"encoding/base64"
"encoding/hex"
"fmt"
"net"
@ -194,6 +195,30 @@ func (cg *ConfigGenerator) GenerateNodeConfig(peerAddresses []string, vpsIP stri
return templates.RenderNodeConfig(data)
}
// GenerateVaultConfig generates vault.yaml configuration for the Vault Guardian.
// The vault config uses key=value format (not YAML, despite the file extension).
// Peer discovery is dynamic via RQLite — no static peer list needed.
func (cg *ConfigGenerator) GenerateVaultConfig(vpsIP string) string {
dataDir := filepath.Join(cg.oramaDir, "data", "vault")
// Bind to WireGuard IP so vault is only accessible over the overlay network.
// If no WG IP is provided, bind to localhost as a safe default.
bindAddr := "127.0.0.1"
if vpsIP != "" {
bindAddr = vpsIP
}
return fmt.Sprintf(`# Vault Guardian Configuration
# Generated by orama node install
listen_address = %s
client_port = 7500
peer_port = 7501
data_dir = %s
rqlite_url = http://127.0.0.1:5001
`, bindAddr, dataDir)
}
// GenerateGatewayConfig generates gateway.yaml configuration
func (cg *ConfigGenerator) GenerateGatewayConfig(peerAddresses []string, enableHTTPS bool, domain string, olricServers []string) (string, error) {
tlsCacheDir := ""
@ -215,8 +240,15 @@ func (cg *ConfigGenerator) GenerateGatewayConfig(peerAddresses []string, enableH
return templates.RenderGatewayConfig(data)
}
// GenerateOlricConfig generates Olric configuration
// GenerateOlricConfig generates Olric configuration.
// Reads the Olric encryption key from secrets if available.
func (cg *ConfigGenerator) GenerateOlricConfig(serverBindAddr string, httpPort int, memberlistBindAddr string, memberlistPort int, memberlistEnv string, advertiseAddr string, peers []string) (string, error) {
// Read encryption key from secrets if available
encryptionKey := ""
if data, err := os.ReadFile(filepath.Join(cg.oramaDir, "secrets", "olric-encryption-key")); err == nil {
encryptionKey = strings.TrimSpace(string(data))
}
data := templates.OlricConfigData{
ServerBindAddr: serverBindAddr,
HTTPPort: httpPort,
@ -225,6 +257,7 @@ func (cg *ConfigGenerator) GenerateOlricConfig(serverBindAddr string, httpPort i
MemberlistEnvironment: memberlistEnv,
MemberlistAdvertiseAddr: advertiseAddr,
Peers: peers,
EncryptionKey: encryptionKey,
}
return templates.RenderOlricConfig(data)
}
@ -299,6 +332,137 @@ func (sg *SecretGenerator) EnsureClusterSecret() (string, error) {
return secret, nil
}
// EnsureRQLiteAuth generates the RQLite auth credentials and JSON auth file.
// Returns (username, password). The auth JSON file is written to secrets/rqlite-auth.json.
func (sg *SecretGenerator) EnsureRQLiteAuth() (string, string, error) {
passwordPath := filepath.Join(sg.oramaDir, "secrets", "rqlite-password")
authFilePath := filepath.Join(sg.oramaDir, "secrets", "rqlite-auth.json")
secretDir := filepath.Dir(passwordPath)
username := "orama"
if err := os.MkdirAll(secretDir, 0700); err != nil {
return "", "", fmt.Errorf("failed to create secrets directory: %w", err)
}
if err := os.Chmod(secretDir, 0700); err != nil {
return "", "", fmt.Errorf("failed to set secrets directory permissions: %w", err)
}
// Try to read existing password
var password string
if data, err := os.ReadFile(passwordPath); err == nil {
password = strings.TrimSpace(string(data))
}
// Generate new password if needed
if password == "" {
bytes := make([]byte, 32)
if _, err := rand.Read(bytes); err != nil {
return "", "", fmt.Errorf("failed to generate RQLite password: %w", err)
}
password = hex.EncodeToString(bytes)
if err := os.WriteFile(passwordPath, []byte(password), 0600); err != nil {
return "", "", fmt.Errorf("failed to save RQLite password: %w", err)
}
if err := ensureSecretFilePermissions(passwordPath); err != nil {
return "", "", err
}
}
// Always regenerate the auth JSON file to ensure consistency
authJSON := fmt.Sprintf(`[{"username": "%s", "password": "%s", "perms": ["all"]}]`, username, password)
if err := os.WriteFile(authFilePath, []byte(authJSON), 0600); err != nil {
return "", "", fmt.Errorf("failed to save RQLite auth file: %w", err)
}
if err := ensureSecretFilePermissions(authFilePath); err != nil {
return "", "", err
}
return username, password, nil
}
// EnsureOlricEncryptionKey gets or generates a 32-byte encryption key for Olric memberlist gossip.
// The key is stored as base64 on disk and returned as base64 (what Olric expects).
func (sg *SecretGenerator) EnsureOlricEncryptionKey() (string, error) {
secretPath := filepath.Join(sg.oramaDir, "secrets", "olric-encryption-key")
secretDir := filepath.Dir(secretPath)
if err := os.MkdirAll(secretDir, 0700); err != nil {
return "", fmt.Errorf("failed to create secrets directory: %w", err)
}
if err := os.Chmod(secretDir, 0700); err != nil {
return "", fmt.Errorf("failed to set secrets directory permissions: %w", err)
}
// Try to read existing key
if data, err := os.ReadFile(secretPath); err == nil {
key := strings.TrimSpace(string(data))
if key != "" {
if err := ensureSecretFilePermissions(secretPath); err != nil {
return "", err
}
return key, nil
}
}
// Generate new 32-byte key, base64 encoded
keyBytes := make([]byte, 32)
if _, err := rand.Read(keyBytes); err != nil {
return "", fmt.Errorf("failed to generate Olric encryption key: %w", err)
}
key := base64.StdEncoding.EncodeToString(keyBytes)
if err := os.WriteFile(secretPath, []byte(key), 0600); err != nil {
return "", fmt.Errorf("failed to save Olric encryption key: %w", err)
}
if err := ensureSecretFilePermissions(secretPath); err != nil {
return "", err
}
return key, nil
}
// EnsureAPIKeyHMACSecret gets or generates the HMAC secret used to hash API keys.
// The secret is a 32-byte random value stored as 64 hex characters.
func (sg *SecretGenerator) EnsureAPIKeyHMACSecret() (string, error) {
secretPath := filepath.Join(sg.oramaDir, "secrets", "api-key-hmac-secret")
secretDir := filepath.Dir(secretPath)
if err := os.MkdirAll(secretDir, 0700); err != nil {
return "", fmt.Errorf("failed to create secrets directory: %w", err)
}
if err := os.Chmod(secretDir, 0700); err != nil {
return "", fmt.Errorf("failed to set secrets directory permissions: %w", err)
}
// Try to read existing secret
if data, err := os.ReadFile(secretPath); err == nil {
secret := strings.TrimSpace(string(data))
if len(secret) == 64 {
if err := ensureSecretFilePermissions(secretPath); err != nil {
return "", err
}
return secret, nil
}
}
// Generate new secret (32 bytes = 64 hex chars)
bytes := make([]byte, 32)
if _, err := rand.Read(bytes); err != nil {
return "", fmt.Errorf("failed to generate API key HMAC secret: %w", err)
}
secret := hex.EncodeToString(bytes)
if err := os.WriteFile(secretPath, []byte(secret), 0600); err != nil {
return "", fmt.Errorf("failed to save API key HMAC secret: %w", err)
}
if err := ensureSecretFilePermissions(secretPath); err != nil {
return "", err
}
return secret, nil
}
func ensureSecretFilePermissions(secretPath string) error {
if err := os.Chmod(secretPath, 0600); err != nil {
return fmt.Errorf("failed to set permissions on %s: %w", secretPath, err)

25
pkg/environments/production/firewall.go
View File

@ -98,7 +98,12 @@ func (fp *FirewallProvisioner) GenerateRules() []string {
}
// Allow all traffic from WireGuard subnet (inter-node encrypted traffic)
rules = append(rules, "ufw allow from 10.0.0.0/8")
rules = append(rules, "ufw allow from 10.0.0.0/24")
// Disable IPv6 — no ip6tables rules exist, so services bound to 0.0.0.0
// may be reachable via IPv6. Disable it entirely at the kernel level.
rules = append(rules, "sysctl -w net.ipv6.conf.all.disable_ipv6=1")
rules = append(rules, "sysctl -w net.ipv6.conf.default.disable_ipv6=1")
// Enable firewall
rules = append(rules, "ufw --force enable")
@ -109,7 +114,7 @@ func (fp *FirewallProvisioner) GenerateRules() []string {
// can be misclassified as "invalid" by conntrack due to reordering/jitter
// (especially between high-latency peers), causing silent packet drops.
// Inserting at position 1 in INPUT ensures this runs before UFW chains.
rules = append(rules, "iptables -I INPUT 1 -i wg0 -s 10.0.0.0/8 -j ACCEPT")
rules = append(rules, "iptables -I INPUT 1 -i wg0 -s 10.0.0.0/24 -j ACCEPT")
return rules
}
@ -130,6 +135,22 @@ func (fp *FirewallProvisioner) Setup() error {
}
}
// Persist IPv6 disable across reboots
if err := fp.persistIPv6Disable(); err != nil {
return fmt.Errorf("failed to persist IPv6 disable: %w", err)
}
return nil
}
// persistIPv6Disable writes a sysctl config to disable IPv6 on boot.
func (fp *FirewallProvisioner) persistIPv6Disable() error {
content := "# Orama Network: disable IPv6 (no ip6tables rules configured)\nnet.ipv6.conf.all.disable_ipv6 = 1\nnet.ipv6.conf.default.disable_ipv6 = 1\n"
cmd := exec.Command("tee", "/etc/sysctl.d/99-orama-disable-ipv6.conf")
cmd.Stdin = strings.NewReader(content)
if output, err := cmd.CombinedOutput(); err != nil {
return fmt.Errorf("failed to write sysctl config: %w\n%s", err, string(output))
}
return nil
}

8
pkg/environments/production/firewall_test.go
View File

@ -18,9 +18,11 @@ func TestFirewallProvisioner_GenerateRules_StandardNode(t *testing.T) {
assertContainsRule(t, rules, "ufw allow 51820/udp")
assertContainsRule(t, rules, "ufw allow 80/tcp")
assertContainsRule(t, rules, "ufw allow 443/tcp")
assertContainsRule(t, rules, "ufw allow from 10.0.0.0/8")
assertContainsRule(t, rules, "ufw allow from 10.0.0.0/24")
assertContainsRule(t, rules, "sysctl -w net.ipv6.conf.all.disable_ipv6=1")
assertContainsRule(t, rules, "sysctl -w net.ipv6.conf.default.disable_ipv6=1")
assertContainsRule(t, rules, "ufw --force enable")
assertContainsRule(t, rules, "iptables -I INPUT 1 -i wg0 -s 10.0.0.0/8 -j ACCEPT")
assertContainsRule(t, rules, "iptables -I INPUT 1 -i wg0 -s 10.0.0.0/24 -j ACCEPT")
// Should NOT contain DNS or Anyone relay
for _, rule := range rules {
@ -76,7 +78,7 @@ func TestFirewallProvisioner_GenerateRules_WireGuardSubnetAllowed(t *testing.T)
rules := fp.GenerateRules()
assertContainsRule(t, rules, "ufw allow from 10.0.0.0/8")
assertContainsRule(t, rules, "ufw allow from 10.0.0.0/24")
}
func TestFirewallProvisioner_GenerateRules_FullConfig(t *testing.T) {

9
pkg/environments/production/installers/caddy.go
View File

@ -7,11 +7,12 @@ import (
"os/exec"
"path/filepath"
"strings"
"github.com/DeBrosOfficial/network/pkg/constants"
)
const (
caddyVersion = "2.10.2"
xcaddyRepo = "github.com/caddyserver/xcaddy/cmd/xcaddy@latest"
xcaddyRepo = "github.com/caddyserver/xcaddy/cmd/xcaddy@latest"
)
// CaddyInstaller handles Caddy installation with custom DNS module
@ -26,7 +27,7 @@ type CaddyInstaller struct {
func NewCaddyInstaller(arch string, logWriter io.Writer, oramaHome string) *CaddyInstaller {
return &CaddyInstaller{
BaseInstaller: NewBaseInstaller(arch, logWriter),
version: caddyVersion,
version: constants.CaddyVersion,
oramaHome: oramaHome,
dnsModule: filepath.Join(oramaHome, "src", "pkg", "caddy", "dns", "orama"),
}
@ -356,7 +357,7 @@ func (ci *CaddyInstaller) generateGoMod() string {
go 1.22
require (
github.com/caddyserver/caddy/v2 v2.` + caddyVersion[2:] + `
github.com/caddyserver/caddy/v2 v2.` + constants.CaddyVersion[2:] + `
github.com/libdns/libdns v1.1.0
)
`

24
pkg/environments/production/installers/coredns.go
View File

@ -9,12 +9,14 @@ import (
"os"
"os/exec"
"path/filepath"
"strings"
"time"
"github.com/DeBrosOfficial/network/pkg/constants"
)
const (
coreDNSVersion = "1.12.0"
coreDNSRepo = "https://github.com/coredns/coredns.git"
coreDNSRepo = "https://github.com/coredns/coredns.git"
)
// CoreDNSInstaller handles CoreDNS installation with RQLite plugin
@ -29,7 +31,7 @@ type CoreDNSInstaller struct {
func NewCoreDNSInstaller(arch string, logWriter io.Writer, oramaHome string) *CoreDNSInstaller {
return &CoreDNSInstaller{
BaseInstaller: NewBaseInstaller(arch, logWriter),
version: coreDNSVersion,
version: constants.CoreDNSVersion,
oramaHome: oramaHome,
rqlitePlugin: filepath.Join(oramaHome, "src", "pkg", "coredns", "rqlite"),
}
@ -322,8 +324,18 @@ rqlite:rqlite
`
}
// generateCorefile creates the CoreDNS configuration (RQLite only)
// generateCorefile creates the CoreDNS configuration (RQLite only).
// If RQLite credentials exist on disk, they are included in the config.
func (ci *CoreDNSInstaller) generateCorefile(domain, rqliteDSN string) string {
// Read RQLite credentials from secrets if available
authBlock := ""
if data, err := os.ReadFile("/opt/orama/.orama/secrets/rqlite-password"); err == nil {
password := strings.TrimSpace(string(data))
if password != "" {
authBlock = fmt.Sprintf(" username orama\n password %s\n", password)
}
}
return fmt.Sprintf(`# CoreDNS configuration for %s
# Uses RQLite for ALL DNS records (static + dynamic)
# Static records (SOA, NS, A) are seeded into RQLite during installation
@ -335,7 +347,7 @@ func (ci *CoreDNSInstaller) generateCorefile(domain, rqliteDSN string) string {
refresh 5s
ttl 30
cache_size 10000
}
%s }
# Enable logging and error reporting
log
@ -350,7 +362,7 @@ func (ci *CoreDNSInstaller) generateCorefile(domain, rqliteDSN string) string {
cache 300
errors
}
`, domain, domain, rqliteDSN)
`, domain, domain, rqliteDSN, authBlock)
}
// seedStaticRecords inserts static zone records into RQLite (non-destructive)

4
pkg/environments/production/installers/gateway.go
View File

@ -7,6 +7,8 @@ import (
"os/exec"
"path/filepath"
"strings"
"github.com/DeBrosOfficial/network/pkg/constants"
)
// GatewayInstaller handles Orama binary installation (including gateway)
@ -124,7 +126,7 @@ func (gi *GatewayInstaller) InstallDeBrosBinaries(oramaHome string) error {
// InstallGo downloads and installs Go toolchain
func (gi *GatewayInstaller) InstallGo() error {
requiredVersion := "1.24.6"
requiredVersion := constants.GoVersion
if goPath, err := exec.LookPath("go"); err == nil {
// Check version - upgrade if too old
out, _ := exec.Command(goPath, "version").Output()

4
pkg/environments/production/installers/ipfs.go
View File

@ -7,6 +7,8 @@ import (
"os"
"os/exec"
"path/filepath"
"github.com/DeBrosOfficial/network/pkg/constants"
)
// IPFSInstaller handles IPFS (Kubo) installation
@ -19,7 +21,7 @@ type IPFSInstaller struct {
func NewIPFSInstaller(arch string, logWriter io.Writer) *IPFSInstaller {
return &IPFSInstaller{
BaseInstaller: NewBaseInstaller(arch, logWriter),
version: "v0.38.2",
version: constants.IPFSKuboVersion,
}
}

4
pkg/environments/production/installers/ipfs_cluster.go
View File

@ -8,6 +8,8 @@ import (
"os/exec"
"path/filepath"
"strings"
"github.com/DeBrosOfficial/network/pkg/constants"
)
// IPFSClusterInstaller handles IPFS Cluster Service installation
@ -42,7 +44,7 @@ func (ici *IPFSClusterInstaller) Install() error {
return fmt.Errorf("go not found - required to install IPFS Cluster. Please install Go first")
}
cmd := exec.Command("go", "install", "github.com/ipfs-cluster/ipfs-cluster/cmd/ipfs-cluster-service@latest")
cmd := exec.Command("go", "install", fmt.Sprintf("github.com/ipfs-cluster/ipfs-cluster/cmd/ipfs-cluster-service@%s", constants.IPFSClusterVersion))
cmd.Env = append(os.Environ(), "GOBIN=/usr/local/bin", "GOPROXY=https://proxy.golang.org|direct", "GONOSUMDB=*")
if err := cmd.Run(); err != nil {
return fmt.Errorf("failed to install IPFS Cluster: %w", err)

4
pkg/environments/production/installers/olric.go
View File

@ -5,6 +5,8 @@ import (
"io"
"os"
"os/exec"
"github.com/DeBrosOfficial/network/pkg/constants"
)
// OlricInstaller handles Olric server installation
@ -17,7 +19,7 @@ type OlricInstaller struct {
func NewOlricInstaller(arch string, logWriter io.Writer) *OlricInstaller {
return &OlricInstaller{
BaseInstaller: NewBaseInstaller(arch, logWriter),
version: "v0.7.0",
version: constants.OlricVersion,
}
}

4
pkg/environments/production/installers/rqlite.go
View File

@ -5,6 +5,8 @@ import (
"io"
"os"
"os/exec"
"github.com/DeBrosOfficial/network/pkg/constants"
)
// RQLiteInstaller handles RQLite installation
@ -17,7 +19,7 @@ type RQLiteInstaller struct {
func NewRQLiteInstaller(arch string, logWriter io.Writer) *RQLiteInstaller {
return &RQLiteInstaller{
BaseInstaller: NewBaseInstaller(arch, logWriter),
version: "8.43.0",
version: constants.RQLiteVersion,
}
}

178
pkg/environments/production/orchestrator.go
View File

@ -1,6 +1,7 @@
package production
import (
"encoding/json"
"fmt"
"io"
"os"
@ -256,13 +257,57 @@ func (ps *ProductionSetup) Phase2ProvisionEnvironment() error {
}
ps.logf(" ✓ Directory structure created")
// Create dedicated orama user for running services (non-root)
if err := ps.fsProvisioner.EnsureOramaUser(); err != nil {
ps.logf(" ⚠️ Could not create orama user: %v (services will run as root)", err)
} else {
ps.logf(" ✓ orama user ensured")
}
return nil
}
// Phase2bInstallBinaries installs external binaries and Orama components
// Phase2bInstallBinaries installs external binaries and Orama components.
// Auto-detects pre-built mode if /opt/orama/manifest.json exists.
func (ps *ProductionSetup) Phase2bInstallBinaries() error {
ps.logf("Phase 2b: Installing binaries...")
// Auto-detect pre-built binary archive
if HasPreBuiltArchive() {
manifest, err := LoadPreBuiltManifest()
if err != nil {
ps.logf(" ⚠️ Pre-built manifest found but unreadable: %v", err)
ps.logf(" Falling back to source mode...")
if err := ps.installFromSource(); err != nil {
return err
}
} else {
if err := ps.installFromPreBuilt(manifest); err != nil {
return err
}
}
} else {
// Source mode: compile everything on the VPS (original behavior)
if err := ps.installFromSource(); err != nil {
return err
}
}
// Anyone relay/client configuration runs after BOTH paths.
// Pre-built mode installs the anon binary via .deb/apt;
// source mode installs it via the relay installer's Install().
// Configuration (anonrc, bandwidth, migration) is always needed.
if err := ps.configureAnyone(); err != nil {
ps.logf(" ⚠️ Anyone configuration warning: %v", err)
}
ps.logf(" ✓ All binaries installed")
return nil
}
// installFromSource installs binaries by compiling from source on the VPS.
// This is the original Phase2bInstallBinaries logic, preserved as fallback.
func (ps *ProductionSetup) installFromSource() error {
// Install system dependencies (always needed for runtime libs)
if err := ps.binaryInstaller.InstallSystemDependencies(); err != nil {
ps.logf(" ⚠️ System dependencies warning: %v", err)
@ -307,7 +352,12 @@ func (ps *ProductionSetup) Phase2bInstallBinaries() error {
ps.logf(" ⚠️ IPFS Cluster install warning: %v", err)
}
// Install Anyone (client or relay based on configuration) — apt-based, not Go
return nil
}
// configureAnyone handles Anyone relay/client installation and configuration.
// This runs after both pre-built and source mode binary installation.
func (ps *ProductionSetup) configureAnyone() error {
if ps.IsAnyoneRelay() {
ps.logf(" Installing Anyone relay (operator mode)...")
relayConfig := installers.AnyoneRelayConfig{
@ -351,7 +401,7 @@ func (ps *ProductionSetup) Phase2bInstallBinaries() error {
}
}
// Install the relay
// Install the relay (apt-based, not Go — idempotent if already installed via .deb)
if err := relayInstaller.Install(); err != nil {
ps.logf(" ⚠️ Anyone relay install warning: %v", err)
}
@ -364,7 +414,7 @@ func (ps *ProductionSetup) Phase2bInstallBinaries() error {
ps.logf(" Installing Anyone client-only mode (SOCKS5 proxy)...")
clientInstaller := installers.NewAnyoneRelayInstaller(ps.arch, ps.logWriter, installers.AnyoneRelayConfig{})
// Install the anon binary (same apt package as relay)
// Install the anon binary (same apt package as relay — idempotent)
if err := clientInstaller.Install(); err != nil {
ps.logf(" ⚠️ Anyone client install warning: %v", err)
}
@ -375,7 +425,6 @@ func (ps *ProductionSetup) Phase2bInstallBinaries() error {
}
}
ps.logf(" ✓ All binaries installed")
return nil
}
@ -436,6 +485,11 @@ func (ps *ProductionSetup) Phase2cInitializeServices(peerAddresses []string, vps
return fmt.Errorf("failed to initialize IPFS Cluster: %w", err)
}
// After init, save own IPFS Cluster peer ID to trusted peers file
if err := ps.saveOwnClusterPeerID(clusterPath); err != nil {
ps.logf(" ⚠️ Could not save IPFS Cluster peer ID to trusted peers: %v", err)
}
// Initialize RQLite data directory
rqliteDataDir := filepath.Join(dataDir, "rqlite")
if err := ps.binaryInstaller.InitializeRQLiteDataDir(rqliteDataDir); err != nil {
@ -446,6 +500,50 @@ func (ps *ProductionSetup) Phase2cInitializeServices(peerAddresses []string, vps
return nil
}
// saveOwnClusterPeerID reads this node's IPFS Cluster peer ID from identity.json
// and appends it to the trusted-peers file so EnsureConfig() can use it.
func (ps *ProductionSetup) saveOwnClusterPeerID(clusterPath string) error {
identityPath := filepath.Join(clusterPath, "identity.json")
data, err := os.ReadFile(identityPath)
if err != nil {
return fmt.Errorf("failed to read identity.json: %w", err)
}
var identity struct {
ID string `json:"id"`
}
if err := json.Unmarshal(data, &identity); err != nil {
return fmt.Errorf("failed to parse identity.json: %w", err)
}
if identity.ID == "" {
return fmt.Errorf("peer ID not found in identity.json")
}
// Read existing trusted peers
trustedPeersPath := filepath.Join(ps.oramaDir, "secrets", "ipfs-cluster-trusted-peers")
var existing []string
if fileData, err := os.ReadFile(trustedPeersPath); err == nil {
for _, line := range strings.Split(strings.TrimSpace(string(fileData)), "\n") {
line = strings.TrimSpace(line)
if line != "" {
if line == identity.ID {
return nil // already present
}
existing = append(existing, line)
}
}
}
existing = append(existing, identity.ID)
content := strings.Join(existing, "\n") + "\n"
if err := os.WriteFile(trustedPeersPath, []byte(content), 0600); err != nil {
return fmt.Errorf("failed to write trusted peers file: %w", err)
}
ps.logf(" ✓ IPFS Cluster peer ID saved to trusted peers: %s", identity.ID)
return nil
}
// Phase3GenerateSecrets generates shared secrets and keys
func (ps *ProductionSetup) Phase3GenerateSecrets() error {
ps.logf("Phase 3: Generating secrets...")
@ -462,6 +560,24 @@ func (ps *ProductionSetup) Phase3GenerateSecrets() error {
}
ps.logf(" ✓ IPFS swarm key ensured")
// RQLite auth credentials
if _, _, err := ps.secretGenerator.EnsureRQLiteAuth(); err != nil {
return fmt.Errorf("failed to ensure RQLite auth: %w", err)
}
ps.logf(" ✓ RQLite auth credentials ensured")
// Olric gossip encryption key
if _, err := ps.secretGenerator.EnsureOlricEncryptionKey(); err != nil {
return fmt.Errorf("failed to ensure Olric encryption key: %w", err)
}
ps.logf(" ✓ Olric encryption key ensured")
// API key HMAC secret
if _, err := ps.secretGenerator.EnsureAPIKeyHMACSecret(); err != nil {
return fmt.Errorf("failed to ensure API key HMAC secret: %w", err)
}
ps.logf(" ✓ API key HMAC secret ensured")
// Node identity (unified architecture)
peerID, err := ps.secretGenerator.EnsureNodeIdentity()
if err != nil {
@ -532,6 +648,14 @@ func (ps *ProductionSetup) Phase4GenerateConfigs(peerAddresses []string, vpsIP s
}
ps.logf(" ✓ Olric config generated")
// Vault Guardian config
vaultConfig := ps.configGenerator.GenerateVaultConfig(vpsIP)
vaultConfigPath := filepath.Join(ps.oramaDir, "data", "vault", "vault.yaml")
if err := os.WriteFile(vaultConfigPath, []byte(vaultConfig), 0644); err != nil {
return fmt.Errorf("failed to save vault config: %w", err)
}
ps.logf(" ✓ Vault config generated")
// Configure CoreDNS (if baseDomain is provided - this is the zone name)
// CoreDNS uses baseDomain (e.g., "dbrs.space") as the authoritative zone
dnsZone := baseDomain
@ -582,6 +706,20 @@ func (ps *ProductionSetup) Phase4GenerateConfigs(peerAddresses []string, vpsIP s
func (ps *ProductionSetup) Phase5CreateSystemdServices(enableHTTPS bool) error {
ps.logf("Phase 5: Creating systemd services...")
// Re-chown all orama directories to the orama user.
// Phases 2b-4 create files as root (IPFS repo, configs, secrets, etc.)
// that must be readable/writable by the orama service user.
if err := exec.Command("id", "orama").Run(); err == nil {
for _, dir := range []string{ps.oramaDir, filepath.Join(ps.oramaHome, "bin")} {
if _, statErr := os.Stat(dir); statErr == nil {
if output, chownErr := exec.Command("chown", "-R", "orama:orama", dir).CombinedOutput(); chownErr != nil {
ps.logf(" ⚠️ Failed to chown %s: %v\n%s", dir, chownErr, string(output))
}
}
}
ps.logf(" ✓ File ownership updated for orama user")
}
// Validate all required binaries are available before creating services
ipfsBinary, err := ps.binaryInstaller.ResolveBinaryPath("ipfs", "/usr/local/bin/ipfs", "/usr/bin/ipfs")
if err != nil {
@ -626,6 +764,13 @@ func (ps *ProductionSetup) Phase5CreateSystemdServices(enableHTTPS bool) error {
}
ps.logf(" ✓ Node service created: orama-node.service (with embedded gateway)")
// Vault Guardian service
vaultUnit := ps.serviceGenerator.GenerateVaultService()
if err := ps.serviceController.WriteServiceUnit("orama-vault.service", vaultUnit); err != nil {
return fmt.Errorf("failed to write Vault service: %w", err)
}
ps.logf(" ✓ Vault service created: orama-vault.service")
// Anyone Relay service (only created when --anyone-relay flag is used)
// A node must run EITHER relay OR client, never both. When writing one
// mode's service, we remove the other to prevent conflicts (they share
@ -664,8 +809,9 @@ func (ps *ProductionSetup) Phase5CreateSystemdServices(enableHTTPS bool) error {
// Caddy service on ALL nodes (any node may host namespaces and need TLS)
if _, err := os.Stat("/usr/bin/caddy"); err == nil {
// Create caddy data directory
// Create caddy data directory and ensure orama user can write to it
exec.Command("mkdir", "-p", "/var/lib/caddy").Run()
exec.Command("chown", "-R", "orama:orama", "/var/lib/caddy").Run()
caddyUnit := ps.serviceGenerator.GenerateCaddyService()
if err := ps.serviceController.WriteServiceUnit("caddy.service", caddyUnit); err != nil {
@ -684,7 +830,7 @@ func (ps *ProductionSetup) Phase5CreateSystemdServices(enableHTTPS bool) error {
// Enable services (unified names - no bootstrap/node distinction)
// Note: orama-gateway.service is no longer needed - each node has an embedded gateway
// Note: orama-rqlite.service is NOT created - RQLite is managed by each node internally
services := []string{"orama-ipfs.service", "orama-ipfs-cluster.service", "orama-olric.service", "orama-node.service"}
services := []string{"orama-ipfs.service", "orama-ipfs-cluster.service", "orama-olric.service", "orama-vault.service", "orama-node.service"}
// Add Anyone service if configured (relay or client)
if ps.IsAnyoneRelay() {
@ -715,8 +861,8 @@ func (ps *ProductionSetup) Phase5CreateSystemdServices(enableHTTPS bool) error {
// services pick up new configs even if already running from a previous install)
ps.logf(" Starting services...")
// Start infrastructure first (IPFS, Olric, Anyone) - RQLite is managed internally by each node
infraServices := []string{"orama-ipfs.service", "orama-olric.service"}
// Start infrastructure first (IPFS, Olric, Vault, Anyone) - RQLite is managed internally by each node
infraServices := []string{"orama-ipfs.service", "orama-olric.service", "orama-vault.service"}
// Add Anyone service if configured (relay or client)
if ps.IsAnyoneRelay() {
@ -851,6 +997,13 @@ func (ps *ProductionSetup) Phase6SetupWireGuard(isFirstNode bool) (privateKey, p
}
ps.logf(" ✓ WireGuard keypair generated")
// Save public key to orama secrets so the gateway (running as orama user)
// can read it without needing root access to /etc/wireguard/wg0.conf
pubKeyPath := filepath.Join(ps.oramaDir, "secrets", "wg-public-key")
if err := os.WriteFile(pubKeyPath, []byte(pubKey), 0600); err != nil {
return "", "", fmt.Errorf("failed to save WG public key: %w", err)
}
if isFirstNode {
// First node: self-assign 10.0.0.1, no peers yet
wp.config = WireGuardConfig{
@ -936,12 +1089,13 @@ func (ps *ProductionSetup) LogSetupComplete(peerID string) {
ps.logf(" %s/logs/olric.log", ps.oramaDir)
ps.logf(" %s/logs/node.log", ps.oramaDir)
ps.logf(" %s/logs/gateway.log", ps.oramaDir)
ps.logf(" %s/logs/vault.log", ps.oramaDir)
// Anyone mode-specific logs and commands
if ps.IsAnyoneRelay() {
ps.logf(" /var/log/anon/notices.log (Anyone Relay)")
ps.logf("\nStart All Services:")
ps.logf(" systemctl start orama-ipfs orama-ipfs-cluster orama-olric orama-anyone-relay orama-node")
ps.logf(" systemctl start orama-ipfs orama-ipfs-cluster orama-olric orama-vault orama-anyone-relay orama-node")
ps.logf("\nAnyone Relay Operator:")
ps.logf(" ORPort: %d", ps.anyoneRelayConfig.ORPort)
ps.logf(" Wallet: %s", ps.anyoneRelayConfig.Wallet)
@ -950,10 +1104,10 @@ func (ps *ProductionSetup) LogSetupComplete(peerID string) {
ps.logf(" IMPORTANT: You need 100 $ANYONE tokens in your wallet to receive rewards")
} else if ps.IsAnyoneClient() {
ps.logf("\nStart All Services:")
ps.logf(" systemctl start orama-ipfs orama-ipfs-cluster orama-olric orama-anyone-client orama-node")
ps.logf(" systemctl start orama-ipfs orama-ipfs-cluster orama-olric orama-vault orama-anyone-client orama-node")
} else {
ps.logf("\nStart All Services:")
ps.logf(" systemctl start orama-ipfs orama-ipfs-cluster orama-olric orama-node")
ps.logf(" systemctl start orama-ipfs orama-ipfs-cluster orama-olric orama-vault orama-node")
}
ps.logf("\nVerify Installation:")

7
pkg/environments/production/paths.go
View File

@ -11,4 +11,11 @@ const (
OramaSecrets = "/opt/orama/.orama/secrets"
OramaData = "/opt/orama/.orama/data"
OramaLogs = "/opt/orama/.orama/logs"
// Pre-built binary archive paths (created by `orama build`)
OramaManifest = "/opt/orama/manifest.json"
OramaManifestSig = "/opt/orama/manifest.sig"
OramaArchiveBin = "/opt/orama/bin" // Pre-built binaries
OramaSystemdDir = "/opt/orama/systemd" // Namespace service templates
OramaPackagesDir = "/opt/orama/packages" // .deb packages (e.g., anon.deb)
)

325
pkg/environments/production/prebuilt.go Normal file
View File

@ -0,0 +1,325 @@
package production
import (
"crypto/sha256"
"encoding/hex"
"encoding/json"
"fmt"
"io"
"os"
"os/exec"
"path/filepath"
"strings"
ethcrypto "github.com/ethereum/go-ethereum/crypto"
)
// PreBuiltManifest describes the contents of a pre-built binary archive.
type PreBuiltManifest struct {
Version string `json:"version"`
Commit string `json:"commit"`
Date string `json:"date"`
Arch string `json:"arch"`
Checksums map[string]string `json:"checksums"` // filename -> sha256
}
// HasPreBuiltArchive checks if a pre-built binary archive has been extracted
// at /opt/orama/ by looking for the manifest.json file.
func HasPreBuiltArchive() bool {
_, err := os.Stat(OramaManifest)
return err == nil
}
// LoadPreBuiltManifest loads and parses the pre-built manifest.
func LoadPreBuiltManifest() (*PreBuiltManifest, error) {
data, err := os.ReadFile(OramaManifest)
if err != nil {
return nil, fmt.Errorf("failed to read manifest: %w", err)
}
var manifest PreBuiltManifest
if err := json.Unmarshal(data, &manifest); err != nil {
return nil, fmt.Errorf("failed to parse manifest: %w", err)
}
return &manifest, nil
}
// OramaSignerAddress is the Ethereum address authorized to sign build archives.
// Archives signed by any other address are rejected during install.
// This is the DeBros deploy wallet — update if the signing key rotates.
const OramaSignerAddress = "0xb5d8a496c8b2412990d7D467E17727fdF5954afC"
// VerifyArchiveSignature verifies that the pre-built archive was signed by the
// authorized Orama signer. Returns nil if the signature is valid, or if no
// signature file exists (unsigned archives are allowed but logged as a warning).
func VerifyArchiveSignature(manifest *PreBuiltManifest) error {
sigData, err := os.ReadFile(OramaManifestSig)
if os.IsNotExist(err) {
return nil // unsigned archive — caller decides whether to proceed
}
if err != nil {
return fmt.Errorf("failed to read manifest.sig: %w", err)
}
// Reproduce the same hash used during signing: SHA256 of compact JSON
manifestJSON, err := json.Marshal(manifest)
if err != nil {
return fmt.Errorf("failed to marshal manifest: %w", err)
}
manifestHash := sha256.Sum256(manifestJSON)
hashHex := hex.EncodeToString(manifestHash[:])
// EVM personal_sign: keccak256("\x19Ethereum Signed Message:\n" + len + message)
msg := []byte(hashHex)
prefix := []byte("\x19Ethereum Signed Message:\n" + fmt.Sprintf("%d", len(msg)))
ethHash := ethcrypto.Keccak256(prefix, msg)
// Decode signature
sigHex := strings.TrimSpace(string(sigData))
if strings.HasPrefix(sigHex, "0x") || strings.HasPrefix(sigHex, "0X") {
sigHex = sigHex[2:]
}
sig, err := hex.DecodeString(sigHex)
if err != nil || len(sig) != 65 {
return fmt.Errorf("invalid signature format in manifest.sig")
}
// Normalize recovery ID
if sig[64] >= 27 {
sig[64] -= 27
}
// Recover public key from signature
pub, err := ethcrypto.SigToPub(ethHash, sig)
if err != nil {
return fmt.Errorf("signature recovery failed: %w", err)
}
recovered := ethcrypto.PubkeyToAddress(*pub).Hex()
expected := strings.ToLower(OramaSignerAddress)
got := strings.ToLower(recovered)
if got != expected {
return fmt.Errorf("archive signed by %s, expected %s — refusing to install", recovered, OramaSignerAddress)
}
return nil
}
// IsArchiveSigned returns true if a manifest.sig file exists alongside the manifest.
func IsArchiveSigned() bool {
_, err := os.Stat(OramaManifestSig)
return err == nil
}
// installFromPreBuilt installs all binaries from a pre-built archive.
// The archive must already be extracted at /opt/orama/ with:
// - /opt/orama/bin/ — all pre-compiled binaries
// - /opt/orama/systemd/ — namespace service templates
// - /opt/orama/packages/ — optional .deb packages
// - /opt/orama/manifest.json — archive metadata
func (ps *ProductionSetup) installFromPreBuilt(manifest *PreBuiltManifest) error {
ps.logf(" Using pre-built binary archive v%s (%s) linux/%s", manifest.Version, manifest.Commit, manifest.Arch)
// Verify archive signature if present
if IsArchiveSigned() {
if err := VerifyArchiveSignature(manifest); err != nil {
return fmt.Errorf("archive signature verification failed: %w", err)
}
ps.logf(" ✓ Archive signature verified")
} else {
ps.logf(" ⚠️ Archive is unsigned — consider using 'orama build --sign'")
}
// Install minimal system dependencies (no build tools needed)
if err := ps.installMinimalSystemDeps(); err != nil {
ps.logf(" ⚠️ System dependencies warning: %v", err)
}
// Copy binaries to runtime locations
if err := ps.deployPreBuiltBinaries(manifest); err != nil {
return fmt.Errorf("failed to deploy pre-built binaries: %w", err)
}
// Set capabilities on binaries that need to bind privileged ports
if err := ps.setCapabilities(); err != nil {
return fmt.Errorf("failed to set capabilities: %w", err)
}
// Disable systemd-resolved stub listener for nameserver nodes
// (needed even in pre-built mode so CoreDNS can bind port 53)
if ps.isNameserver {
if err := ps.disableResolvedStub(); err != nil {
ps.logf(" ⚠️ Failed to disable systemd-resolved stub: %v", err)
}
}
// Install Anyone relay from .deb package if available
if ps.IsAnyoneRelay() || ps.IsAnyoneClient() {
if err := ps.installAnyonFromPreBuilt(); err != nil {
ps.logf(" ⚠️ Anyone install warning: %v", err)
}
}
ps.logf(" ✓ All pre-built binaries installed")
return nil
}
// installMinimalSystemDeps installs only runtime dependencies (no build tools).
func (ps *ProductionSetup) installMinimalSystemDeps() error {
ps.logf(" Installing minimal system dependencies...")
cmd := exec.Command("apt-get", "update")
if err := cmd.Run(); err != nil {
ps.logf(" Warning: apt update failed")
}
// Only install runtime deps — no build-essential, make, nodejs, npm needed
cmd = exec.Command("apt-get", "install", "-y", "curl", "wget", "unzip")
if err := cmd.Run(); err != nil {
return fmt.Errorf("failed to install minimal dependencies: %w", err)
}
ps.logf(" ✓ Minimal system dependencies installed (no build tools needed)")
return nil
}
// deployPreBuiltBinaries copies pre-built binaries to their runtime locations.
func (ps *ProductionSetup) deployPreBuiltBinaries(manifest *PreBuiltManifest) error {
ps.logf(" Deploying pre-built binaries...")
// Binary → destination mapping
// Most go to /usr/local/bin/, caddy goes to /usr/bin/
type binaryDest struct {
name string
dest string
}
binaries := []binaryDest{
{name: "orama", dest: "/usr/local/bin/orama"},
{name: "orama-node", dest: "/usr/local/bin/orama-node"},
{name: "gateway", dest: "/usr/local/bin/gateway"},
{name: "identity", dest: "/usr/local/bin/identity"},
{name: "sfu", dest: "/usr/local/bin/sfu"},
{name: "turn", dest: "/usr/local/bin/turn"},
{name: "olric-server", dest: "/usr/local/bin/olric-server"},
{name: "ipfs", dest: "/usr/local/bin/ipfs"},
{name: "ipfs-cluster-service", dest: "/usr/local/bin/ipfs-cluster-service"},
{name: "rqlited", dest: "/usr/local/bin/rqlited"},
{name: "coredns", dest: "/usr/local/bin/coredns"},
{name: "caddy", dest: "/usr/bin/caddy"},
}
// Note: vault-guardian stays at /opt/orama/bin/ (from archive extraction)
// and is referenced by absolute path in the systemd service — no copy needed.
for _, bin := range binaries {
srcPath := filepath.Join(OramaArchiveBin, bin.name)
// Skip optional binaries (e.g., coredns on non-nameserver nodes)
if _, ok := manifest.Checksums[bin.name]; !ok {
continue
}
if _, err := os.Stat(srcPath); os.IsNotExist(err) {
ps.logf(" ⚠️ Binary %s not found in archive, skipping", bin.name)
continue
}
if err := copyBinary(srcPath, bin.dest); err != nil {
return fmt.Errorf("failed to copy %s: %w", bin.name, err)
}
ps.logf(" ✓ %s → %s", bin.name, bin.dest)
}
return nil
}
// setCapabilities sets cap_net_bind_service on binaries that need to bind privileged ports.
// Both the /opt/orama/bin/ originals (used by systemd) and /usr/local/bin/ copies need caps.
func (ps *ProductionSetup) setCapabilities() error {
caps := []string{
filepath.Join(OramaArchiveBin, "orama-node"), // systemd uses this path
"/usr/local/bin/orama-node", // PATH copy
"/usr/bin/caddy", // caddy's standard location
}
for _, binary := range caps {
if _, err := os.Stat(binary); os.IsNotExist(err) {
continue
}
cmd := exec.Command("setcap", "cap_net_bind_service=+ep", binary)
if err := cmd.Run(); err != nil {
return fmt.Errorf("setcap failed on %s: %w (node won't be able to bind port 443)", binary, err)
}
ps.logf(" ✓ setcap on %s", binary)
}
return nil
}
// disableResolvedStub disables systemd-resolved's stub listener so CoreDNS can bind port 53.
func (ps *ProductionSetup) disableResolvedStub() error {
// Delegate to the coredns installer's method
return ps.binaryInstaller.coredns.DisableResolvedStubListener()
}
// installAnyonFromPreBuilt installs the Anyone relay .deb from the packages dir,
// falling back to apt install if the .deb is not bundled.
func (ps *ProductionSetup) installAnyonFromPreBuilt() error {
debPath := filepath.Join(OramaPackagesDir, "anon.deb")
if _, err := os.Stat(debPath); err == nil {
ps.logf(" Installing Anyone from bundled .deb...")
cmd := exec.Command("dpkg", "-i", debPath)
if err := cmd.Run(); err != nil {
ps.logf(" ⚠️ dpkg -i failed, falling back to apt...")
cmd = exec.Command("apt-get", "install", "-y", "anon")
if err := cmd.Run(); err != nil {
return fmt.Errorf("failed to install anon: %w", err)
}
}
ps.logf(" ✓ Anyone installed from .deb")
return nil
}
// No .deb bundled — fall back to apt (the existing path in source mode)
ps.logf(" Installing Anyone via apt (not bundled in archive)...")
cmd := exec.Command("apt-get", "install", "-y", "anon")
if err := cmd.Run(); err != nil {
return fmt.Errorf("failed to install anon via apt: %w", err)
}
ps.logf(" ✓ Anyone installed via apt")
return nil
}
// copyBinary copies a file from src to dest, preserving executable permissions.
// It removes the destination first to avoid ETXTBSY ("text file busy") errors
// when overwriting a binary that is currently running.
func copyBinary(src, dest string) error {
// Ensure parent directory exists
if err := os.MkdirAll(filepath.Dir(dest), 0755); err != nil {
return err
}
// Remove the old binary first. On Linux, if the binary is running,
// rm unlinks the filename while the kernel keeps the inode alive for
// the running process. Writing a new file at the same path creates a
// fresh inode — no ETXTBSY conflict.
_ = os.Remove(dest)
srcFile, err := os.Open(src)
if err != nil {
return err
}
defer srcFile.Close()
destFile, err := os.OpenFile(dest, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0755)
if err != nil {
return err
}
defer destFile.Close()
if _, err := io.Copy(destFile, srcFile); err != nil {
return err
}
return nil
}

34
pkg/environments/production/provisioner.go
View File

@ -34,6 +34,7 @@ func (fp *FilesystemProvisioner) EnsureDirectoryStructure() error {
filepath.Join(fp.oramaDir, "data", "ipfs", "repo"),
filepath.Join(fp.oramaDir, "data", "ipfs-cluster"),
filepath.Join(fp.oramaDir, "data", "rqlite"),
filepath.Join(fp.oramaDir, "data", "vault"),
filepath.Join(fp.oramaDir, "logs"),
filepath.Join(fp.oramaDir, "tls-cache"),
filepath.Join(fp.oramaDir, "backups"),
@ -65,6 +66,7 @@ func (fp *FilesystemProvisioner) EnsureDirectoryStructure() error {
"ipfs.log",
"ipfs-cluster.log",
"node.log",
"vault.log",
"anyone-client.log",
}
@ -81,6 +83,38 @@ func (fp *FilesystemProvisioner) EnsureDirectoryStructure() error {
return nil
}
// EnsureOramaUser creates the 'orama' system user and group for running services.
// Sets ownership of the orama data directory to the new user.
func (fp *FilesystemProvisioner) EnsureOramaUser() error {
// Check if user already exists
if err := exec.Command("id", "orama").Run(); err == nil {
return nil // user already exists
}
// Create system user with no login shell and home at /opt/orama
cmd := exec.Command("useradd", "--system", "--no-create-home",
"--home-dir", fp.oramaHome, "--shell", "/usr/sbin/nologin", "orama")
if output, err := cmd.CombinedOutput(); err != nil {
return fmt.Errorf("failed to create orama user: %w\n%s", err, string(output))
}
// Set ownership of orama directories
chown := exec.Command("chown", "-R", "orama:orama", fp.oramaDir)
if output, err := chown.CombinedOutput(); err != nil {
return fmt.Errorf("failed to chown %s: %w\n%s", fp.oramaDir, err, string(output))
}
// Also chown the bin directory
binDir := filepath.Join(fp.oramaHome, "bin")
if _, err := os.Stat(binDir); err == nil {
chown = exec.Command("chown", "-R", "orama:orama", binDir)
if output, err := chown.CombinedOutput(); err != nil {
return fmt.Errorf("failed to chown %s: %w\n%s", binDir, err, string(output))
}
}
return nil
}
// StateDetector checks for existing production state
type StateDetector struct {

98
pkg/environments/production/services.go
View File

@ -8,6 +8,17 @@ import (
"strings"
)
// oramaServiceHardening contains common systemd security directives for orama services.
const oramaServiceHardening = `User=orama
Group=orama
ProtectSystem=strict
ProtectHome=yes
NoNewPrivileges=yes
PrivateDevices=yes
ProtectKernelTunables=yes
ProtectKernelModules=yes
RestrictNamespaces=yes`
// SystemdServiceGenerator generates systemd unit files
type SystemdServiceGenerator struct {
oramaHome string
@ -34,6 +45,8 @@ Wants=network-online.target
[Service]
Type=simple
%[6]s
ReadWritePaths=%[3]s
Environment=HOME=%[1]s
Environment=IPFS_PATH=%[2]s
ExecStartPre=/bin/bash -c 'if [ -f %[3]s/secrets/swarm.key ] && [ ! -f %[2]s/swarm.key ]; then cp %[3]s/secrets/swarm.key %[2]s/swarm.key && chmod 600 %[2]s/swarm.key; fi'
@ -52,7 +65,7 @@ MemoryMax=4G
[Install]
WantedBy=multi-user.target
`, ssg.oramaHome, ipfsRepoPath, ssg.oramaDir, logFile, ipfsBinary)
`, ssg.oramaHome, ipfsRepoPath, ssg.oramaDir, logFile, ipfsBinary, oramaServiceHardening)
}
// GenerateIPFSClusterService generates the IPFS Cluster systemd unit
@ -75,6 +88,8 @@ Requires=orama-ipfs.service
[Service]
Type=simple
%[6]s
ReadWritePaths=%[7]s
WorkingDirectory=%[1]s
Environment=HOME=%[1]s
Environment=IPFS_CLUSTER_PATH=%[2]s
@ -96,7 +111,7 @@ MemoryMax=2G
[Install]
WantedBy=multi-user.target
`, ssg.oramaHome, clusterPath, logFile, clusterBinary, clusterSecret)
`, ssg.oramaHome, clusterPath, logFile, clusterBinary, clusterSecret, oramaServiceHardening, ssg.oramaDir)
}
// GenerateRQLiteService generates the RQLite systemd unit
@ -128,6 +143,8 @@ Wants=network-online.target
[Service]
Type=simple
%[6]s
ReadWritePaths=%[7]s
Environment=HOME=%[1]s
ExecStart=%[5]s %[2]s
Restart=always
@ -143,7 +160,7 @@ KillMode=mixed
[Install]
WantedBy=multi-user.target
`, ssg.oramaHome, args, logFile, dataDir, rqliteBinary)
`, ssg.oramaHome, args, logFile, dataDir, rqliteBinary, oramaServiceHardening, ssg.oramaDir)
}
// GenerateOlricService generates the Olric systemd unit
@ -158,6 +175,8 @@ Wants=network-online.target
[Service]
Type=simple
%[6]s
ReadWritePaths=%[4]s
Environment=HOME=%[1]s
Environment=OLRIC_SERVER_CONFIG=%[2]s
ExecStart=%[5]s
@ -175,7 +194,7 @@ MemoryMax=4G
[Install]
WantedBy=multi-user.target
`, ssg.oramaHome, olricConfigPath, logFile, ssg.oramaDir, olricBinary)
`, ssg.oramaHome, olricConfigPath, logFile, ssg.oramaDir, olricBinary, oramaServiceHardening)
}
// GenerateNodeService generates the Orama Node systemd unit
@ -193,6 +212,9 @@ Requires=wg-quick@wg0.service
[Service]
Type=simple
%[5]s
AmbientCapabilities=CAP_NET_ADMIN
ReadWritePaths=%[2]s
WorkingDirectory=%[1]s
Environment=HOME=%[1]s
ExecStart=%[1]s/bin/orama-node --config %[2]s/configs/%[3]s
@ -211,7 +233,51 @@ OOMScoreAdjust=-500
[Install]
WantedBy=multi-user.target
`, ssg.oramaHome, ssg.oramaDir, configFile, logFile)
`, ssg.oramaHome, ssg.oramaDir, configFile, logFile, oramaServiceHardening)
}
// GenerateVaultService generates the Orama Vault Guardian systemd unit.
// The vault guardian runs on every node, storing Shamir secret shares.
// It binds to the WireGuard overlay only (no public exposure).
func (ssg *SystemdServiceGenerator) GenerateVaultService() string {
logFile := filepath.Join(ssg.oramaDir, "logs", "vault.log")
dataDir := filepath.Join(ssg.oramaDir, "data", "vault")
return fmt.Sprintf(`[Unit]
Description=Orama Vault Guardian
After=network-online.target wg-quick@wg0.service
Wants=network-online.target
Requires=wg-quick@wg0.service
PartOf=orama-node.service
[Service]
Type=simple
User=orama
Group=orama
ProtectSystem=strict
ProtectHome=yes
NoNewPrivileges=yes
PrivateDevices=yes
ProtectKernelTunables=yes
ProtectKernelModules=yes
RestrictNamespaces=yes
ReadWritePaths=%[2]s
ExecStart=%[1]s/bin/vault-guardian --config %[2]s/vault.yaml
Restart=on-failure
RestartSec=5
StandardOutput=append:%[3]s
StandardError=append:%[3]s
SyslogIdentifier=orama-vault
PrivateTmp=yes
LimitMEMLOCK=67108864
MemoryMax=512M
TimeoutStopSec=30
KillMode=mixed
[Install]
WantedBy=multi-user.target
`, ssg.oramaHome, dataDir, logFile)
}
// GenerateGatewayService generates the Orama Gateway systemd unit
@ -224,6 +290,8 @@ Wants=orama-node.service orama-olric.service
[Service]
Type=simple
%[4]s
ReadWritePaths=%[2]s
WorkingDirectory=%[1]s
Environment=HOME=%[1]s
ExecStart=%[1]s/bin/gateway --config %[2]s/data/gateway.yaml
@ -241,7 +309,7 @@ MemoryMax=4G
[Install]
WantedBy=multi-user.target
`, ssg.oramaHome, ssg.oramaDir, logFile)
`, ssg.oramaHome, ssg.oramaDir, logFile, oramaServiceHardening)
}
// GenerateAnyoneClientService generates the Anyone Client SOCKS5 proxy systemd unit.
@ -316,7 +384,7 @@ WantedBy=multi-user.target
// GenerateCoreDNSService generates the CoreDNS systemd unit
func (ssg *SystemdServiceGenerator) GenerateCoreDNSService() string {
return `[Unit]
return fmt.Sprintf(`[Unit]
Description=CoreDNS DNS Server with RQLite backend
Documentation=https://coredns.io
After=network-online.target orama-node.service
@ -324,11 +392,16 @@ Wants=network-online.target orama-node.service
[Service]
Type=simple
%[1]s
ReadWritePaths=%[2]s
AmbientCapabilities=CAP_NET_BIND_SERVICE
CapabilityBoundingSet=CAP_NET_BIND_SERVICE
ExecStart=/usr/local/bin/coredns -conf /etc/coredns/Corefile
Restart=on-failure
RestartSec=5
SyslogIdentifier=coredns
PrivateTmp=yes
LimitNOFILE=65536
TimeoutStopSec=30
KillMode=mixed
@ -336,12 +409,12 @@ MemoryMax=1G
[Install]
WantedBy=multi-user.target
`
`, oramaServiceHardening, ssg.oramaDir)
}
// GenerateCaddyService generates the Caddy systemd unit for SSL/TLS
func (ssg *SystemdServiceGenerator) GenerateCaddyService() string {
return `[Unit]
return fmt.Sprintf(`[Unit]
Description=Caddy HTTP/2 Server
Documentation=https://caddyserver.com/docs/
After=network-online.target orama-node.service coredns.service
@ -350,6 +423,11 @@ Wants=orama-node.service
[Service]
Type=simple
%[1]s
ReadWritePaths=%[2]s /var/lib/caddy /etc/caddy
Environment=XDG_DATA_HOME=/var/lib/caddy
AmbientCapabilities=CAP_NET_BIND_SERVICE
CapabilityBoundingSet=CAP_NET_BIND_SERVICE
ExecStart=/usr/bin/caddy run --environ --config /etc/caddy/Caddyfile
ExecReload=/usr/bin/caddy reload --config /etc/caddy/Caddyfile
TimeoutStopSec=5s
@ -364,7 +442,7 @@ MemoryMax=2G
[Install]
WantedBy=multi-user.target
`
`, oramaServiceHardening, ssg.oramaDir)
}
// SystemdController manages systemd service operations

4
pkg/environments/production/wireguard.go
View File

@ -117,8 +117,8 @@ func (wp *WireGuardProvisioner) GenerateConfig() string {
// Accept all WireGuard subnet traffic before UFW's conntrack "invalid" drop.
// Without this, packets reordered by the tunnel get silently dropped.
sb.WriteString("PostUp = iptables -I INPUT 1 -i wg0 -s 10.0.0.0/8 -j ACCEPT\n")
sb.WriteString("PostDown = iptables -D INPUT -i wg0 -s 10.0.0.0/8 -j ACCEPT\n")
sb.WriteString("PostUp = iptables -I INPUT 1 -i wg0 -s 10.0.0.0/24 -j ACCEPT\n")
sb.WriteString("PostDown = iptables -D INPUT -i wg0 -s 10.0.0.0/24 -j ACCEPT\n")
for _, peer := range wp.config.Peers {
sb.WriteString("\n[Peer]\n")

4
pkg/environments/production/wireguard_test.go
View File

@ -95,10 +95,10 @@ func TestWireGuardProvisioner_GenerateConfig_NoPeers(t *testing.T) {
if !strings.Contains(config, "PrivateKey = dGVzdHByaXZhdGVrZXl0ZXN0cHJpdmF0ZWtleXM=") {
t.Error("config should contain PrivateKey")
}
if !strings.Contains(config, "PostUp = iptables -I INPUT 1 -i wg0 -s 10.0.0.0/8 -j ACCEPT") {
if !strings.Contains(config, "PostUp = iptables -I INPUT 1 -i wg0 -s 10.0.0.0/24 -j ACCEPT") {
t.Error("config should contain PostUp iptables rule for WireGuard subnet")
}
if !strings.Contains(config, "PostDown = iptables -D INPUT -i wg0 -s 10.0.0.0/8 -j ACCEPT") {
if !strings.Contains(config, "PostDown = iptables -D INPUT -i wg0 -s 10.0.0.0/24 -j ACCEPT") {
t.Error("config should contain PostDown iptables cleanup rule")
}
if strings.Contains(config, "[Peer]") {

3
pkg/environments/templates/olric.yaml
View File

@ -15,3 +15,6 @@ memberlist:
- "{{.}}"
{{- end}}
{{- end}}
{{- if .EncryptionKey}}
encryptionKey: "{{.EncryptionKey}}"
{{- end}}

1
pkg/environments/templates/render.go
View File

@ -65,6 +65,7 @@ type OlricConfigData struct {
MemberlistEnvironment string // "local", "lan", or "wan"
MemberlistAdvertiseAddr string // Advertise address (WG IP) so other nodes can reach us
Peers []string // Seed peers for memberlist (host:port)
EncryptionKey string // Base64-encoded 32-byte key for memberlist gossip encryption (empty = no encryption)
}
// SystemdIPFSData holds parameters for systemd IPFS service rendering

10
pkg/environments/templates/systemd_gateway.service
View File

@ -5,6 +5,16 @@ Wants=orama-node.service
[Service]
Type=simple
User=orama
Group=orama
ProtectSystem=strict
ProtectHome=yes
NoNewPrivileges=yes
PrivateDevices=yes
ProtectKernelTunables=yes
ProtectKernelModules=yes
RestrictNamespaces=yes
ReadWritePaths={{.OramaDir}}
WorkingDirectory={{.HomeDir}}
Environment=HOME={{.HomeDir}}
ExecStart={{.HomeDir}}/bin/gateway --config {{.OramaDir}}/data/gateway.yaml

10
pkg/environments/templates/systemd_ipfs.service
View File

@ -5,6 +5,16 @@ Wants=network-online.target
[Service]
Type=simple
User=orama
Group=orama
ProtectSystem=strict
ProtectHome=yes
NoNewPrivileges=yes
PrivateDevices=yes
ProtectKernelTunables=yes
ProtectKernelModules=yes
RestrictNamespaces=yes
ReadWritePaths={{.IPFSRepoPath}} {{.OramaDir}}
Environment=HOME={{.HomeDir}}
Environment=IPFS_PATH={{.IPFSRepoPath}}
ExecStartPre=/bin/bash -c 'if [ -f {{.SecretsDir}}/swarm.key ] && [ ! -f {{.IPFSRepoPath}}/swarm.key ]; then cp {{.SecretsDir}}/swarm.key {{.IPFSRepoPath}}/swarm.key && chmod 600 {{.IPFSRepoPath}}/swarm.key; fi'

10
pkg/environments/templates/systemd_ipfs_cluster.service
View File

@ -6,6 +6,16 @@ Requires=orama-ipfs-{{.NodeType}}.service
[Service]
Type=simple
User=orama
Group=orama
ProtectSystem=strict
ProtectHome=yes
NoNewPrivileges=yes
PrivateDevices=yes
ProtectKernelTunables=yes
ProtectKernelModules=yes
RestrictNamespaces=yes
ReadWritePaths={{.ClusterPath}} {{.OramaDir}}
WorkingDirectory={{.HomeDir}}
Environment=HOME={{.HomeDir}}
Environment=CLUSTER_PATH={{.ClusterPath}}

10
pkg/environments/templates/systemd_node.service
View File

@ -6,6 +6,16 @@ Requires=orama-ipfs-cluster-{{.NodeType}}.service
[Service]
Type=simple
User=orama
Group=orama
ProtectSystem=strict
ProtectHome=yes
NoNewPrivileges=yes
PrivateDevices=yes
ProtectKernelTunables=yes
ProtectKernelModules=yes
RestrictNamespaces=yes
ReadWritePaths={{.OramaDir}}
WorkingDirectory={{.HomeDir}}
Environment=HOME={{.HomeDir}}
ExecStart={{.HomeDir}}/bin/orama-node --config {{.OramaDir}}/configs/{{.ConfigFile}}

10
pkg/environments/templates/systemd_olric.service
View File

@ -5,6 +5,16 @@ Wants=network-online.target
[Service]
Type=simple
User=orama
Group=orama
ProtectSystem=strict
ProtectHome=yes
NoNewPrivileges=yes
PrivateDevices=yes
ProtectKernelTunables=yes
ProtectKernelModules=yes
RestrictNamespaces=yes
ReadWritePaths={{.OramaDir}}
Environment=HOME={{.HomeDir}}
Environment=OLRIC_SERVER_CONFIG={{.ConfigPath}}
ExecStart=/usr/local/bin/olric-server

24
pkg/gateway/auth/crypto.go Normal file
View File

@ -0,0 +1,24 @@
package auth
import (
"crypto/hmac"
"crypto/sha256"
"encoding/hex"
)
// sha256Hex returns the lowercase hex-encoded SHA-256 hash of the input string.
// Used to hash refresh tokens before storage — deterministic so we can hash on
// insert and hash on lookup without storing the raw token.
func sha256Hex(s string) string {
h := sha256.Sum256([]byte(s))
return hex.EncodeToString(h[:])
}
// HmacSHA256Hex computes HMAC-SHA256 of data with the given secret key and
// returns the result as a lowercase hex string. Used for API key hashing —
// fast and deterministic, allowing direct DB lookup by hash.
func HmacSHA256Hex(data, secret string) string {
mac := hmac.New(sha256.New, []byte(secret))
mac.Write([]byte(data))
return hex.EncodeToString(mac.Sum(nil))
}

51
pkg/gateway/auth/service.go
View File

@ -24,14 +24,15 @@ import (
// Service handles authentication business logic
type Service struct {
logger *logging.ColoredLogger
orm client.NetworkClient
signingKey *rsa.PrivateKey
keyID string
edSigningKey ed25519.PrivateKey
edKeyID string
preferEdDSA bool
defaultNS string
logger *logging.ColoredLogger
orm client.NetworkClient
signingKey *rsa.PrivateKey
keyID string
edSigningKey ed25519.PrivateKey
edKeyID string
preferEdDSA bool
defaultNS string
apiKeyHMACSecret string // HMAC secret for hashing API keys before storage
}
func NewService(logger *logging.ColoredLogger, orm client.NetworkClient, signingKeyPEM string, defaultNS string) (*Service, error) {
@ -61,6 +62,21 @@ func NewService(logger *logging.ColoredLogger, orm client.NetworkClient, signing
return s, nil
}
// SetAPIKeyHMACSecret configures the HMAC secret used to hash API keys before storage.
// When set, API keys are stored as HMAC-SHA256(key, secret) in the database.
func (s *Service) SetAPIKeyHMACSecret(secret string) {
s.apiKeyHMACSecret = secret
}
// HashAPIKey returns the HMAC-SHA256 hash of an API key if the HMAC secret is set,
// or returns the raw key for backward compatibility during rolling upgrade.
func (s *Service) HashAPIKey(key string) string {
if s.apiKeyHMACSecret == "" {
return key
}
return HmacSHA256Hex(key, s.apiKeyHMACSecret)
}
// SetEdDSAKey configures an Ed25519 signing key for EdDSA JWT support.
// When set, new tokens are signed with EdDSA; RS256 is still accepted for verification.
func (s *Service) SetEdDSAKey(privKey ed25519.PrivateKey) {
@ -207,9 +223,10 @@ func (s *Service) IssueTokens(ctx context.Context, wallet, namespace string) (st
internalCtx := client.WithInternalAuth(ctx)
db := s.orm.Database()
hashedRefresh := sha256Hex(refresh)
if _, err := db.Query(internalCtx,
"INSERT INTO refresh_tokens(namespace_id, subject, token, audience, expires_at) VALUES (?, ?, ?, ?, datetime('now', '+30 days'))",
nsID, wallet, refresh, "gateway",
nsID, wallet, hashedRefresh, "gateway",
); err != nil {
return "", "", 0, fmt.Errorf("failed to store refresh token: %w", err)
}
@ -227,8 +244,9 @@ func (s *Service) RefreshToken(ctx context.Context, refreshToken, namespace stri
return "", "", 0, err
}
hashedRefresh := sha256Hex(refreshToken)
q := "SELECT subject FROM refresh_tokens WHERE namespace_id = ? AND token = ? AND revoked_at IS NULL AND (expires_at IS NULL OR expires_at > datetime('now')) LIMIT 1"
res, err := db.Query(internalCtx, q, nsID, refreshToken)
res, err := db.Query(internalCtx, q, nsID, hashedRefresh)
if err != nil || res == nil || res.Count == 0 {
return "", "", 0, fmt.Errorf("invalid or expired refresh token")
}
@ -262,7 +280,8 @@ func (s *Service) RevokeToken(ctx context.Context, namespace, token string, all
}
if token != "" {
_, err := db.Query(internalCtx, "UPDATE refresh_tokens SET revoked_at = datetime('now') WHERE namespace_id = ? AND token = ? AND revoked_at IS NULL", nsID, token)
hashedToken := sha256Hex(token)
_, err := db.Query(internalCtx, "UPDATE refresh_tokens SET revoked_at = datetime('now') WHERE namespace_id = ? AND token = ? AND revoked_at IS NULL", nsID, hashedToken)
return err
}
@ -335,19 +354,21 @@ func (s *Service) GetOrCreateAPIKey(ctx context.Context, wallet, namespace strin
}
apiKey = "ak_" + base64.RawURLEncoding.EncodeToString(buf) + ":" + namespace
if _, err := db.Query(internalCtx, "INSERT INTO api_keys(key, name, namespace_id) VALUES (?, ?, ?)", apiKey, "", nsID); err != nil {
// Store the HMAC hash of the key (not the raw key) if HMAC secret is configured
hashedKey := s.HashAPIKey(apiKey)
if _, err := db.Query(internalCtx, "INSERT INTO api_keys(key, name, namespace_id) VALUES (?, ?, ?)", hashedKey, "", nsID); err != nil {
return "", fmt.Errorf("failed to store api key: %w", err)
}
// Link wallet -> api_key
rid, err := db.Query(internalCtx, "SELECT id FROM api_keys WHERE key = ? LIMIT 1", apiKey)
rid, err := db.Query(internalCtx, "SELECT id FROM api_keys WHERE key = ? LIMIT 1", hashedKey)
if err == nil && rid != nil && rid.Count > 0 && len(rid.Rows) > 0 && len(rid.Rows[0]) > 0 {
apiKeyID := rid.Rows[0][0]
_, _ = db.Query(internalCtx, "INSERT OR IGNORE INTO wallet_api_keys(namespace_id, wallet, api_key_id) VALUES (?, ?, ?)", nsID, strings.ToLower(wallet), apiKeyID)
}
// Record ownerships
_, _ = db.Query(internalCtx, "INSERT OR IGNORE INTO namespace_ownership(namespace_id, owner_type, owner_id) VALUES (?, 'api_key', ?)", nsID, apiKey)
// Record ownerships — store the hash in ownership too
_, _ = db.Query(internalCtx, "INSERT OR IGNORE INTO namespace_ownership(namespace_id, owner_type, owner_id) VALUES (?, 'api_key', ?)", nsID, hashedKey)
_, _ = db.Query(internalCtx, "INSERT OR IGNORE INTO namespace_ownership(namespace_id, owner_type, owner_id) VALUES (?, 'wallet', ?)", nsID, wallet)
return apiKey, nil

Some files were not shown because too many files have changed in this diff Show More