DeBros/network
4
0
Fork 0
mirror of https://github.com/DeBrosOfficial/network.git synced 2026-01-30 01:53:02 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat: refactor API gateway and CLI utilities for improved functionality

Browse Source 
- Updated the API gateway documentation to reflect changes in architecture and functionality, emphasizing its role as a multi-functional entry point for decentralized services.
- Refactored CLI commands to utilize utility functions for better code organization and maintainability.
- Introduced new utility functions for handling peer normalization, service management, and port validation, enhancing the overall CLI experience.
- Added a new production installation script to streamline the setup process for users, including detailed dry-run summaries for better visibility.
- Enhanced validation mechanisms for configuration files and swarm keys, ensuring robust error handling and user feedback during setup.
This commit is contained in:
anonpenguin23 2025-12-31 10:16:26 +02:00
parent 54aab4841d
commit b3b1905fb2
41 changed files with 4814 additions and 6761 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

18
.cursor/rules/network.mdc
View File

@ -83,24 +83,10 @@ When learning a skill, follow this **collaborative, goal-oriented workflow**. Yo
# Sonr Gateway (or Sonr Network Gateway)
This project implements a high-performance, multi-protocol API gateway designed to bridge client applications with a decentralized backend infrastructure. It serves as a unified entry point that handles secure user authentication via JWT, provides RESTful access to a distributed key-value cache (Olric), and facilitates decentralized storage interactions with IPFS. Beyond standard HTTP routing and reverse proxying, the gateway supports real-time communication through Pub/Sub mechanisms (WebSockets), mobile engagement via push notifications, and low-level traffic routing using TCP SNI (Server Name Indication) for encrypted service discovery.
This project implements a high-performance, multi-functional API gateway designed to bridge client applications with a decentralized infrastructure. It serves as a unified entry point for diverse services including distributed caching (via Olric), decentralized storage (IPFS), serverless function execution, and real-time pub/sub messaging. The gateway handles critical cross-cutting concerns such as JWT-based authentication, secure anonymous proxying, and mobile push notifications, ensuring that requests are validated, authorized, and efficiently routed across the network's ecosystem.
**Architecture:** Edge Gateway / Middleware Layer (part of a larger Distributed System)
**Architecture:** Edge Gateway / Middleware-heavy Microservice
## Tech Stack
- **backend:** Go
## Patterns
- Reverse Proxy
- Middleware Chain
- Adapter Pattern (for storage/cache backends)
- and Observer Pattern (via Pub/Sub).
## Domain Entities
- `JWT (Authentication Tokens)`
- `Namespaces (Resource Isolation)`
- `Pub/Sub Topics`
- `Distributed Cache (Olric)`
- `Push Notifications`
- `and SNI Routes.`

17
CHANGELOG.md
View File

@ -13,6 +13,23 @@ The format is based on [Keep a Changelog][keepachangelog] and adheres to [Semant
### Deprecated
### Fixed
## [0.81.0] - 2025-12-31
### Added
- Implemented a new, robust authentication service within the Gateway for handling wallet-based challenges, signature verification (ETH/SOL), JWT issuance, and API key management.
- Introduced automatic recovery logic for RQLite to detect and recover from split-brain scenarios and ensure cluster stability during restarts.
### Changed
- Refactored the production installation command (`dbn prod install`) by moving installer logic and utility functions into a dedicated `pkg/cli/utils` package for better modularity and maintainability.
- Reworked the core logic for starting and managing LibP2P, RQLite, and the HTTP Gateway within the Node, including improved peer reconnection and cluster configuration synchronization.
### Deprecated
### Removed
### Fixed
- Corrected IPFS Cluster configuration logic to properly handle port assignments and ensure correct IPFS API addresses are used, resolving potential connection issues between cluster components.
## [0.80.0] - 2025-12-29
### Added

2
Makefile
View File

@ -19,7 +19,7 @@ test-e2e:
.PHONY: build clean test run-node run-node2 run-node3 run-example deps tidy fmt vet lint clear-ports install-hooks kill
VERSION := 0.80.0
VERSION := 0.81.0
COMMIT ?= $(shell git rev-parse --short HEAD 2>/dev/null || echo unknown)
DATE ?= $(shell date -u +%Y-%m-%dT%H:%M:%SZ)
LDFLAGS := -X 'main.version=$(VERSION)' -X 'main.commit=$(COMMIT)' -X 'main.date=$(DATE)'

775
pkg/cli/prod_commands.go
View File

@ -2,8 +2,6 @@ package cli
import (
"bufio"
"encoding/hex"
"errors"
"flag"
"fmt"
"net"
@ -11,269 +9,12 @@ import (
"os/exec"
"path/filepath"
"strings"
"syscall"
"time"
"github.com/DeBrosOfficial/network/pkg/config"
"github.com/DeBrosOfficial/network/pkg/cli/utils"
"github.com/DeBrosOfficial/network/pkg/environments/production"
"github.com/DeBrosOfficial/network/pkg/installer"
"github.com/multiformats/go-multiaddr"
)
// IPFSPeerInfo holds IPFS peer information for configuring Peering.Peers
type IPFSPeerInfo struct {
PeerID string
Addrs []string
}
// IPFSClusterPeerInfo contains IPFS Cluster peer information for cluster discovery
type IPFSClusterPeerInfo struct {
PeerID string
Addrs []string
}
// validateSwarmKey validates that a swarm key is 64 hex characters
func validateSwarmKey(key string) error {
key = strings.TrimSpace(key)
if len(key) != 64 {
return fmt.Errorf("swarm key must be 64 hex characters (32 bytes), got %d", len(key))
}
if _, err := hex.DecodeString(key); err != nil {
return fmt.Errorf("swarm key must be valid hexadecimal: %w", err)
}
return nil
}
// runInteractiveInstaller launches the TUI installer
func runInteractiveInstaller() {
config, err := installer.Run()
if err != nil {
fmt.Fprintf(os.Stderr, "❌ %v\n", err)
os.Exit(1)
}
// Convert TUI config to install args and run installation
var args []string
args = append(args, "--vps-ip", config.VpsIP)
args = append(args, "--domain", config.Domain)
args = append(args, "--branch", config.Branch)
if config.NoPull {
args = append(args, "--no-pull")
}
if !config.IsFirstNode {
if config.JoinAddress != "" {
args = append(args, "--join", config.JoinAddress)
}
if config.ClusterSecret != "" {
args = append(args, "--cluster-secret", config.ClusterSecret)
}
if config.SwarmKeyHex != "" {
args = append(args, "--swarm-key", config.SwarmKeyHex)
}
if len(config.Peers) > 0 {
args = append(args, "--peers", strings.Join(config.Peers, ","))
}
// Pass IPFS peer info for Peering.Peers configuration
if config.IPFSPeerID != "" {
args = append(args, "--ipfs-peer", config.IPFSPeerID)
}
if len(config.IPFSSwarmAddrs) > 0 {
args = append(args, "--ipfs-addrs", strings.Join(config.IPFSSwarmAddrs, ","))
}
// Pass IPFS Cluster peer info for cluster peer_addresses configuration
if config.IPFSClusterPeerID != "" {
args = append(args, "--ipfs-cluster-peer", config.IPFSClusterPeerID)
}
if len(config.IPFSClusterAddrs) > 0 {
args = append(args, "--ipfs-cluster-addrs", strings.Join(config.IPFSClusterAddrs, ","))
}
}
// Re-run with collected args
handleProdInstall(args)
}
// showDryRunSummary displays what would be done during installation without making changes
func showDryRunSummary(vpsIP, domain, branch string, peers []string, joinAddress string, isFirstNode bool, oramaDir string) {
fmt.Print("\n" + strings.Repeat("=", 70) + "\n")
fmt.Printf("DRY RUN - No changes will be made\n")
fmt.Print(strings.Repeat("=", 70) + "\n\n")
fmt.Printf("📋 Installation Summary:\n")
fmt.Printf(" VPS IP: %s\n", vpsIP)
fmt.Printf(" Domain: %s\n", domain)
fmt.Printf(" Branch: %s\n", branch)
if isFirstNode {
fmt.Printf(" Node Type: First node (creates new cluster)\n")
} else {
fmt.Printf(" Node Type: Joining existing cluster\n")
if joinAddress != "" {
fmt.Printf(" Join Address: %s\n", joinAddress)
}
if len(peers) > 0 {
fmt.Printf(" Peers: %d peer(s)\n", len(peers))
for _, peer := range peers {
fmt.Printf(" - %s\n", peer)
}
}
}
fmt.Printf("\n📁 Directories that would be created:\n")
fmt.Printf(" %s/configs/\n", oramaDir)
fmt.Printf(" %s/secrets/\n", oramaDir)
fmt.Printf(" %s/data/ipfs/repo/\n", oramaDir)
fmt.Printf(" %s/data/ipfs-cluster/\n", oramaDir)
fmt.Printf(" %s/data/rqlite/\n", oramaDir)
fmt.Printf(" %s/logs/\n", oramaDir)
fmt.Printf(" %s/tls-cache/\n", oramaDir)
fmt.Printf("\n🔧 Binaries that would be installed:\n")
fmt.Printf(" - Go (if not present)\n")
fmt.Printf(" - RQLite 8.43.0\n")
fmt.Printf(" - IPFS/Kubo 0.38.2\n")
fmt.Printf(" - IPFS Cluster (latest)\n")
fmt.Printf(" - Olric 0.7.0\n")
fmt.Printf(" - anyone-client (npm)\n")
fmt.Printf(" - DeBros binaries (built from %s branch)\n", branch)
fmt.Printf("\n🔐 Secrets that would be generated:\n")
fmt.Printf(" - Cluster secret (64-hex)\n")
fmt.Printf(" - IPFS swarm key\n")
fmt.Printf(" - Node identity (Ed25519 keypair)\n")
fmt.Printf("\n📝 Configuration files that would be created:\n")
fmt.Printf(" - %s/configs/node.yaml\n", oramaDir)
fmt.Printf(" - %s/configs/olric/config.yaml\n", oramaDir)
fmt.Printf("\n⚙️ Systemd services that would be created:\n")
fmt.Printf(" - debros-ipfs.service\n")
fmt.Printf(" - debros-ipfs-cluster.service\n")
fmt.Printf(" - debros-olric.service\n")
fmt.Printf(" - debros-node.service (includes embedded gateway + RQLite)\n")
fmt.Printf(" - debros-anyone-client.service\n")
fmt.Printf("\n🌐 Ports that would be used:\n")
fmt.Printf(" External (must be open in firewall):\n")
fmt.Printf(" - 80 (HTTP for ACME/Let's Encrypt)\n")
fmt.Printf(" - 443 (HTTPS gateway)\n")
fmt.Printf(" - 4101 (IPFS swarm)\n")
fmt.Printf(" - 7001 (RQLite Raft)\n")
fmt.Printf(" Internal (localhost only):\n")
fmt.Printf(" - 4501 (IPFS API)\n")
fmt.Printf(" - 5001 (RQLite HTTP)\n")
fmt.Printf(" - 6001 (Unified gateway)\n")
fmt.Printf(" - 8080 (IPFS gateway)\n")
fmt.Printf(" - 9050 (Anyone SOCKS5)\n")
fmt.Printf(" - 9094 (IPFS Cluster API)\n")
fmt.Printf(" - 3320/3322 (Olric)\n")
fmt.Print("\n" + strings.Repeat("=", 70) + "\n")
fmt.Printf("To proceed with installation, run without --dry-run\n")
fmt.Print(strings.Repeat("=", 70) + "\n\n")
}
// validateGeneratedConfig loads and validates the generated node configuration
func validateGeneratedConfig(oramaDir string) error {
configPath := filepath.Join(oramaDir, "configs", "node.yaml")
// Check if config file exists
if _, err := os.Stat(configPath); os.IsNotExist(err) {
return fmt.Errorf("configuration file not found at %s", configPath)
}
// Load the config file
file, err := os.Open(configPath)
if err != nil {
return fmt.Errorf("failed to open config file: %w", err)
}
defer file.Close()
var cfg config.Config
if err := config.DecodeStrict(file, &cfg); err != nil {
return fmt.Errorf("failed to parse config: %w", err)
}
// Validate the configuration
if errs := cfg.Validate(); len(errs) > 0 {
var errMsgs []string
for _, e := range errs {
errMsgs = append(errMsgs, e.Error())
}
return fmt.Errorf("configuration validation errors:\n - %s", strings.Join(errMsgs, "\n - "))
}
return nil
}
// validateDNSRecord validates that the domain points to the expected IP address
// Returns nil if DNS is valid, warning message if DNS doesn't match but continues,
// or error if DNS lookup fails completely
func validateDNSRecord(domain, expectedIP string) error {
if domain == "" {
return nil // No domain provided, skip validation
}
ips, err := net.LookupIP(domain)
if err != nil {
// DNS lookup failed - this is a warning, not a fatal error
// The user might be setting up DNS after installation
fmt.Printf(" ⚠️ DNS lookup failed for %s: %v\n", domain, err)
fmt.Printf(" Make sure DNS is configured before enabling HTTPS\n")
return nil
}
// Check if any resolved IP matches the expected IP
for _, ip := range ips {
if ip.String() == expectedIP {
fmt.Printf(" ✓ DNS validated: %s → %s\n", domain, expectedIP)
return nil
}
}
// DNS doesn't point to expected IP - warn but continue
resolvedIPs := make([]string, len(ips))
for i, ip := range ips {
resolvedIPs[i] = ip.String()
}
fmt.Printf(" ⚠️ DNS mismatch: %s resolves to %v, expected %s\n", domain, resolvedIPs, expectedIP)
fmt.Printf(" HTTPS certificate generation may fail until DNS is updated\n")
return nil
}
// normalizePeers normalizes and validates peer multiaddrs
func normalizePeers(peersStr string) ([]string, error) {
if peersStr == "" {
return nil, nil
}
// Split by comma and trim whitespace
rawPeers := strings.Split(peersStr, ",")
peers := make([]string, 0, len(rawPeers))
seen := make(map[string]bool)
for _, peer := range rawPeers {
peer = strings.TrimSpace(peer)
if peer == "" {
continue
}
// Validate multiaddr format
if _, err := multiaddr.NewMultiaddr(peer); err != nil {
return nil, fmt.Errorf("invalid multiaddr %q: %w", peer, err)
}
// Deduplicate
if !seen[peer] {
peers = append(peers, peer)
seen[peer] = true
}
}
return peers, nil
}
// HandleProdCommand handles production environment commands
func HandleProdCommand(args []string) {
if len(args) == 0 {
@ -368,294 +109,6 @@ func showProdHelp() {
fmt.Printf(" orama logs node --follow\n")
}
func handleProdInstall(args []string) {
// Parse arguments using flag.FlagSet
fs := flag.NewFlagSet("install", flag.ContinueOnError)
fs.SetOutput(os.Stderr)
force := fs.Bool("force", false, "Reconfigure all settings")
skipResourceChecks := fs.Bool("ignore-resource-checks", false, "Skip disk/RAM/CPU prerequisite validation")
vpsIP := fs.String("vps-ip", "", "VPS public IP address")
domain := fs.String("domain", "", "Domain for this node (e.g., node-123.orama.network)")
peersStr := fs.String("peers", "", "Comma-separated peer multiaddrs to connect to")
joinAddress := fs.String("join", "", "RQLite join address (IP:port) to join existing cluster")
branch := fs.String("branch", "main", "Git branch to use (main or nightly)")
clusterSecret := fs.String("cluster-secret", "", "Hex-encoded 32-byte cluster secret (for joining existing cluster)")
swarmKey := fs.String("swarm-key", "", "64-hex IPFS swarm key (for joining existing private network)")
ipfsPeerID := fs.String("ipfs-peer", "", "IPFS peer ID to connect to (auto-discovered from peer domain)")
ipfsAddrs := fs.String("ipfs-addrs", "", "Comma-separated IPFS swarm addresses (auto-discovered from peer domain)")
ipfsClusterPeerID := fs.String("ipfs-cluster-peer", "", "IPFS Cluster peer ID to connect to (auto-discovered from peer domain)")
ipfsClusterAddrs := fs.String("ipfs-cluster-addrs", "", "Comma-separated IPFS Cluster addresses (auto-discovered from peer domain)")
interactive := fs.Bool("interactive", false, "Run interactive TUI installer")
dryRun := fs.Bool("dry-run", false, "Show what would be done without making changes")
noPull := fs.Bool("no-pull", false, "Skip git clone/pull, use existing /home/debros/src")
if err := fs.Parse(args); err != nil {
if err == flag.ErrHelp {
return
}
fmt.Fprintf(os.Stderr, "❌ Failed to parse flags: %v\n", err)
os.Exit(1)
}
// Launch TUI installer if --interactive flag or no required args provided
if *interactive || (*vpsIP == "" && len(args) == 0) {
runInteractiveInstaller()
return
}
// Validate branch
if *branch != "main" && *branch != "nightly" {
fmt.Fprintf(os.Stderr, "❌ Invalid branch: %s (must be 'main' or 'nightly')\n", *branch)
os.Exit(1)
}
// Normalize and validate peers
peers, err := normalizePeers(*peersStr)
if err != nil {
fmt.Fprintf(os.Stderr, "❌ Invalid peers: %v\n", err)
fmt.Fprintf(os.Stderr, " Example: --peers /ip4/10.0.0.1/tcp/4001/p2p/Qm...,/ip4/10.0.0.2/tcp/4001/p2p/Qm...\n")
os.Exit(1)
}
// Validate setup requirements
if os.Geteuid() != 0 {
fmt.Fprintf(os.Stderr, "❌ Production install must be run as root (use sudo)\n")
os.Exit(1)
}
// Validate VPS IP is provided
if *vpsIP == "" {
fmt.Fprintf(os.Stderr, "❌ --vps-ip is required\n")
fmt.Fprintf(os.Stderr, " Usage: sudo orama install --vps-ip <public_ip>\n")
fmt.Fprintf(os.Stderr, " Or run: sudo orama install --interactive\n")
os.Exit(1)
}
// Determine if this is the first node (creates new cluster) or joining existing cluster
isFirstNode := len(peers) == 0 && *joinAddress == ""
if isFirstNode {
fmt.Printf("ℹ️ First node detected - will create new cluster\n")
} else {
fmt.Printf("ℹ️ Joining existing cluster\n")
// Cluster secret is required when joining
if *clusterSecret == "" {
fmt.Fprintf(os.Stderr, "❌ --cluster-secret is required when joining an existing cluster\n")
fmt.Fprintf(os.Stderr, " Provide the 64-hex secret from an existing node (cat ~/.orama/secrets/cluster-secret)\n")
os.Exit(1)
}
if err := production.ValidateClusterSecret(*clusterSecret); err != nil {
fmt.Fprintf(os.Stderr, "❌ Invalid --cluster-secret: %v\n", err)
os.Exit(1)
}
// Swarm key is required when joining
if *swarmKey == "" {
fmt.Fprintf(os.Stderr, "❌ --swarm-key is required when joining an existing cluster\n")
fmt.Fprintf(os.Stderr, " Provide the 64-hex swarm key from an existing node:\n")
fmt.Fprintf(os.Stderr, " cat ~/.orama/secrets/swarm.key | tail -1\n")
os.Exit(1)
}
if err := validateSwarmKey(*swarmKey); err != nil {
fmt.Fprintf(os.Stderr, "❌ Invalid --swarm-key: %v\n", err)
os.Exit(1)
}
}
oramaHome := "/home/debros"
oramaDir := oramaHome + "/.orama"
// If cluster secret was provided, save it to secrets directory before setup
if *clusterSecret != "" {
secretsDir := filepath.Join(oramaDir, "secrets")
if err := os.MkdirAll(secretsDir, 0755); err != nil {
fmt.Fprintf(os.Stderr, "❌ Failed to create secrets directory: %v\n", err)
os.Exit(1)
}
secretPath := filepath.Join(secretsDir, "cluster-secret")
if err := os.WriteFile(secretPath, []byte(*clusterSecret), 0600); err != nil {
fmt.Fprintf(os.Stderr, "❌ Failed to save cluster secret: %v\n", err)
os.Exit(1)
}
fmt.Printf(" ✓ Cluster secret saved\n")
}
// If swarm key was provided, save it to secrets directory in full format
if *swarmKey != "" {
secretsDir := filepath.Join(oramaDir, "secrets")
if err := os.MkdirAll(secretsDir, 0755); err != nil {
fmt.Fprintf(os.Stderr, "❌ Failed to create secrets directory: %v\n", err)
os.Exit(1)
}
// Convert 64-hex key to full swarm.key format
swarmKeyContent := fmt.Sprintf("/key/swarm/psk/1.0.0/\n/base16/\n%s\n", strings.ToUpper(*swarmKey))
swarmKeyPath := filepath.Join(secretsDir, "swarm.key")
if err := os.WriteFile(swarmKeyPath, []byte(swarmKeyContent), 0600); err != nil {
fmt.Fprintf(os.Stderr, "❌ Failed to save swarm key: %v\n", err)
os.Exit(1)
}
fmt.Printf(" ✓ Swarm key saved\n")
}
// Store IPFS peer info for later use in IPFS configuration
var ipfsPeerInfo *IPFSPeerInfo
if *ipfsPeerID != "" && *ipfsAddrs != "" {
ipfsPeerInfo = &IPFSPeerInfo{
PeerID: *ipfsPeerID,
Addrs: strings.Split(*ipfsAddrs, ","),
}
}
// Store IPFS Cluster peer info for cluster peer discovery
var ipfsClusterPeerInfo *IPFSClusterPeerInfo
if *ipfsClusterPeerID != "" {
var addrs []string
if *ipfsClusterAddrs != "" {
addrs = strings.Split(*ipfsClusterAddrs, ",")
}
ipfsClusterPeerInfo = &IPFSClusterPeerInfo{
PeerID: *ipfsClusterPeerID,
Addrs: addrs,
}
}
setup := production.NewProductionSetup(oramaHome, os.Stdout, *force, *branch, *noPull, *skipResourceChecks)
// Inform user if skipping git pull
if *noPull {
fmt.Printf(" ⚠️ --no-pull flag enabled: Skipping git clone/pull\n")
fmt.Printf(" Using existing repository at /home/debros/src\n")
}
// Check port availability before proceeding
if err := ensurePortsAvailable("install", defaultPorts()); err != nil {
fmt.Fprintf(os.Stderr, "❌ %v\n", err)
os.Exit(1)
}
// Validate DNS if domain is provided
if *domain != "" {
fmt.Printf("\n🌐 Pre-flight DNS validation...\n")
validateDNSRecord(*domain, *vpsIP)
}
// Dry-run mode: show what would be done and exit
if *dryRun {
showDryRunSummary(*vpsIP, *domain, *branch, peers, *joinAddress, isFirstNode, oramaDir)
return
}
// Save branch preference for future upgrades
if err := production.SaveBranchPreference(oramaDir, *branch); err != nil {
fmt.Fprintf(os.Stderr, "⚠️ Warning: Failed to save branch preference: %v\n", err)
}
// Phase 1: Check prerequisites
fmt.Printf("\n📋 Phase 1: Checking prerequisites...\n")
if err := setup.Phase1CheckPrerequisites(); err != nil {
fmt.Fprintf(os.Stderr, "❌ Prerequisites check failed: %v\n", err)
os.Exit(1)
}
// Phase 2: Provision environment
fmt.Printf("\n🛠️ Phase 2: Provisioning environment...\n")
if err := setup.Phase2ProvisionEnvironment(); err != nil {
fmt.Fprintf(os.Stderr, "❌ Environment provisioning failed: %v\n", err)
os.Exit(1)
}
// Phase 2b: Install binaries
fmt.Printf("\nPhase 2b: Installing binaries...\n")
if err := setup.Phase2bInstallBinaries(); err != nil {
fmt.Fprintf(os.Stderr, "❌ Binary installation failed: %v\n", err)
os.Exit(1)
}
// Phase 3: Generate secrets FIRST (before service initialization)
// This ensures cluster secret and swarm key exist before repos are seeded
fmt.Printf("\n🔐 Phase 3: Generating secrets...\n")
if err := setup.Phase3GenerateSecrets(); err != nil {
fmt.Fprintf(os.Stderr, "❌ Secret generation failed: %v\n", err)
os.Exit(1)
}
// Phase 4: Generate configs (BEFORE service initialization)
// This ensures node.yaml exists before services try to access it
fmt.Printf("\n⚙️ Phase 4: Generating configurations...\n")
enableHTTPS := *domain != ""
if err := setup.Phase4GenerateConfigs(peers, *vpsIP, enableHTTPS, *domain, *joinAddress); err != nil {
fmt.Fprintf(os.Stderr, "❌ Configuration generation failed: %v\n", err)
os.Exit(1)
}
// Validate generated configuration
fmt.Printf(" Validating generated configuration...\n")
if err := validateGeneratedConfig(oramaDir); err != nil {
fmt.Fprintf(os.Stderr, "❌ Configuration validation failed: %v\n", err)
os.Exit(1)
}
fmt.Printf(" ✓ Configuration validated\n")
// Phase 2c: Initialize services (after config is in place)
fmt.Printf("\nPhase 2c: Initializing services...\n")
var prodIPFSPeer *production.IPFSPeerInfo
if ipfsPeerInfo != nil {
prodIPFSPeer = &production.IPFSPeerInfo{
PeerID: ipfsPeerInfo.PeerID,
Addrs: ipfsPeerInfo.Addrs,
}
}
var prodIPFSClusterPeer *production.IPFSClusterPeerInfo
if ipfsClusterPeerInfo != nil {
prodIPFSClusterPeer = &production.IPFSClusterPeerInfo{
PeerID: ipfsClusterPeerInfo.PeerID,
Addrs: ipfsClusterPeerInfo.Addrs,
}
}
if err := setup.Phase2cInitializeServices(peers, *vpsIP, prodIPFSPeer, prodIPFSClusterPeer); err != nil {
fmt.Fprintf(os.Stderr, "❌ Service initialization failed: %v\n", err)
os.Exit(1)
}
// Phase 5: Create systemd services
fmt.Printf("\n🔧 Phase 5: Creating systemd services...\n")
if err := setup.Phase5CreateSystemdServices(enableHTTPS); err != nil {
fmt.Fprintf(os.Stderr, "❌ Service creation failed: %v\n", err)
os.Exit(1)
}
// Log completion with actual peer ID
setup.LogSetupComplete(setup.NodePeerID)
fmt.Printf("✅ Production installation complete!\n\n")
// For first node, print important secrets and identifiers
if isFirstNode {
fmt.Printf("📋 Save these for joining future nodes:\n\n")
// Print cluster secret
clusterSecretPath := filepath.Join(oramaDir, "secrets", "cluster-secret")
if clusterSecretData, err := os.ReadFile(clusterSecretPath); err == nil {
fmt.Printf(" Cluster Secret (--cluster-secret):\n")
fmt.Printf(" %s\n\n", string(clusterSecretData))
}
// Print swarm key
swarmKeyPath := filepath.Join(oramaDir, "secrets", "swarm.key")
if swarmKeyData, err := os.ReadFile(swarmKeyPath); err == nil {
swarmKeyContent := strings.TrimSpace(string(swarmKeyData))
lines := strings.Split(swarmKeyContent, "\n")
if len(lines) >= 3 {
// Extract just the hex part (last line)
fmt.Printf(" IPFS Swarm Key (--swarm-key, last line only):\n")
fmt.Printf(" %s\n\n", lines[len(lines)-1])
}
}
// Print peer ID
fmt.Printf(" Node Peer ID:\n")
fmt.Printf(" %s\n\n", setup.NodePeerID)
}
}
func handleProdUpgrade(args []string) {
// Parse arguments using flag.FlagSet
fs := flag.NewFlagSet("upgrade", flag.ContinueOnError)
@ -767,7 +220,7 @@ func handleProdUpgrade(args []string) {
}
// Check port availability after stopping services
if err := ensurePortsAvailable("prod upgrade", defaultPorts()); err != nil {
if err := utils.EnsurePortsAvailable("prod upgrade", utils.DefaultPorts()); err != nil {
fmt.Fprintf(os.Stderr, "❌ %v\n", err)
os.Exit(1)
}
@ -945,7 +398,7 @@ func handleProdUpgrade(args []string) {
fmt.Fprintf(os.Stderr, " ⚠️ Warning: Failed to reload systemd daemon: %v\n", err)
}
// Restart services to apply changes - use getProductionServices to only restart existing services
services := getProductionServices()
services := utils.GetProductionServices()
if len(services) == 0 {
fmt.Printf(" ⚠️ No services found to restart\n")
} else {
@ -991,10 +444,9 @@ func handleProdStatus() {
fmt.Printf("Services:\n")
found := false
for _, svc := range serviceNames {
cmd := exec.Command("systemctl", "is-active", "--quiet", svc)
err := cmd.Run()
active, _ := utils.IsServiceActive(svc)
status := "❌ Inactive"
if err == nil {
if active {
status = "✅ Active"
found = true
}
@ -1016,52 +468,6 @@ func handleProdStatus() {
fmt.Printf("\nView logs with: dbn prod logs <service>\n")
}
// resolveServiceName resolves service aliases to actual systemd service names
func resolveServiceName(alias string) ([]string, error) {
// Service alias mapping (unified - no bootstrap/node distinction)
aliases := map[string][]string{
"node": {"debros-node"},
"ipfs": {"debros-ipfs"},
"cluster": {"debros-ipfs-cluster"},
"ipfs-cluster": {"debros-ipfs-cluster"},
"gateway": {"debros-gateway"},
"olric": {"debros-olric"},
"rqlite": {"debros-node"}, // RQLite logs are in node logs
}
// Check if it's an alias
if serviceNames, ok := aliases[strings.ToLower(alias)]; ok {
// Filter to only existing services
var existing []string
for _, svc := range serviceNames {
unitPath := filepath.Join("/etc/systemd/system", svc+".service")
if _, err := os.Stat(unitPath); err == nil {
existing = append(existing, svc)
}
}
if len(existing) == 0 {
return nil, fmt.Errorf("no services found for alias %q", alias)
}
return existing, nil
}
// Check if it's already a full service name
unitPath := filepath.Join("/etc/systemd/system", alias+".service")
if _, err := os.Stat(unitPath); err == nil {
return []string{alias}, nil
}
// Try without .service suffix
if !strings.HasSuffix(alias, ".service") {
unitPath = filepath.Join("/etc/systemd/system", alias+".service")
if _, err := os.Stat(unitPath); err == nil {
return []string{alias}, nil
}
}
return nil, fmt.Errorf("service %q not found. Use: node, ipfs, cluster, gateway, olric, or full service name", alias)
}
func handleProdLogs(args []string) {
if len(args) == 0 {
fmt.Fprintf(os.Stderr, "Usage: dbn prod logs <service> [--follow]\n")
@ -1079,7 +485,7 @@ func handleProdLogs(args []string) {
}
// Resolve service alias to actual service names
serviceNames, err := resolveServiceName(serviceAlias)
serviceNames, err := utils.ResolveServiceName(serviceAlias)
if err != nil {
fmt.Fprintf(os.Stderr, "❌ %v\n", err)
fmt.Fprintf(os.Stderr, "\nAvailable service aliases: node, ipfs, cluster, gateway, olric\n")
@ -1138,145 +544,6 @@ func handleProdLogs(args []string) {
}
}
// errServiceNotFound marks units that systemd does not know about.
var errServiceNotFound = errors.New("service not found")
type portSpec struct {
Name string
Port int
}
var servicePorts = map[string][]portSpec{
"debros-gateway": {{"Gateway API", 6001}},
"debros-olric": {{"Olric HTTP", 3320}, {"Olric Memberlist", 3322}},
"debros-node": {{"RQLite HTTP", 5001}, {"RQLite Raft", 7001}},
"debros-ipfs": {{"IPFS API", 4501}, {"IPFS Gateway", 8080}, {"IPFS Swarm", 4101}},
"debros-ipfs-cluster": {{"IPFS Cluster API", 9094}},
}
// defaultPorts is used for fresh installs/upgrades before unit files exist.
func defaultPorts() []portSpec {
return []portSpec{
{"IPFS Swarm", 4001},
{"IPFS API", 4501},
{"IPFS Gateway", 8080},
{"Gateway API", 6001},
{"RQLite HTTP", 5001},
{"RQLite Raft", 7001},
{"IPFS Cluster API", 9094},
{"Olric HTTP", 3320},
{"Olric Memberlist", 3322},
}
}
func isServiceActive(service string) (bool, error) {
cmd := exec.Command("systemctl", "is-active", "--quiet", service)
if err := cmd.Run(); err != nil {
if exitErr, ok := err.(*exec.ExitError); ok {
switch exitErr.ExitCode() {
case 3:
return false, nil
case 4:
return false, errServiceNotFound
}
}
return false, err
}
return true, nil
}
func isServiceEnabled(service string) (bool, error) {
cmd := exec.Command("systemctl", "is-enabled", "--quiet", service)
if err := cmd.Run(); err != nil {
if exitErr, ok := err.(*exec.ExitError); ok {
switch exitErr.ExitCode() {
case 1:
return false, nil // Service is disabled
case 4:
return false, errServiceNotFound
}
}
return false, err
}
return true, nil
}
func collectPortsForServices(services []string, skipActive bool) ([]portSpec, error) {
seen := make(map[int]portSpec)
for _, svc := range services {
if skipActive {
active, err := isServiceActive(svc)
if err != nil {
return nil, fmt.Errorf("unable to check %s: %w", svc, err)
}
if active {
continue
}
}
for _, spec := range servicePorts[svc] {
if _, ok := seen[spec.Port]; !ok {
seen[spec.Port] = spec
}
}
}
ports := make([]portSpec, 0, len(seen))
for _, spec := range seen {
ports = append(ports, spec)
}
return ports, nil
}
func ensurePortsAvailable(action string, ports []portSpec) error {
for _, spec := range ports {
ln, err := net.Listen("tcp", fmt.Sprintf("0.0.0.0:%d", spec.Port))
if err != nil {
if errors.Is(err, syscall.EADDRINUSE) || strings.Contains(err.Error(), "address already in use") {
return fmt.Errorf("%s cannot continue: %s (port %d) is already in use", action, spec.Name, spec.Port)
}
return fmt.Errorf("%s cannot continue: failed to inspect %s (port %d): %w", action, spec.Name, spec.Port, err)
}
_ = ln.Close()
}
return nil
}
// getProductionServices returns a list of all DeBros production service names that exist
func getProductionServices() []string {
// Unified service names (no bootstrap/node distinction)
allServices := []string{
"debros-gateway",
"debros-node",
"debros-olric",
"debros-ipfs-cluster",
"debros-ipfs",
"debros-anyone-client",
}
// Filter to only existing services by checking if unit file exists
var existing []string
for _, svc := range allServices {
unitPath := filepath.Join("/etc/systemd/system", svc+".service")
if _, err := os.Stat(unitPath); err == nil {
existing = append(existing, svc)
}
}
return existing
}
func isServiceMasked(service string) (bool, error) {
cmd := exec.Command("systemctl", "is-enabled", service)
output, err := cmd.CombinedOutput()
if err != nil {
outputStr := string(output)
if strings.Contains(outputStr, "masked") {
return true, nil
}
return false, err
}
return false, nil
}
func handleProdStart() {
if os.Geteuid() != 0 {
fmt.Fprintf(os.Stderr, "❌ Production commands must be run as root (use sudo)\n")
@ -1285,7 +552,7 @@ func handleProdStart() {
fmt.Printf("Starting all DeBros production services...\n")
services := getProductionServices()
services := utils.GetProductionServices()
if len(services) == 0 {
fmt.Printf(" ⚠️ No DeBros services found\n")
return
@ -1301,7 +568,7 @@ func handleProdStart() {
inactive := make([]string, 0, len(services))
for _, svc := range services {
// Check if service is masked and unmask it
masked, err := isServiceMasked(svc)
masked, err := utils.IsServiceMasked(svc)
if err == nil && masked {
fmt.Printf(" ⚠️ %s is masked, unmasking...\n", svc)
if err := exec.Command("systemctl", "unmask", svc).Run(); err != nil {
@ -1311,7 +578,7 @@ func handleProdStart() {
}
}
active, err := isServiceActive(svc)
active, err := utils.IsServiceActive(svc)
if err != nil {
fmt.Printf(" ⚠️ Unable to check %s: %v\n", svc, err)
continue
@ -1319,7 +586,7 @@ func handleProdStart() {
if active {
fmt.Printf(" ℹ️ %s already running\n", svc)
// Re-enable if disabled (in case it was stopped with 'dbn prod stop')
enabled, err := isServiceEnabled(svc)
enabled, err := utils.IsServiceEnabled(svc)
if err == nil && !enabled {
if err := exec.Command("systemctl", "enable", svc).Run(); err != nil {
fmt.Printf(" ⚠️ Failed to re-enable %s: %v\n", svc, err)
@ -1338,12 +605,12 @@ func handleProdStart() {
}
// Check port availability for services we're about to start
ports, err := collectPortsForServices(inactive, false)
ports, err := utils.CollectPortsForServices(inactive, false)
if err != nil {
fmt.Fprintf(os.Stderr, "❌ %v\n", err)
os.Exit(1)
}
if err := ensurePortsAvailable("prod start", ports); err != nil {
if err := utils.EnsurePortsAvailable("prod start", ports); err != nil {
fmt.Fprintf(os.Stderr, "❌ %v\n", err)
os.Exit(1)
}
@ -1351,7 +618,7 @@ func handleProdStart() {
// Enable and start inactive services
for _, svc := range inactive {
// Re-enable the service first (in case it was disabled by 'dbn prod stop')
enabled, err := isServiceEnabled(svc)
enabled, err := utils.IsServiceEnabled(svc)
if err == nil && !enabled {
if err := exec.Command("systemctl", "enable", svc).Run(); err != nil {
fmt.Printf(" ⚠️ Failed to enable %s: %v\n", svc, err)
@ -1385,7 +652,7 @@ func handleProdStop() {
fmt.Printf("Stopping all DeBros production services...\n")
services := getProductionServices()
services := utils.GetProductionServices()
if len(services) == 0 {
fmt.Printf(" ⚠️ No DeBros services found\n")
return
@ -1424,7 +691,7 @@ func handleProdStop() {
hadError := false
for _, svc := range services {
active, err := isServiceActive(svc)
active, err := utils.IsServiceActive(svc)
if err != nil {
fmt.Printf(" ⚠️ Unable to check %s: %v\n", svc, err)
hadError = true
@ -1441,7 +708,7 @@ func handleProdStop() {
} else {
// Wait and verify again
time.Sleep(1 * time.Second)
if stillActive, _ := isServiceActive(svc); stillActive {
if stillActive, _ := utils.IsServiceActive(svc); stillActive {
fmt.Printf(" ❌ %s restarted itself (Restart=always)\n", svc)
hadError = true
} else {
@ -1451,7 +718,7 @@ func handleProdStop() {
}
// Disable the service to prevent it from auto-starting on boot
enabled, err := isServiceEnabled(svc)
enabled, err := utils.IsServiceEnabled(svc)
if err != nil {
fmt.Printf(" ⚠️ Unable to check if %s is enabled: %v\n", svc, err)
// Continue anyway - try to disable
@ -1486,7 +753,7 @@ func handleProdRestart() {
fmt.Printf("Restarting all DeBros production services...\n")
services := getProductionServices()
services := utils.GetProductionServices()
if len(services) == 0 {
fmt.Printf(" ⚠️ No DeBros services found\n")
return
@ -1495,7 +762,7 @@ func handleProdRestart() {
// Stop all active services first
fmt.Printf(" Stopping services...\n")
for _, svc := range services {
active, err := isServiceActive(svc)
active, err := utils.IsServiceActive(svc)
if err != nil {
fmt.Printf(" ⚠️ Unable to check %s: %v\n", svc, err)
continue
@ -1512,12 +779,12 @@ func handleProdRestart() {
}
// Check port availability before restarting
ports, err := collectPortsForServices(services, false)
ports, err := utils.CollectPortsForServices(services, false)
if err != nil {
fmt.Fprintf(os.Stderr, "❌ %v\n", err)
os.Exit(1)
}
if err := ensurePortsAvailable("prod restart", ports); err != nil {
if err := utils.EnsurePortsAvailable("prod restart", ports); err != nil {
fmt.Fprintf(os.Stderr, "❌ %v\n", err)
os.Exit(1)
}

4
pkg/cli/prod_commands_test.go
View File

@ -2,6 +2,8 @@ package cli
import (
"testing"
"github.com/DeBrosOfficial/network/pkg/cli/utils"
)
// TestProdCommandFlagParsing verifies that prod command flags are parsed correctly
@ -156,7 +158,7 @@ func TestNormalizePeers(t *testing.T) {
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
peers, err := normalizePeers(tt.input)
peers, err := utils.NormalizePeers(tt.input)
if tt.expectError && err == nil {
t.Errorf("expected error but got none")

264
pkg/cli/prod_install.go Normal file
View File

@ -0,0 +1,264 @@
package cli
import (
"flag"
"fmt"
"os"
"path/filepath"
"strings"
"github.com/DeBrosOfficial/network/pkg/cli/utils"
"github.com/DeBrosOfficial/network/pkg/environments/production"
)
func handleProdInstall(args []string) {
// Parse arguments using flag.FlagSet
fs := flag.NewFlagSet("install", flag.ContinueOnError)
fs.SetOutput(os.Stderr)
vpsIP := fs.String("vps-ip", "", "Public IP of this VPS (required)")
domain := fs.String("domain", "", "Domain name for HTTPS (optional, e.g. gateway.example.com)")
branch := fs.String("branch", "main", "Git branch to use (main or nightly)")
noPull := fs.Bool("no-pull", false, "Skip git clone/pull, use existing repository in /home/debros/src")
force := fs.Bool("force", false, "Force reconfiguration even if already installed")
dryRun := fs.Bool("dry-run", false, "Show what would be done without making changes")
skipResourceChecks := fs.Bool("skip-checks", false, "Skip minimum resource checks (RAM/CPU)")
// Cluster join flags
joinAddress := fs.String("join", "", "Join an existing cluster (e.g. 1.2.3.4:7001)")
clusterSecret := fs.String("cluster-secret", "", "Cluster secret for IPFS Cluster (required if joining)")
swarmKey := fs.String("swarm-key", "", "IPFS Swarm key (required if joining)")
peersStr := fs.String("peers", "", "Comma-separated list of bootstrap peer multiaddrs")
// IPFS/Cluster specific info for Peering configuration
ipfsPeerID := fs.String("ipfs-peer", "", "Peer ID of existing IPFS node to peer with")
ipfsAddrs := fs.String("ipfs-addrs", "", "Comma-separated multiaddrs of existing IPFS node")
ipfsClusterPeerID := fs.String("ipfs-cluster-peer", "", "Peer ID of existing IPFS Cluster node")
ipfsClusterAddrs := fs.String("ipfs-cluster-addrs", "", "Comma-separated multiaddrs of existing IPFS Cluster node")
if err := fs.Parse(args); err != nil {
if err == flag.ErrHelp {
return
}
fmt.Fprintf(os.Stderr, "❌ Failed to parse flags: %v\n", err)
os.Exit(1)
}
// Validate required flags
if *vpsIP == "" && !*dryRun {
fmt.Fprintf(os.Stderr, "❌ Error: --vps-ip is required for installation\n")
fmt.Fprintf(os.Stderr, " Example: dbn prod install --vps-ip 1.2.3.4\n")
os.Exit(1)
}
if os.Geteuid() != 0 && !*dryRun {
fmt.Fprintf(os.Stderr, "❌ Production installation must be run as root (use sudo)\n")
os.Exit(1)
}
oramaHome := "/home/debros"
oramaDir := oramaHome + "/.orama"
fmt.Printf("🚀 Starting production installation...\n\n")
isFirstNode := *joinAddress == ""
peers, err := utils.NormalizePeers(*peersStr)
if err != nil {
fmt.Fprintf(os.Stderr, "❌ Invalid peers: %v\n", err)
os.Exit(1)
}
// If cluster secret was provided, save it to secrets directory before setup
if *clusterSecret != "" {
secretsDir := filepath.Join(oramaDir, "secrets")
if err := os.MkdirAll(secretsDir, 0755); err != nil {
fmt.Fprintf(os.Stderr, "❌ Failed to create secrets directory: %v\n", err)
os.Exit(1)
}
secretPath := filepath.Join(secretsDir, "cluster-secret")
if err := os.WriteFile(secretPath, []byte(*clusterSecret), 0600); err != nil {
fmt.Fprintf(os.Stderr, "❌ Failed to save cluster secret: %v\n", err)
os.Exit(1)
}
fmt.Printf(" ✓ Cluster secret saved\n")
}
// If swarm key was provided, save it to secrets directory in full format
if *swarmKey != "" {
secretsDir := filepath.Join(oramaDir, "secrets")
if err := os.MkdirAll(secretsDir, 0755); err != nil {
fmt.Fprintf(os.Stderr, "❌ Failed to create secrets directory: %v\n", err)
os.Exit(1)
}
// Convert 64-hex key to full swarm.key format
swarmKeyContent := fmt.Sprintf("/key/swarm/psk/1.0.0/\n/base16/\n%s\n", strings.ToUpper(*swarmKey))
swarmKeyPath := filepath.Join(secretsDir, "swarm.key")
if err := os.WriteFile(swarmKeyPath, []byte(swarmKeyContent), 0600); err != nil {
fmt.Fprintf(os.Stderr, "❌ Failed to save swarm key: %v\n", err)
os.Exit(1)
}
fmt.Printf(" ✓ Swarm key saved\n")
}
// Store IPFS peer info for peering
var ipfsPeerInfo *utils.IPFSPeerInfo
if *ipfsPeerID != "" {
var addrs []string
if *ipfsAddrs != "" {
addrs = strings.Split(*ipfsAddrs, ",")
}
ipfsPeerInfo = &utils.IPFSPeerInfo{
PeerID: *ipfsPeerID,
Addrs: addrs,
}
}
// Store IPFS Cluster peer info for cluster peer discovery
var ipfsClusterPeerInfo *utils.IPFSClusterPeerInfo
if *ipfsClusterPeerID != "" {
var addrs []string
if *ipfsClusterAddrs != "" {
addrs = strings.Split(*ipfsClusterAddrs, ",")
}
ipfsClusterPeerInfo = &utils.IPFSClusterPeerInfo{
PeerID: *ipfsClusterPeerID,
Addrs: addrs,
}
}
setup := production.NewProductionSetup(oramaHome, os.Stdout, *force, *branch, *noPull, *skipResourceChecks)
// Inform user if skipping git pull
if *noPull {
fmt.Printf(" ⚠️ --no-pull flag enabled: Skipping git clone/pull\n")
fmt.Printf(" Using existing repository at /home/debros/src\n")
}
// Check port availability before proceeding
if err := utils.EnsurePortsAvailable("install", utils.DefaultPorts()); err != nil {
fmt.Fprintf(os.Stderr, "❌ %v\n", err)
os.Exit(1)
}
// Validate DNS if domain is provided
if *domain != "" {
fmt.Printf("\n🌐 Pre-flight DNS validation...\n")
utils.ValidateDNSRecord(*domain, *vpsIP)
}
// Dry-run mode: show what would be done and exit
if *dryRun {
utils.ShowDryRunSummary(*vpsIP, *domain, *branch, peers, *joinAddress, isFirstNode, oramaDir)
return
}
// Save branch preference for future upgrades
if err := production.SaveBranchPreference(oramaDir, *branch); err != nil {
fmt.Fprintf(os.Stderr, "⚠️ Warning: Failed to save branch preference: %v\n", err)
}
// Phase 1: Check prerequisites
fmt.Printf("\n📋 Phase 1: Checking prerequisites...\n")
if err := setup.Phase1CheckPrerequisites(); err != nil {
fmt.Fprintf(os.Stderr, "❌ Prerequisites check failed: %v\n", err)
os.Exit(1)
}
// Phase 2: Provision environment
fmt.Printf("\n🛠️ Phase 2: Provisioning environment...\n")
if err := setup.Phase2ProvisionEnvironment(); err != nil {
fmt.Fprintf(os.Stderr, "❌ Environment provisioning failed: %v\n", err)
os.Exit(1)
}
// Phase 2b: Install binaries
fmt.Printf("\nPhase 2b: Installing binaries...\n")
if err := setup.Phase2bInstallBinaries(); err != nil {
fmt.Fprintf(os.Stderr, "❌ Binary installation failed: %v\n", err)
os.Exit(1)
}
// Phase 3: Generate secrets FIRST (before service initialization)
// This ensures cluster secret and swarm key exist before repos are seeded
fmt.Printf("\n🔐 Phase 3: Generating secrets...\n")
if err := setup.Phase3GenerateSecrets(); err != nil {
fmt.Fprintf(os.Stderr, "❌ Secret generation failed: %v\n", err)
os.Exit(1)
}
// Phase 4: Generate configs (BEFORE service initialization)
// This ensures node.yaml exists before services try to access it
fmt.Printf("\n⚙️ Phase 4: Generating configurations...\n")
enableHTTPS := *domain != ""
if err := setup.Phase4GenerateConfigs(peers, *vpsIP, enableHTTPS, *domain, *joinAddress); err != nil {
fmt.Fprintf(os.Stderr, "❌ Configuration generation failed: %v\n", err)
os.Exit(1)
}
// Validate generated configuration
fmt.Printf(" Validating generated configuration...\n")
if err := utils.ValidateGeneratedConfig(oramaDir); err != nil {
fmt.Fprintf(os.Stderr, "❌ Configuration validation failed: %v\n", err)
os.Exit(1)
}
fmt.Printf(" ✓ Configuration validated\n")
// Phase 2c: Initialize services (after config is in place)
fmt.Printf("\nPhase 2c: Initializing services...\n")
var prodIPFSPeer *production.IPFSPeerInfo
if ipfsPeerInfo != nil {
prodIPFSPeer = &production.IPFSPeerInfo{
PeerID: ipfsPeerInfo.PeerID,
Addrs: ipfsPeerInfo.Addrs,
}
}
var prodIPFSClusterPeer *production.IPFSClusterPeerInfo
if ipfsClusterPeerInfo != nil {
prodIPFSClusterPeer = &production.IPFSClusterPeerInfo{
PeerID: ipfsClusterPeerInfo.PeerID,
Addrs: ipfsClusterPeerInfo.Addrs,
}
}
if err := setup.Phase2cInitializeServices(peers, *vpsIP, prodIPFSPeer, prodIPFSClusterPeer); err != nil {
fmt.Fprintf(os.Stderr, "❌ Service initialization failed: %v\n", err)
os.Exit(1)
}
// Phase 5: Create systemd services
fmt.Printf("\n🔧 Phase 5: Creating systemd services...\n")
if err := setup.Phase5CreateSystemdServices(enableHTTPS); err != nil {
fmt.Fprintf(os.Stderr, "❌ Service creation failed: %v\n", err)
os.Exit(1)
}
// Log completion with actual peer ID
setup.LogSetupComplete(setup.NodePeerID)
fmt.Printf("✅ Production installation complete!\n\n")
// For first node, print important secrets and identifiers
if isFirstNode {
fmt.Printf("📋 Save these for joining future nodes:\n\n")
// Print cluster secret
clusterSecretPath := filepath.Join(oramaDir, "secrets", "cluster-secret")
if clusterSecretData, err := os.ReadFile(clusterSecretPath); err == nil {
fmt.Printf(" Cluster Secret (--cluster-secret):\n")
fmt.Printf(" %s\n\n", string(clusterSecretData))
}
// Print swarm key
swarmKeyPath := filepath.Join(oramaDir, "secrets", "swarm.key")
if swarmKeyData, err := os.ReadFile(swarmKeyPath); err == nil {
swarmKeyContent := strings.TrimSpace(string(swarmKeyData))
lines := strings.Split(swarmKeyContent, "\n")
if len(lines) >= 3 {
// Extract just the hex part (last line)
fmt.Printf(" IPFS Swarm Key (--swarm-key, last line only):\n")
fmt.Printf(" %s\n\n", lines[len(lines)-1])
}
}
// Print peer ID
fmt.Printf(" Node Peer ID:\n")
fmt.Printf(" %s\n\n", setup.NodePeerID)
}
}

97
pkg/cli/utils/install.go Normal file
View File

@ -0,0 +1,97 @@
package utils
import (
"fmt"
"strings"
)
// IPFSPeerInfo holds IPFS peer information for configuring Peering.Peers
type IPFSPeerInfo struct {
PeerID string
Addrs []string
}
// IPFSClusterPeerInfo contains IPFS Cluster peer information for cluster discovery
type IPFSClusterPeerInfo struct {
PeerID string
Addrs []string
}
// ShowDryRunSummary displays what would be done during installation without making changes
func ShowDryRunSummary(vpsIP, domain, branch string, peers []string, joinAddress string, isFirstNode bool, oramaDir string) {
fmt.Print("\n" + strings.Repeat("=", 70) + "\n")
fmt.Printf("DRY RUN - No changes will be made\n")
fmt.Print(strings.Repeat("=", 70) + "\n\n")
fmt.Printf("📋 Installation Summary:\n")
fmt.Printf(" VPS IP: %s\n", vpsIP)
fmt.Printf(" Domain: %s\n", domain)
fmt.Printf(" Branch: %s\n", branch)
if isFirstNode {
fmt.Printf(" Node Type: First node (creates new cluster)\n")
} else {
fmt.Printf(" Node Type: Joining existing cluster\n")
if joinAddress != "" {
fmt.Printf(" Join Address: %s\n", joinAddress)
}
if len(peers) > 0 {
fmt.Printf(" Peers: %d peer(s)\n", len(peers))
for _, peer := range peers {
fmt.Printf(" - %s\n", peer)
}
}
}
fmt.Printf("\n📁 Directories that would be created:\n")
fmt.Printf(" %s/configs/\n", oramaDir)
fmt.Printf(" %s/secrets/\n", oramaDir)
fmt.Printf(" %s/data/ipfs/repo/\n", oramaDir)
fmt.Printf(" %s/data/ipfs-cluster/\n", oramaDir)
fmt.Printf(" %s/data/rqlite/\n", oramaDir)
fmt.Printf(" %s/logs/\n", oramaDir)
fmt.Printf(" %s/tls-cache/\n", oramaDir)
fmt.Printf("\n🔧 Binaries that would be installed:\n")
fmt.Printf(" - Go (if not present)\n")
fmt.Printf(" - RQLite 8.43.0\n")
fmt.Printf(" - IPFS/Kubo 0.38.2\n")
fmt.Printf(" - IPFS Cluster (latest)\n")
fmt.Printf(" - Olric 0.7.0\n")
fmt.Printf(" - anyone-client (npm)\n")
fmt.Printf(" - DeBros binaries (built from %s branch)\n", branch)
fmt.Printf("\n🔐 Secrets that would be generated:\n")
fmt.Printf(" - Cluster secret (64-hex)\n")
fmt.Printf(" - IPFS swarm key\n")
fmt.Printf(" - Node identity (Ed25519 keypair)\n")
fmt.Printf("\n📝 Configuration files that would be created:\n")
fmt.Printf(" - %s/configs/node.yaml\n", oramaDir)
fmt.Printf(" - %s/configs/olric/config.yaml\n", oramaDir)
fmt.Printf("\n⚙️ Systemd services that would be created:\n")
fmt.Printf(" - debros-ipfs.service\n")
fmt.Printf(" - debros-ipfs-cluster.service\n")
fmt.Printf(" - debros-olric.service\n")
fmt.Printf(" - debros-node.service (includes embedded gateway + RQLite)\n")
fmt.Printf(" - debros-anyone-client.service\n")
fmt.Printf("\n🌐 Ports that would be used:\n")
fmt.Printf(" External (must be open in firewall):\n")
fmt.Printf(" - 80 (HTTP for ACME/Let's Encrypt)\n")
fmt.Printf(" - 443 (HTTPS gateway)\n")
fmt.Printf(" - 4101 (IPFS swarm)\n")
fmt.Printf(" - 7001 (RQLite Raft)\n")
fmt.Printf(" Internal (localhost only):\n")
fmt.Printf(" - 4501 (IPFS API)\n")
fmt.Printf(" - 5001 (RQLite HTTP)\n")
fmt.Printf(" - 6001 (Unified gateway)\n")
fmt.Printf(" - 8080 (IPFS gateway)\n")
fmt.Printf(" - 9050 (Anyone SOCKS5)\n")
fmt.Printf(" - 9094 (IPFS Cluster API)\n")
fmt.Printf(" - 3320/3322 (Olric)\n")
fmt.Print("\n" + strings.Repeat("=", 70) + "\n")
fmt.Printf("To proceed with installation, run without --dry-run\n")
fmt.Print(strings.Repeat("=", 70) + "\n\n")
}

217
pkg/cli/utils/systemd.go Normal file
View File

@ -0,0 +1,217 @@
package utils
import (
"errors"
"fmt"
"net"
"os"
"os/exec"
"path/filepath"
"strings"
"syscall"
)
var ErrServiceNotFound = errors.New("service not found")
// PortSpec defines a port and its name for checking availability
type PortSpec struct {
Name string
Port int
}
var ServicePorts = map[string][]PortSpec{
"debros-gateway": {
{Name: "Gateway API", Port: 6001},
},
"debros-olric": {
{Name: "Olric HTTP", Port: 3320},
{Name: "Olric Memberlist", Port: 3322},
},
"debros-node": {
{Name: "RQLite HTTP", Port: 5001},
{Name: "RQLite Raft", Port: 7001},
},
"debros-ipfs": {
{Name: "IPFS API", Port: 4501},
{Name: "IPFS Gateway", Port: 8080},
{Name: "IPFS Swarm", Port: 4101},
},
"debros-ipfs-cluster": {
{Name: "IPFS Cluster API", Port: 9094},
},
}
// DefaultPorts is used for fresh installs/upgrades before unit files exist.
func DefaultPorts() []PortSpec {
return []PortSpec{
{Name: "IPFS Swarm", Port: 4001},
{Name: "IPFS API", Port: 4501},
{Name: "IPFS Gateway", Port: 8080},
{Name: "Gateway API", Port: 6001},
{Name: "RQLite HTTP", Port: 5001},
{Name: "RQLite Raft", Port: 7001},
{Name: "IPFS Cluster API", Port: 9094},
{Name: "Olric HTTP", Port: 3320},
{Name: "Olric Memberlist", Port: 3322},
}
}
// ResolveServiceName resolves service aliases to actual systemd service names
func ResolveServiceName(alias string) ([]string, error) {
// Service alias mapping (unified - no bootstrap/node distinction)
aliases := map[string][]string{
"node": {"debros-node"},
"ipfs": {"debros-ipfs"},
"cluster": {"debros-ipfs-cluster"},
"ipfs-cluster": {"debros-ipfs-cluster"},
"gateway": {"debros-gateway"},
"olric": {"debros-olric"},
"rqlite": {"debros-node"}, // RQLite logs are in node logs
}
// Check if it's an alias
if serviceNames, ok := aliases[strings.ToLower(alias)]; ok {
// Filter to only existing services
var existing []string
for _, svc := range serviceNames {
unitPath := filepath.Join("/etc/systemd/system", svc+".service")
if _, err := os.Stat(unitPath); err == nil {
existing = append(existing, svc)
}
}
if len(existing) == 0 {
return nil, fmt.Errorf("no services found for alias %q", alias)
}
return existing, nil
}
// Check if it's already a full service name
unitPath := filepath.Join("/etc/systemd/system", alias+".service")
if _, err := os.Stat(unitPath); err == nil {
return []string{alias}, nil
}
// Try without .service suffix
if !strings.HasSuffix(alias, ".service") {
unitPath = filepath.Join("/etc/systemd/system", alias+".service")
if _, err := os.Stat(unitPath); err == nil {
return []string{alias}, nil
}
}
return nil, fmt.Errorf("service %q not found. Use: node, ipfs, cluster, gateway, olric, or full service name", alias)
}
// IsServiceActive checks if a systemd service is currently active (running)
func IsServiceActive(service string) (bool, error) {
cmd := exec.Command("systemctl", "is-active", "--quiet", service)
if err := cmd.Run(); err != nil {
if exitErr, ok := err.(*exec.ExitError); ok {
switch exitErr.ExitCode() {
case 3:
return false, nil
case 4:
return false, ErrServiceNotFound
}
}
return false, err
}
return true, nil
}
// IsServiceEnabled checks if a systemd service is enabled to start on boot
func IsServiceEnabled(service string) (bool, error) {
cmd := exec.Command("systemctl", "is-enabled", "--quiet", service)
if err := cmd.Run(); err != nil {
if exitErr, ok := err.(*exec.ExitError); ok {
switch exitErr.ExitCode() {
case 1:
return false, nil // Service is disabled
case 4:
return false, ErrServiceNotFound
}
}
return false, err
}
return true, nil
}
// IsServiceMasked checks if a systemd service is masked
func IsServiceMasked(service string) (bool, error) {
cmd := exec.Command("systemctl", "is-enabled", service)
output, err := cmd.CombinedOutput()
if err != nil {
outputStr := string(output)
if strings.Contains(outputStr, "masked") {
return true, nil
}
return false, err
}
return false, nil
}
// GetProductionServices returns a list of all DeBros production service names that exist
func GetProductionServices() []string {
// Unified service names (no bootstrap/node distinction)
allServices := []string{
"debros-gateway",
"debros-node",
"debros-olric",
"debros-ipfs-cluster",
"debros-ipfs",
"debros-anyone-client",
}
// Filter to only existing services by checking if unit file exists
var existing []string
for _, svc := range allServices {
unitPath := filepath.Join("/etc/systemd/system", svc+".service")
if _, err := os.Stat(unitPath); err == nil {
existing = append(existing, svc)
}
}
return existing
}
// CollectPortsForServices returns a list of ports used by the specified services
func CollectPortsForServices(services []string, skipActive bool) ([]PortSpec, error) {
seen := make(map[int]PortSpec)
for _, svc := range services {
if skipActive {
active, err := IsServiceActive(svc)
if err != nil {
return nil, fmt.Errorf("unable to check %s: %w", svc, err)
}
if active {
continue
}
}
for _, spec := range ServicePorts[svc] {
if _, ok := seen[spec.Port]; !ok {
seen[spec.Port] = spec
}
}
}
ports := make([]PortSpec, 0, len(seen))
for _, spec := range seen {
ports = append(ports, spec)
}
return ports, nil
}
// EnsurePortsAvailable checks if the specified ports are available
func EnsurePortsAvailable(action string, ports []PortSpec) error {
for _, spec := range ports {
ln, err := net.Listen("tcp", fmt.Sprintf("0.0.0.0:%d", spec.Port))
if err != nil {
if errors.Is(err, syscall.EADDRINUSE) || strings.Contains(err.Error(), "address already in use") {
return fmt.Errorf("%s cannot continue: %s (port %d) is already in use", action, spec.Name, spec.Port)
}
return fmt.Errorf("%s cannot continue: failed to inspect %s (port %d): %w", action, spec.Name, spec.Port, err)
}
_ = ln.Close()
}
return nil
}

113
pkg/cli/utils/validation.go Normal file
View File

@ -0,0 +1,113 @@
package utils
import (
"fmt"
"net"
"os"
"path/filepath"
"strings"
"github.com/DeBrosOfficial/network/pkg/config"
"github.com/multiformats/go-multiaddr"
)
// ValidateGeneratedConfig loads and validates the generated node configuration
func ValidateGeneratedConfig(oramaDir string) error {
configPath := filepath.Join(oramaDir, "configs", "node.yaml")
// Check if config file exists
if _, err := os.Stat(configPath); os.IsNotExist(err) {
return fmt.Errorf("configuration file not found at %s", configPath)
}
// Load the config file
file, err := os.Open(configPath)
if err != nil {
return fmt.Errorf("failed to open config file: %w", err)
}
defer file.Close()
var cfg config.Config
if err := config.DecodeStrict(file, &cfg); err != nil {
return fmt.Errorf("failed to parse config: %w", err)
}
// Validate the configuration
if errs := cfg.Validate(); len(errs) > 0 {
var errMsgs []string
for _, e := range errs {
errMsgs = append(errMsgs, e.Error())
}
return fmt.Errorf("configuration validation errors:\n - %s", strings.Join(errMsgs, "\n - "))
}
return nil
}
// ValidateDNSRecord validates that the domain points to the expected IP address
// Returns nil if DNS is valid, warning message if DNS doesn't match but continues,
// or error if DNS lookup fails completely
func ValidateDNSRecord(domain, expectedIP string) error {
if domain == "" {
return nil // No domain provided, skip validation
}
ips, err := net.LookupIP(domain)
if err != nil {
// DNS lookup failed - this is a warning, not a fatal error
// The user might be setting up DNS after installation
fmt.Printf(" ⚠️ DNS lookup failed for %s: %v\n", domain, err)
fmt.Printf(" Make sure DNS is configured before enabling HTTPS\n")
return nil
}
// Check if any resolved IP matches the expected IP
for _, ip := range ips {
if ip.String() == expectedIP {
fmt.Printf(" ✓ DNS validated: %s → %s\n", domain, expectedIP)
return nil
}
}
// DNS doesn't point to expected IP - warn but continue
resolvedIPs := make([]string, len(ips))
for i, ip := range ips {
resolvedIPs[i] = ip.String()
}
fmt.Printf(" ⚠️ DNS mismatch: %s resolves to %v, expected %s\n", domain, resolvedIPs, expectedIP)
fmt.Printf(" HTTPS certificate generation may fail until DNS is updated\n")
return nil
}
// NormalizePeers normalizes and validates peer multiaddrs
func NormalizePeers(peersStr string) ([]string, error) {
if peersStr == "" {
return nil, nil
}
// Split by comma and trim whitespace
rawPeers := strings.Split(peersStr, ",")
peers := make([]string, 0, len(rawPeers))
seen := make(map[string]bool)
for _, peer := range rawPeers {
peer = strings.TrimSpace(peer)
if peer == "" {
continue
}
// Validate multiaddr format
if _, err := multiaddr.NewMultiaddr(peer); err != nil {
return nil, fmt.Errorf("invalid multiaddr %q: %w", peer, err)
}
// Deduplicate
if !seen[peer] {
peers = append(peers, peer)
seen[peer] = true
}
}
return peers, nil
}

13
pkg/config/validate.go
View File

@ -1,6 +1,7 @@
package config
import (
"encoding/hex"
"fmt"
"net"
"os"
@ -585,3 +586,15 @@ func extractTCPPort(multiaddrStr string) string {
}
return ""
}
// ValidateSwarmKey validates that a swarm key is 64 hex characters
func ValidateSwarmKey(key string) error {
key = strings.TrimSpace(key)
if len(key) != 64 {
return fmt.Errorf("swarm key must be 64 hex characters (32 bytes), got %d", len(key))
}
if _, err := hex.DecodeString(key); err != nil {
return fmt.Errorf("swarm key must be valid hexadecimal: %w", err)
}
return nil
}

287
pkg/environments/development/ipfs.go Normal file
View File

@ -0,0 +1,287 @@
package development
import (
"context"
"encoding/json"
"fmt"
"net/http"
"net/url"
"os"
"os/exec"
"path/filepath"
"strings"
"time"
"github.com/DeBrosOfficial/network/pkg/tlsutil"
)
// ipfsNodeInfo holds information about an IPFS node for peer discovery
type ipfsNodeInfo struct {
name string
ipfsPath string
apiPort int
swarmPort int
gatewayPort int
peerID string
}
func (pm *ProcessManager) buildIPFSNodes(topology *Topology) []ipfsNodeInfo {
var nodes []ipfsNodeInfo
for _, nodeSpec := range topology.Nodes {
nodes = append(nodes, ipfsNodeInfo{
name: nodeSpec.Name,
ipfsPath: filepath.Join(pm.oramaDir, nodeSpec.DataDir, "ipfs/repo"),
apiPort: nodeSpec.IPFSAPIPort,
swarmPort: nodeSpec.IPFSSwarmPort,
gatewayPort: nodeSpec.IPFSGatewayPort,
peerID: "",
})
}
return nodes
}
func (pm *ProcessManager) startIPFS(ctx context.Context) error {
topology := DefaultTopology()
nodes := pm.buildIPFSNodes(topology)
for i := range nodes {
os.MkdirAll(nodes[i].ipfsPath, 0755)
if _, err := os.Stat(filepath.Join(nodes[i].ipfsPath, "config")); os.IsNotExist(err) {
fmt.Fprintf(pm.logWriter, " Initializing IPFS (%s)...\n", nodes[i].name)
cmd := exec.CommandContext(ctx, "ipfs", "init", "--profile=server", "--repo-dir="+nodes[i].ipfsPath)
if _, err := cmd.CombinedOutput(); err != nil {
fmt.Fprintf(pm.logWriter, " Warning: ipfs init failed: %v\n", err)
}
swarmKeyPath := filepath.Join(pm.oramaDir, "swarm.key")
if data, err := os.ReadFile(swarmKeyPath); err == nil {
os.WriteFile(filepath.Join(nodes[i].ipfsPath, "swarm.key"), data, 0600)
}
}
peerID, err := configureIPFSRepo(nodes[i].ipfsPath, nodes[i].apiPort, nodes[i].gatewayPort, nodes[i].swarmPort)
if err != nil {
fmt.Fprintf(pm.logWriter, " Warning: failed to configure IPFS repo for %s: %v\n", nodes[i].name, err)
} else {
nodes[i].peerID = peerID
fmt.Fprintf(pm.logWriter, " Peer ID for %s: %s\n", nodes[i].name, peerID)
}
}
for i := range nodes {
pidPath := filepath.Join(pm.pidsDir, fmt.Sprintf("ipfs-%s.pid", nodes[i].name))
logPath := filepath.Join(pm.oramaDir, "logs", fmt.Sprintf("ipfs-%s.log", nodes[i].name))
cmd := exec.CommandContext(ctx, "ipfs", "daemon", "--enable-pubsub-experiment", "--repo-dir="+nodes[i].ipfsPath)
logFile, _ := os.Create(logPath)
cmd.Stdout = logFile
cmd.Stderr = logFile
if err := cmd.Start(); err != nil {
return fmt.Errorf("failed to start ipfs-%s: %w", nodes[i].name, err)
}
os.WriteFile(pidPath, []byte(fmt.Sprintf("%d", cmd.Process.Pid)), 0644)
pm.processes[fmt.Sprintf("ipfs-%s", nodes[i].name)] = &ManagedProcess{
Name: fmt.Sprintf("ipfs-%s", nodes[i].name),
PID: cmd.Process.Pid,
StartTime: time.Now(),
LogPath: logPath,
}
fmt.Fprintf(pm.logWriter, "✓ IPFS (%s) started (PID: %d, API: %d, Swarm: %d)\n", nodes[i].name, cmd.Process.Pid, nodes[i].apiPort, nodes[i].swarmPort)
}
time.Sleep(2 * time.Second)
if err := pm.seedIPFSPeersWithHTTP(ctx, nodes); err != nil {
fmt.Fprintf(pm.logWriter, "⚠️ Failed to seed IPFS peers: %v\n", err)
}
return nil
}
func configureIPFSRepo(repoPath string, apiPort, gatewayPort, swarmPort int) (string, error) {
configPath := filepath.Join(repoPath, "config")
data, err := os.ReadFile(configPath)
if err != nil {
return "", fmt.Errorf("failed to read IPFS config: %w", err)
}
var config map[string]interface{}
if err := json.Unmarshal(data, &config); err != nil {
return "", fmt.Errorf("failed to parse IPFS config: %w", err)
}
config["Addresses"] = map[string]interface{}{
"API": []string{fmt.Sprintf("/ip4/127.0.0.1/tcp/%d", apiPort)},
"Gateway": []string{fmt.Sprintf("/ip4/127.0.0.1/tcp/%d", gatewayPort)},
"Swarm": []string{
fmt.Sprintf("/ip4/0.0.0.0/tcp/%d", swarmPort),
fmt.Sprintf("/ip6/::/tcp/%d", swarmPort),
},
}
config["AutoConf"] = map[string]interface{}{
"Enabled": false,
}
config["Bootstrap"] = []string{}
if dns, ok := config["DNS"].(map[string]interface{}); ok {
dns["Resolvers"] = map[string]interface{}{}
} else {
config["DNS"] = map[string]interface{}{
"Resolvers": map[string]interface{}{},
}
}
if routing, ok := config["Routing"].(map[string]interface{}); ok {
routing["DelegatedRouters"] = []string{}
} else {
config["Routing"] = map[string]interface{}{
"DelegatedRouters": []string{},
}
}
if ipns, ok := config["Ipns"].(map[string]interface{}); ok {
ipns["DelegatedPublishers"] = []string{}
} else {
config["Ipns"] = map[string]interface{}{
"DelegatedPublishers": []string{},
}
}
if api, ok := config["API"].(map[string]interface{}); ok {
api["HTTPHeaders"] = map[string][]string{
"Access-Control-Allow-Origin": {"*"},
"Access-Control-Allow-Methods": {"GET", "PUT", "POST", "DELETE", "OPTIONS"},
"Access-Control-Allow-Headers": {"Content-Type", "X-Requested-With"},
"Access-Control-Expose-Headers": {"Content-Length", "Content-Range"},
}
} else {
config["API"] = map[string]interface{}{
"HTTPHeaders": map[string][]string{
"Access-Control-Allow-Origin": {"*"},
"Access-Control-Allow-Methods": {"GET", "PUT", "POST", "DELETE", "OPTIONS"},
"Access-Control-Allow-Headers": {"Content-Type", "X-Requested-With"},
"Access-Control-Expose-Headers": {"Content-Length", "Content-Range"},
},
}
}
updatedData, err := json.MarshalIndent(config, "", " ")
if err != nil {
return "", fmt.Errorf("failed to marshal IPFS config: %w", err)
}
if err := os.WriteFile(configPath, updatedData, 0644); err != nil {
return "", fmt.Errorf("failed to write IPFS config: %w", err)
}
if id, ok := config["Identity"].(map[string]interface{}); ok {
if peerID, ok := id["PeerID"].(string); ok {
return peerID, nil
}
}
return "", fmt.Errorf("could not extract peer ID from config")
}
func (pm *ProcessManager) seedIPFSPeersWithHTTP(ctx context.Context, nodes []ipfsNodeInfo) error {
fmt.Fprintf(pm.logWriter, " Seeding IPFS local bootstrap peers via HTTP API...\n")
for _, node := range nodes {
if err := pm.waitIPFSReady(ctx, node); err != nil {
fmt.Fprintf(pm.logWriter, " Warning: failed to wait for IPFS readiness for %s: %v\n", node.name, err)
}
}
for i, node := range nodes {
httpURL := fmt.Sprintf("http://127.0.0.1:%d/api/v0/bootstrap/rm?all=true", node.apiPort)
if err := pm.ipfsHTTPCall(ctx, httpURL, "POST"); err != nil {
fmt.Fprintf(pm.logWriter, " Warning: failed to clear bootstrap for %s: %v\n", node.name, err)
}
for j, otherNode := range nodes {
if i == j {
continue
}
multiaddr := fmt.Sprintf("/ip4/127.0.0.1/tcp/%d/p2p/%s", otherNode.swarmPort, otherNode.peerID)
httpURL := fmt.Sprintf("http://127.0.0.1:%d/api/v0/bootstrap/add?arg=%s", node.apiPort, url.QueryEscape(multiaddr))
if err := pm.ipfsHTTPCall(ctx, httpURL, "POST"); err != nil {
fmt.Fprintf(pm.logWriter, " Warning: failed to add bootstrap peer for %s: %v\n", node.name, err)
}
}
}
return nil
}
func (pm *ProcessManager) waitIPFSReady(ctx context.Context, node ipfsNodeInfo) error {
maxRetries := 30
retryInterval := 500 * time.Millisecond
for attempt := 0; attempt < maxRetries; attempt++ {
httpURL := fmt.Sprintf("http://127.0.0.1:%d/api/v0/version", node.apiPort)
if err := pm.ipfsHTTPCall(ctx, httpURL, "POST"); err == nil {
return nil
}
select {
case <-time.After(retryInterval):
continue
case <-ctx.Done():
return ctx.Err()
}
}
return fmt.Errorf("IPFS daemon %s did not become ready", node.name)
}
func (pm *ProcessManager) ipfsHTTPCall(ctx context.Context, urlStr string, method string) error {
client := tlsutil.NewHTTPClient(5 * time.Second)
req, err := http.NewRequestWithContext(ctx, method, urlStr, nil)
if err != nil {
return fmt.Errorf("failed to create request: %w", err)
}
resp, err := client.Do(req)
if err != nil {
return fmt.Errorf("HTTP call failed: %w", err)
}
defer resp.Body.Close()
if resp.StatusCode >= 400 {
return fmt.Errorf("HTTP %d", resp.StatusCode)
}
return nil
}
func readIPFSConfigValue(ctx context.Context, repoPath string, key string) (string, error) {
configPath := filepath.Join(repoPath, "config")
data, err := os.ReadFile(configPath)
if err != nil {
return "", fmt.Errorf("failed to read IPFS config: %w", err)
}
lines := strings.Split(string(data), "\n")
for _, line := range lines {
line = strings.TrimSpace(line)
if strings.Contains(line, key) {
parts := strings.SplitN(line, ":", 2)
if len(parts) == 2 {
value := strings.TrimSpace(parts[1])
value = strings.Trim(value, `",`)
if value != "" {
return value, nil
}
}
}
}
return "", fmt.Errorf("key %s not found in IPFS config", key)
}

314
pkg/environments/development/ipfs_cluster.go Normal file
View File

@ -0,0 +1,314 @@
package development
import (
"context"
"encoding/json"
"fmt"
"net/http"
"os"
"os/exec"
"path/filepath"
"strings"
"time"
)
func (pm *ProcessManager) startIPFSCluster(ctx context.Context) error {
topology := DefaultTopology()
var nodes []struct {
name string
clusterPath string
restAPIPort int
clusterPort int
ipfsPort int
}
for _, nodeSpec := range topology.Nodes {
nodes = append(nodes, struct {
name string
clusterPath string
restAPIPort int
clusterPort int
ipfsPort int
}{
nodeSpec.Name,
filepath.Join(pm.oramaDir, nodeSpec.DataDir, "ipfs-cluster"),
nodeSpec.ClusterAPIPort,
nodeSpec.ClusterPort,
nodeSpec.IPFSAPIPort,
})
}
fmt.Fprintf(pm.logWriter, " Waiting for IPFS daemons to be ready...\n")
ipfsNodes := pm.buildIPFSNodes(topology)
for _, ipfsNode := range ipfsNodes {
if err := pm.waitIPFSReady(ctx, ipfsNode); err != nil {
fmt.Fprintf(pm.logWriter, " Warning: IPFS %s did not become ready: %v\n", ipfsNode.name, err)
}
}
secretPath := filepath.Join(pm.oramaDir, "cluster-secret")
clusterSecret, err := os.ReadFile(secretPath)
if err != nil {
return fmt.Errorf("failed to read cluster secret: %w", err)
}
clusterSecretHex := strings.TrimSpace(string(clusterSecret))
bootstrapMultiaddr := ""
{
node := nodes[0]
if err := pm.cleanClusterState(node.clusterPath); err != nil {
fmt.Fprintf(pm.logWriter, " Warning: failed to clean cluster state for %s: %v\n", node.name, err)
}
os.MkdirAll(node.clusterPath, 0755)
fmt.Fprintf(pm.logWriter, " Initializing IPFS Cluster (%s)...\n", node.name)
cmd := exec.CommandContext(ctx, "ipfs-cluster-service", "init", "--force")
cmd.Env = append(os.Environ(),
fmt.Sprintf("IPFS_CLUSTER_PATH=%s", node.clusterPath),
fmt.Sprintf("CLUSTER_SECRET=%s", clusterSecretHex),
)
if output, err := cmd.CombinedOutput(); err != nil {
fmt.Fprintf(pm.logWriter, " Warning: ipfs-cluster-service init failed: %v (output: %s)\n", err, string(output))
}
if err := pm.ensureIPFSClusterPorts(node.clusterPath, node.restAPIPort, node.clusterPort); err != nil {
fmt.Fprintf(pm.logWriter, " Warning: failed to update IPFS Cluster config for %s: %v\n", node.name, err)
}
pidPath := filepath.Join(pm.pidsDir, fmt.Sprintf("ipfs-cluster-%s.pid", node.name))
logPath := filepath.Join(pm.oramaDir, "logs", fmt.Sprintf("ipfs-cluster-%s.log", node.name))
cmd = exec.CommandContext(ctx, "ipfs-cluster-service", "daemon")
cmd.Env = append(os.Environ(), fmt.Sprintf("IPFS_CLUSTER_PATH=%s", node.clusterPath))
logFile, _ := os.Create(logPath)
cmd.Stdout = logFile
cmd.Stderr = logFile
if err := cmd.Start(); err != nil {
return err
}
os.WriteFile(pidPath, []byte(fmt.Sprintf("%d", cmd.Process.Pid)), 0644)
fmt.Fprintf(pm.logWriter, "✓ IPFS Cluster (%s) started (PID: %d, API: %d)\n", node.name, cmd.Process.Pid, node.restAPIPort)
if err := pm.waitClusterReady(ctx, node.name, node.restAPIPort); err != nil {
fmt.Fprintf(pm.logWriter, " Warning: IPFS Cluster %s did not become ready: %v\n", node.name, err)
}
time.Sleep(500 * time.Millisecond)
peerID, err := pm.waitForClusterPeerID(ctx, filepath.Join(node.clusterPath, "identity.json"))
if err != nil {
fmt.Fprintf(pm.logWriter, " Warning: failed to read bootstrap peer ID: %v\n", err)
} else {
bootstrapMultiaddr = fmt.Sprintf("/ip4/127.0.0.1/tcp/%d/p2p/%s", node.clusterPort, peerID)
}
}
for i := 1; i < len(nodes); i++ {
node := nodes[i]
if err := pm.cleanClusterState(node.clusterPath); err != nil {
fmt.Fprintf(pm.logWriter, " Warning: failed to clean cluster state for %s: %v\n", node.name, err)
}
os.MkdirAll(node.clusterPath, 0755)
fmt.Fprintf(pm.logWriter, " Initializing IPFS Cluster (%s)...\n", node.name)
cmd := exec.CommandContext(ctx, "ipfs-cluster-service", "init", "--force")
cmd.Env = append(os.Environ(),
fmt.Sprintf("IPFS_CLUSTER_PATH=%s", node.clusterPath),
fmt.Sprintf("CLUSTER_SECRET=%s", clusterSecretHex),
)
if output, err := cmd.CombinedOutput(); err != nil {
fmt.Fprintf(pm.logWriter, " Warning: ipfs-cluster-service init failed for %s: %v (output: %s)\n", node.name, err, string(output))
}
if err := pm.ensureIPFSClusterPorts(node.clusterPath, node.restAPIPort, node.clusterPort); err != nil {
fmt.Fprintf(pm.logWriter, " Warning: failed to update IPFS Cluster config for %s: %v\n", node.name, err)
}
pidPath := filepath.Join(pm.pidsDir, fmt.Sprintf("ipfs-cluster-%s.pid", node.name))
logPath := filepath.Join(pm.oramaDir, "logs", fmt.Sprintf("ipfs-cluster-%s.log", node.name))
args := []string{"daemon"}
if bootstrapMultiaddr != "" {
args = append(args, "--bootstrap", bootstrapMultiaddr)
}
cmd = exec.CommandContext(ctx, "ipfs-cluster-service", args...)
cmd.Env = append(os.Environ(), fmt.Sprintf("IPFS_CLUSTER_PATH=%s", node.clusterPath))
logFile, _ := os.Create(logPath)
cmd.Stdout = logFile
cmd.Stderr = logFile
if err := cmd.Start(); err != nil {
continue
}
os.WriteFile(pidPath, []byte(fmt.Sprintf("%d", cmd.Process.Pid)), 0644)
fmt.Fprintf(pm.logWriter, "✓ IPFS Cluster (%s) started (PID: %d, API: %d)\n", node.name, cmd.Process.Pid, node.restAPIPort)
if err := pm.waitClusterReady(ctx, node.name, node.restAPIPort); err != nil {
fmt.Fprintf(pm.logWriter, " Warning: IPFS Cluster %s did not become ready: %v\n", node.name, err)
}
}
fmt.Fprintf(pm.logWriter, " Waiting for IPFS Cluster peers to form...\n")
if err := pm.waitClusterFormed(ctx, nodes[0].restAPIPort); err != nil {
fmt.Fprintf(pm.logWriter, " Warning: IPFS Cluster did not form fully: %v\n", err)
}
time.Sleep(1 * time.Second)
return nil
}
func (pm *ProcessManager) waitForClusterPeerID(ctx context.Context, identityPath string) (string, error) {
maxRetries := 30
retryInterval := 500 * time.Millisecond
for attempt := 0; attempt < maxRetries; attempt++ {
data, err := os.ReadFile(identityPath)
if err == nil {
var identity map[string]interface{}
if err := json.Unmarshal(data, &identity); err == nil {
if id, ok := identity["id"].(string); ok {
return id, nil
}
}
}
select {
case <-time.After(retryInterval):
continue
case <-ctx.Done():
return "", ctx.Err()
}
}
return "", fmt.Errorf("could not read cluster peer ID")
}
func (pm *ProcessManager) waitClusterReady(ctx context.Context, name string, restAPIPort int) error {
maxRetries := 30
retryInterval := 500 * time.Millisecond
for attempt := 0; attempt < maxRetries; attempt++ {
httpURL := fmt.Sprintf("http://127.0.0.1:%d/peers", restAPIPort)
resp, err := http.Get(httpURL)
if err == nil && resp.StatusCode == 200 {
resp.Body.Close()
return nil
}
if resp != nil {
resp.Body.Close()
}
select {
case <-time.After(retryInterval):
continue
case <-ctx.Done():
return ctx.Err()
}
}
return fmt.Errorf("IPFS Cluster %s did not become ready", name)
}
func (pm *ProcessManager) waitClusterFormed(ctx context.Context, bootstrapRestAPIPort int) error {
maxRetries := 30
retryInterval := 1 * time.Second
requiredPeers := 3
for attempt := 0; attempt < maxRetries; attempt++ {
httpURL := fmt.Sprintf("http://127.0.0.1:%d/peers", bootstrapRestAPIPort)
resp, err := http.Get(httpURL)
if err == nil && resp.StatusCode == 200 {
dec := json.NewDecoder(resp.Body)
peerCount := 0
for {
var peer interface{}
if err := dec.Decode(&peer); err != nil {
break
}
peerCount++
}
resp.Body.Close()
if peerCount >= requiredPeers {
return nil
}
}
if resp != nil {
resp.Body.Close()
}
select {
case <-time.After(retryInterval):
continue
case <-ctx.Done():
return ctx.Err()
}
}
return fmt.Errorf("IPFS Cluster did not form fully")
}
func (pm *ProcessManager) cleanClusterState(clusterPath string) error {
pebblePath := filepath.Join(clusterPath, "pebble")
os.RemoveAll(pebblePath)
peerstorePath := filepath.Join(clusterPath, "peerstore")
os.Remove(peerstorePath)
serviceJSONPath := filepath.Join(clusterPath, "service.json")
os.Remove(serviceJSONPath)
lockPath := filepath.Join(clusterPath, "cluster.lock")
os.Remove(lockPath)
return nil
}
func (pm *ProcessManager) ensureIPFSClusterPorts(clusterPath string, restAPIPort int, clusterPort int) error {
serviceJSONPath := filepath.Join(clusterPath, "service.json")
data, err := os.ReadFile(serviceJSONPath)
if err != nil {
return err
}
var config map[string]interface{}
json.Unmarshal(data, &config)
portOffset := restAPIPort - 9094
proxyPort := 9095 + portOffset
pinsvcPort := 9097 + portOffset
ipfsPort := 4501 + (portOffset / 10)
if api, ok := config["api"].(map[string]interface{}); ok {
if restapi, ok := api["restapi"].(map[string]interface{}); ok {
restapi["http_listen_multiaddress"] = fmt.Sprintf("/ip4/0.0.0.0/tcp/%d", restAPIPort)
}
if proxy, ok := api["ipfsproxy"].(map[string]interface{}); ok {
proxy["listen_multiaddress"] = fmt.Sprintf("/ip4/127.0.0.1/tcp/%d", proxyPort)
proxy["node_multiaddress"] = fmt.Sprintf("/ip4/127.0.0.1/tcp/%d", ipfsPort)
}
if pinsvc, ok := api["pinsvcapi"].(map[string]interface{}); ok {
pinsvc["http_listen_multiaddress"] = fmt.Sprintf("/ip4/127.0.0.1/tcp/%d", pinsvcPort)
}
}
if cluster, ok := config["cluster"].(map[string]interface{}); ok {
cluster["listen_multiaddress"] = []string{
fmt.Sprintf("/ip4/0.0.0.0/tcp/%d", clusterPort),
fmt.Sprintf("/ip4/127.0.0.1/tcp/%d", clusterPort),
}
}
if connector, ok := config["ipfs_connector"].(map[string]interface{}); ok {
if ipfshttp, ok := connector["ipfshttp"].(map[string]interface{}); ok {
ipfshttp["node_multiaddress"] = fmt.Sprintf("/ip4/127.0.0.1/tcp/%d", ipfsPort)
}
}
updatedData, _ := json.MarshalIndent(config, "", " ")
return os.WriteFile(serviceJSONPath, updatedData, 0644)
}

206
pkg/environments/development/process.go Normal file
View File

@ -0,0 +1,206 @@
package development
import (
"context"
"fmt"
"os"
"os/exec"
"path/filepath"
"runtime"
"strconv"
"strings"
"time"
)
func (pm *ProcessManager) printStartupSummary(topology *Topology) {
fmt.Fprintf(pm.logWriter, "\n✅ Development environment ready!\n")
fmt.Fprintf(pm.logWriter, "═══════════════════════════════════════\n\n")
fmt.Fprintf(pm.logWriter, "📡 Access your nodes via unified gateway ports:\n\n")
for _, node := range topology.Nodes {
fmt.Fprintf(pm.logWriter, " %s:\n", node.Name)
fmt.Fprintf(pm.logWriter, " curl http://localhost:%d/health\n", node.UnifiedGatewayPort)
fmt.Fprintf(pm.logWriter, " curl http://localhost:%d/rqlite/http/db/execute\n", node.UnifiedGatewayPort)
fmt.Fprintf(pm.logWriter, " curl http://localhost:%d/cluster/health\n\n", node.UnifiedGatewayPort)
}
fmt.Fprintf(pm.logWriter, "🌐 Main Gateway:\n")
fmt.Fprintf(pm.logWriter, " curl http://localhost:%d/v1/status\n\n", topology.GatewayPort)
fmt.Fprintf(pm.logWriter, "📊 Other Services:\n")
fmt.Fprintf(pm.logWriter, " Olric: http://localhost:%d\n", topology.OlricHTTPPort)
fmt.Fprintf(pm.logWriter, " Anon SOCKS: 127.0.0.1:%d\n\n", topology.AnonSOCKSPort)
fmt.Fprintf(pm.logWriter, "📝 Useful Commands:\n")
fmt.Fprintf(pm.logWriter, " ./bin/orama dev status - Check service status\n")
fmt.Fprintf(pm.logWriter, " ./bin/orama dev logs node-1 - View logs\n")
fmt.Fprintf(pm.logWriter, " ./bin/orama dev down - Stop all services\n\n")
fmt.Fprintf(pm.logWriter, "📂 Logs: %s/logs\n", pm.oramaDir)
fmt.Fprintf(pm.logWriter, "⚙️ Config: %s\n\n", pm.oramaDir)
}
func (pm *ProcessManager) stopProcess(name string) error {
pidPath := filepath.Join(pm.pidsDir, fmt.Sprintf("%s.pid", name))
pidBytes, err := os.ReadFile(pidPath)
if err != nil {
return nil
}
pid, err := strconv.Atoi(strings.TrimSpace(string(pidBytes)))
if err != nil {
os.Remove(pidPath)
return nil
}
if !checkProcessRunning(pid) {
os.Remove(pidPath)
fmt.Fprintf(pm.logWriter, "✓ %s (not running)\n", name)
return nil
}
proc, err := os.FindProcess(pid)
if err != nil {
os.Remove(pidPath)
return nil
}
proc.Signal(os.Interrupt)
gracefulShutdown := false
for i := 0; i < 20; i++ {
time.Sleep(100 * time.Millisecond)
if !checkProcessRunning(pid) {
gracefulShutdown = true
break
}
}
if !gracefulShutdown && checkProcessRunning(pid) {
proc.Signal(os.Kill)
time.Sleep(200 * time.Millisecond)
if runtime.GOOS != "windows" {
exec.Command("pkill", "-9", "-P", fmt.Sprintf("%d", pid)).Run()
}
if checkProcessRunning(pid) {
exec.Command("kill", "-9", fmt.Sprintf("%d", pid)).Run()
time.Sleep(100 * time.Millisecond)
}
}
os.Remove(pidPath)
if gracefulShutdown {
fmt.Fprintf(pm.logWriter, "✓ %s stopped gracefully\n", name)
} else {
fmt.Fprintf(pm.logWriter, "✓ %s stopped (forced)\n", name)
}
return nil
}
func checkProcessRunning(pid int) bool {
proc, err := os.FindProcess(pid)
if err != nil {
return false
}
err = proc.Signal(os.Signal(nil))
return err == nil
}
func (pm *ProcessManager) startNode(name, configFile, logPath string) error {
pidPath := filepath.Join(pm.pidsDir, fmt.Sprintf("%s.pid", name))
cmd := exec.Command("./bin/orama-node", "--config", configFile)
logFile, _ := os.Create(logPath)
cmd.Stdout = logFile
cmd.Stderr = logFile
if err := cmd.Start(); err != nil {
return fmt.Errorf("failed to start %s: %w", name, err)
}
os.WriteFile(pidPath, []byte(fmt.Sprintf("%d", cmd.Process.Pid)), 0644)
fmt.Fprintf(pm.logWriter, "✓ %s started (PID: %d)\n", strings.Title(name), cmd.Process.Pid)
time.Sleep(1 * time.Second)
return nil
}
func (pm *ProcessManager) startGateway(ctx context.Context) error {
pidPath := filepath.Join(pm.pidsDir, "gateway.pid")
logPath := filepath.Join(pm.oramaDir, "logs", "gateway.log")
cmd := exec.Command("./bin/gateway", "--config", "gateway.yaml")
logFile, _ := os.Create(logPath)
cmd.Stdout = logFile
cmd.Stderr = logFile
if err := cmd.Start(); err != nil {
return fmt.Errorf("failed to start gateway: %w", err)
}
os.WriteFile(pidPath, []byte(fmt.Sprintf("%d", cmd.Process.Pid)), 0644)
fmt.Fprintf(pm.logWriter, "✓ Gateway started (PID: %d, listen: 6001)\n", cmd.Process.Pid)
return nil
}
func (pm *ProcessManager) startOlric(ctx context.Context) error {
pidPath := filepath.Join(pm.pidsDir, "olric.pid")
logPath := filepath.Join(pm.oramaDir, "logs", "olric.log")
configPath := filepath.Join(pm.oramaDir, "olric-config.yaml")
cmd := exec.CommandContext(ctx, "olric-server")
cmd.Env = append(os.Environ(), fmt.Sprintf("OLRIC_SERVER_CONFIG=%s", configPath))
logFile, _ := os.Create(logPath)
cmd.Stdout = logFile
cmd.Stderr = logFile
if err := cmd.Start(); err != nil {
return fmt.Errorf("failed to start olric: %w", err)
}
os.WriteFile(pidPath, []byte(fmt.Sprintf("%d", cmd.Process.Pid)), 0644)
fmt.Fprintf(pm.logWriter, "✓ Olric started (PID: %d)\n", cmd.Process.Pid)
time.Sleep(1 * time.Second)
return nil
}
func (pm *ProcessManager) startAnon(ctx context.Context) error {
if runtime.GOOS != "darwin" {
return nil
}
pidPath := filepath.Join(pm.pidsDir, "anon.pid")
logPath := filepath.Join(pm.oramaDir, "logs", "anon.log")
cmd := exec.CommandContext(ctx, "npx", "anyone-client")
logFile, _ := os.Create(logPath)
cmd.Stdout = logFile
cmd.Stderr = logFile
if err := cmd.Start(); err != nil {
fmt.Fprintf(pm.logWriter, " ⚠️ Failed to start Anon: %v\n", err)
return nil
}
os.WriteFile(pidPath, []byte(fmt.Sprintf("%d", cmd.Process.Pid)), 0644)
fmt.Fprintf(pm.logWriter, "✓ Anon proxy started (PID: %d, SOCKS: 9050)\n", cmd.Process.Pid)
return nil
}
func (pm *ProcessManager) startNodes(ctx context.Context) error {
topology := DefaultTopology()
for _, nodeSpec := range topology.Nodes {
logPath := filepath.Join(pm.oramaDir, "logs", fmt.Sprintf("%s.log", nodeSpec.Name))
if err := pm.startNode(nodeSpec.Name, nodeSpec.ConfigFilename, logPath); err != nil {
return fmt.Errorf("failed to start %s: %w", nodeSpec.Name, err)
}
time.Sleep(500 * time.Millisecond)
}
return nil
}

931
pkg/environments/development/runner.go
View File

@ -2,21 +2,12 @@ package development
import (
"context"
"encoding/json"
"fmt"
"io"
"net/http"
"net/url"
"os"
"os/exec"
"path/filepath"
"runtime"
"strconv"
"strings"
"sync"
"time"
"github.com/DeBrosOfficial/network/pkg/tlsutil"
)
// ProcessManager manages all dev environment processes
@ -69,13 +60,11 @@ func (pm *ProcessManager) StartAll(ctx context.Context) error {
{"Olric", pm.startOlric},
{"Anon", pm.startAnon},
{"Nodes (Network)", pm.startNodes},
// Gateway is now per-node (embedded in each node) - no separate main gateway needed
}
for _, svc := range services {
if err := svc.fn(ctx); err != nil {
fmt.Fprintf(pm.logWriter, "⚠️ Failed to start %s: %v\n", svc.name, err)
// Continue starting others, don't fail
}
}
@ -99,35 +88,6 @@ func (pm *ProcessManager) StartAll(ctx context.Context) error {
return nil
}
// printStartupSummary prints the final startup summary with key endpoints
func (pm *ProcessManager) printStartupSummary(topology *Topology) {
fmt.Fprintf(pm.logWriter, "\n✅ Development environment ready!\n")
fmt.Fprintf(pm.logWriter, "═══════════════════════════════════════\n\n")
fmt.Fprintf(pm.logWriter, "📡 Access your nodes via unified gateway ports:\n\n")
for _, node := range topology.Nodes {
fmt.Fprintf(pm.logWriter, " %s:\n", node.Name)
fmt.Fprintf(pm.logWriter, " curl http://localhost:%d/health\n", node.UnifiedGatewayPort)
fmt.Fprintf(pm.logWriter, " curl http://localhost:%d/rqlite/http/db/execute\n", node.UnifiedGatewayPort)
fmt.Fprintf(pm.logWriter, " curl http://localhost:%d/cluster/health\n\n", node.UnifiedGatewayPort)
}
fmt.Fprintf(pm.logWriter, "🌐 Main Gateway:\n")
fmt.Fprintf(pm.logWriter, " curl http://localhost:%d/v1/status\n\n", topology.GatewayPort)
fmt.Fprintf(pm.logWriter, "📊 Other Services:\n")
fmt.Fprintf(pm.logWriter, " Olric: http://localhost:%d\n", topology.OlricHTTPPort)
fmt.Fprintf(pm.logWriter, " Anon SOCKS: 127.0.0.1:%d\n\n", topology.AnonSOCKSPort)
fmt.Fprintf(pm.logWriter, "📝 Useful Commands:\n")
fmt.Fprintf(pm.logWriter, " ./bin/orama dev status - Check service status\n")
fmt.Fprintf(pm.logWriter, " ./bin/orama dev logs node-1 - View logs\n")
fmt.Fprintf(pm.logWriter, " ./bin/orama dev down - Stop all services\n\n")
fmt.Fprintf(pm.logWriter, "📂 Logs: %s/logs\n", pm.oramaDir)
fmt.Fprintf(pm.logWriter, "⚙️ Config: %s\n\n", pm.oramaDir)
}
// StopAll stops all running processes
func (pm *ProcessManager) StopAll(ctx context.Context) error {
fmt.Fprintf(pm.logWriter, "\n🛑 Stopping development environment...\n\n")
@ -153,7 +113,6 @@ func (pm *ProcessManager) StopAll(ctx context.Context) error {
fmt.Fprintf(pm.logWriter, "Stopping %d services...\n\n", len(services))
// Stop all processes sequentially (in dependency order) and wait for each
stoppedCount := 0
for _, svc := range services {
if err := pm.stopProcess(svc); err != nil {
@ -161,8 +120,6 @@ func (pm *ProcessManager) StopAll(ctx context.Context) error {
} else {
stoppedCount++
}
// Show progress
fmt.Fprintf(pm.logWriter, " [%d/%d] stopped\n", stoppedCount, len(services))
}
@ -224,7 +181,8 @@ func (pm *ProcessManager) Status(ctx context.Context) {
pidPath := filepath.Join(pm.pidsDir, fmt.Sprintf("%s.pid", svc.name))
running := false
if pidBytes, err := os.ReadFile(pidPath); err == nil {
pid, _ := strconv.Atoi(string(pidBytes))
var pid int
fmt.Sscanf(string(pidBytes), "%d", &pid)
if checkProcessRunning(pid) {
running = true
}
@ -252,888 +210,3 @@ func (pm *ProcessManager) Status(ctx context.Context) {
fmt.Fprintf(pm.logWriter, "\nLogs directory: %s/logs\n\n", pm.oramaDir)
}
// Helper functions for starting individual services
// buildIPFSNodes constructs ipfsNodeInfo from topology
func (pm *ProcessManager) buildIPFSNodes(topology *Topology) []ipfsNodeInfo {
var nodes []ipfsNodeInfo
for _, nodeSpec := range topology.Nodes {
nodes = append(nodes, ipfsNodeInfo{
name: nodeSpec.Name,
ipfsPath: filepath.Join(pm.oramaDir, nodeSpec.DataDir, "ipfs/repo"),
apiPort: nodeSpec.IPFSAPIPort,
swarmPort: nodeSpec.IPFSSwarmPort,
gatewayPort: nodeSpec.IPFSGatewayPort,
peerID: "",
})
}
return nodes
}
// startNodes starts all network nodes
func (pm *ProcessManager) startNodes(ctx context.Context) error {
topology := DefaultTopology()
for _, nodeSpec := range topology.Nodes {
logPath := filepath.Join(pm.oramaDir, "logs", fmt.Sprintf("%s.log", nodeSpec.Name))
if err := pm.startNode(nodeSpec.Name, nodeSpec.ConfigFilename, logPath); err != nil {
return fmt.Errorf("failed to start %s: %w", nodeSpec.Name, err)
}
time.Sleep(500 * time.Millisecond)
}
return nil
}
// ipfsNodeInfo holds information about an IPFS node for peer discovery
type ipfsNodeInfo struct {
name string
ipfsPath string
apiPort int
swarmPort int
gatewayPort int
peerID string
}
// readIPFSConfigValue reads a single config value from IPFS repo without daemon running
func readIPFSConfigValue(ctx context.Context, repoPath string, key string) (string, error) {
configPath := filepath.Join(repoPath, "config")
data, err := os.ReadFile(configPath)
if err != nil {
return "", fmt.Errorf("failed to read IPFS config: %w", err)
}
// Simple JSON parse to extract the value - only works for string values
lines := strings.Split(string(data), "\n")
for _, line := range lines {
line = strings.TrimSpace(line)
if strings.Contains(line, key) {
// Extract the value after the colon
parts := strings.SplitN(line, ":", 2)
if len(parts) == 2 {
value := strings.TrimSpace(parts[1])
value = strings.Trim(value, `",`)
if value != "" {
return value, nil
}
}
}
}
return "", fmt.Errorf("key %s not found in IPFS config", key)
}
// configureIPFSRepo directly modifies IPFS config JSON to set addresses, bootstrap, and CORS headers
// This avoids shell commands which fail on some systems and instead manipulates the config directly
// Returns the peer ID from the config
func configureIPFSRepo(repoPath string, apiPort, gatewayPort, swarmPort int) (string, error) {
configPath := filepath.Join(repoPath, "config")
// Read existing config
data, err := os.ReadFile(configPath)
if err != nil {
return "", fmt.Errorf("failed to read IPFS config: %w", err)
}
var config map[string]interface{}
if err := json.Unmarshal(data, &config); err != nil {
return "", fmt.Errorf("failed to parse IPFS config: %w", err)
}
// Set Addresses
config["Addresses"] = map[string]interface{}{
"API": []string{fmt.Sprintf("/ip4/127.0.0.1/tcp/%d", apiPort)},
"Gateway": []string{fmt.Sprintf("/ip4/127.0.0.1/tcp/%d", gatewayPort)},
"Swarm": []string{
fmt.Sprintf("/ip4/0.0.0.0/tcp/%d", swarmPort),
fmt.Sprintf("/ip6/::/tcp/%d", swarmPort),
},
}
// Disable AutoConf for private swarm
config["AutoConf"] = map[string]interface{}{
"Enabled": false,
}
// Clear Bootstrap (will be set via HTTP API after startup)
config["Bootstrap"] = []string{}
// Clear DNS Resolvers
if dns, ok := config["DNS"].(map[string]interface{}); ok {
dns["Resolvers"] = map[string]interface{}{}
} else {
config["DNS"] = map[string]interface{}{
"Resolvers": map[string]interface{}{},
}
}
// Clear Routing DelegatedRouters
if routing, ok := config["Routing"].(map[string]interface{}); ok {
routing["DelegatedRouters"] = []string{}
} else {
config["Routing"] = map[string]interface{}{
"DelegatedRouters": []string{},
}
}
// Clear IPNS DelegatedPublishers
if ipns, ok := config["Ipns"].(map[string]interface{}); ok {
ipns["DelegatedPublishers"] = []string{}
} else {
config["Ipns"] = map[string]interface{}{
"DelegatedPublishers": []string{},
}
}
// Set API HTTPHeaders with CORS (must be map[string][]string)
if api, ok := config["API"].(map[string]interface{}); ok {
api["HTTPHeaders"] = map[string][]string{
"Access-Control-Allow-Origin": {"*"},
"Access-Control-Allow-Methods": {"GET", "PUT", "POST", "DELETE", "OPTIONS"},
"Access-Control-Allow-Headers": {"Content-Type", "X-Requested-With"},
"Access-Control-Expose-Headers": {"Content-Length", "Content-Range"},
}
} else {
config["API"] = map[string]interface{}{
"HTTPHeaders": map[string][]string{
"Access-Control-Allow-Origin": {"*"},
"Access-Control-Allow-Methods": {"GET", "PUT", "POST", "DELETE", "OPTIONS"},
"Access-Control-Allow-Headers": {"Content-Type", "X-Requested-With"},
"Access-Control-Expose-Headers": {"Content-Length", "Content-Range"},
},
}
}
// Write config back
updatedData, err := json.MarshalIndent(config, "", " ")
if err != nil {
return "", fmt.Errorf("failed to marshal IPFS config: %w", err)
}
if err := os.WriteFile(configPath, updatedData, 0644); err != nil {
return "", fmt.Errorf("failed to write IPFS config: %w", err)
}
// Extract and return peer ID
if id, ok := config["Identity"].(map[string]interface{}); ok {
if peerID, ok := id["PeerID"].(string); ok {
return peerID, nil
}
}
return "", fmt.Errorf("could not extract peer ID from config")
}
// seedIPFSPeersWithHTTP configures each IPFS node to bootstrap with its local peers using HTTP API
func (pm *ProcessManager) seedIPFSPeersWithHTTP(ctx context.Context, nodes []ipfsNodeInfo) error {
fmt.Fprintf(pm.logWriter, " Seeding IPFS local bootstrap peers via HTTP API...\n")
// Wait for all IPFS daemons to be ready before trying to configure them
for _, node := range nodes {
if err := pm.waitIPFSReady(ctx, node); err != nil {
fmt.Fprintf(pm.logWriter, " Warning: failed to wait for IPFS readiness for %s: %v\n", node.name, err)
}
}
// For each node, clear default bootstrap and add local peers via HTTP
for i, node := range nodes {
// Clear bootstrap peers
httpURL := fmt.Sprintf("http://127.0.0.1:%d/api/v0/bootstrap/rm?all=true", node.apiPort)
if err := pm.ipfsHTTPCall(ctx, httpURL, "POST"); err != nil {
fmt.Fprintf(pm.logWriter, " Warning: failed to clear bootstrap for %s: %v\n", node.name, err)
}
// Add other nodes as bootstrap peers
for j, otherNode := range nodes {
if i == j {
continue // Skip self
}
multiaddr := fmt.Sprintf("/ip4/127.0.0.1/tcp/%d/p2p/%s", otherNode.swarmPort, otherNode.peerID)
httpURL := fmt.Sprintf("http://127.0.0.1:%d/api/v0/bootstrap/add?arg=%s", node.apiPort, url.QueryEscape(multiaddr))
if err := pm.ipfsHTTPCall(ctx, httpURL, "POST"); err != nil {
fmt.Fprintf(pm.logWriter, " Warning: failed to add bootstrap peer for %s: %v\n", node.name, err)
}
}
}
return nil
}
// waitIPFSReady polls the IPFS daemon's HTTP API until it's ready
func (pm *ProcessManager) waitIPFSReady(ctx context.Context, node ipfsNodeInfo) error {
maxRetries := 30
retryInterval := 500 * time.Millisecond
for attempt := 0; attempt < maxRetries; attempt++ {
httpURL := fmt.Sprintf("http://127.0.0.1:%d/api/v0/version", node.apiPort)
if err := pm.ipfsHTTPCall(ctx, httpURL, "POST"); err == nil {
return nil // IPFS is ready
}
select {
case <-time.After(retryInterval):
continue
case <-ctx.Done():
return ctx.Err()
}
}
return fmt.Errorf("IPFS daemon %s did not become ready after %d seconds", node.name, (maxRetries * int(retryInterval.Seconds())))
}
// ipfsHTTPCall makes an HTTP call to IPFS API
func (pm *ProcessManager) ipfsHTTPCall(ctx context.Context, urlStr string, method string) error {
client := tlsutil.NewHTTPClient(5 * time.Second)
req, err := http.NewRequestWithContext(ctx, method, urlStr, nil)
if err != nil {
return fmt.Errorf("failed to create request: %w", err)
}
resp, err := client.Do(req)
if err != nil {
return fmt.Errorf("HTTP call failed: %w", err)
}
defer resp.Body.Close()
if resp.StatusCode >= 400 {
body, _ := io.ReadAll(resp.Body)
return fmt.Errorf("HTTP %d: %s", resp.StatusCode, string(body))
}
return nil
}
func (pm *ProcessManager) startIPFS(ctx context.Context) error {
topology := DefaultTopology()
nodes := pm.buildIPFSNodes(topology)
// Phase 1: Initialize repos and configure addresses
for i := range nodes {
os.MkdirAll(nodes[i].ipfsPath, 0755)
// Initialize IPFS if needed
if _, err := os.Stat(filepath.Join(nodes[i].ipfsPath, "config")); os.IsNotExist(err) {
fmt.Fprintf(pm.logWriter, " Initializing IPFS (%s)...\n", nodes[i].name)
cmd := exec.CommandContext(ctx, "ipfs", "init", "--profile=server", "--repo-dir="+nodes[i].ipfsPath)
if _, err := cmd.CombinedOutput(); err != nil {
fmt.Fprintf(pm.logWriter, " Warning: ipfs init failed: %v\n", err)
}
// Copy swarm key
swarmKeyPath := filepath.Join(pm.oramaDir, "swarm.key")
if data, err := os.ReadFile(swarmKeyPath); err == nil {
os.WriteFile(filepath.Join(nodes[i].ipfsPath, "swarm.key"), data, 0600)
}
}
// Configure the IPFS config directly (addresses, bootstrap, DNS, routing, CORS headers)
// This replaces shell commands which can fail on some systems
peerID, err := configureIPFSRepo(nodes[i].ipfsPath, nodes[i].apiPort, nodes[i].gatewayPort, nodes[i].swarmPort)
if err != nil {
fmt.Fprintf(pm.logWriter, " Warning: failed to configure IPFS repo for %s: %v\n", nodes[i].name, err)
} else {
nodes[i].peerID = peerID
fmt.Fprintf(pm.logWriter, " Peer ID for %s: %s\n", nodes[i].name, peerID)
}
}
// Phase 2: Start all IPFS daemons
for i := range nodes {
pidPath := filepath.Join(pm.pidsDir, fmt.Sprintf("ipfs-%s.pid", nodes[i].name))
logPath := filepath.Join(pm.oramaDir, "logs", fmt.Sprintf("ipfs-%s.log", nodes[i].name))
cmd := exec.CommandContext(ctx, "ipfs", "daemon", "--enable-pubsub-experiment", "--repo-dir="+nodes[i].ipfsPath)
logFile, _ := os.Create(logPath)
cmd.Stdout = logFile
cmd.Stderr = logFile
if err := cmd.Start(); err != nil {
return fmt.Errorf("failed to start ipfs-%s: %w", nodes[i].name, err)
}
os.WriteFile(pidPath, []byte(fmt.Sprintf("%d", cmd.Process.Pid)), 0644)
pm.processes[fmt.Sprintf("ipfs-%s", nodes[i].name)] = &ManagedProcess{
Name: fmt.Sprintf("ipfs-%s", nodes[i].name),
PID: cmd.Process.Pid,
StartTime: time.Now(),
LogPath: logPath,
}
fmt.Fprintf(pm.logWriter, "✓ IPFS (%s) started (PID: %d, API: %d, Swarm: %d)\n", nodes[i].name, cmd.Process.Pid, nodes[i].apiPort, nodes[i].swarmPort)
}
time.Sleep(2 * time.Second)
// Phase 3: Seed IPFS peers via HTTP API after all daemons are running
if err := pm.seedIPFSPeersWithHTTP(ctx, nodes); err != nil {
fmt.Fprintf(pm.logWriter, "⚠️ Failed to seed IPFS peers: %v\n", err)
}
return nil
}
func (pm *ProcessManager) startIPFSCluster(ctx context.Context) error {
topology := DefaultTopology()
var nodes []struct {
name string
clusterPath string
restAPIPort int
clusterPort int
ipfsPort int
}
for _, nodeSpec := range topology.Nodes {
nodes = append(nodes, struct {
name string
clusterPath string
restAPIPort int
clusterPort int
ipfsPort int
}{
nodeSpec.Name,
filepath.Join(pm.oramaDir, nodeSpec.DataDir, "ipfs-cluster"),
nodeSpec.ClusterAPIPort,
nodeSpec.ClusterPort,
nodeSpec.IPFSAPIPort,
})
}
// Wait for all IPFS daemons to be ready before starting cluster services
fmt.Fprintf(pm.logWriter, " Waiting for IPFS daemons to be ready...\n")
ipfsNodes := pm.buildIPFSNodes(topology)
for _, ipfsNode := range ipfsNodes {
if err := pm.waitIPFSReady(ctx, ipfsNode); err != nil {
fmt.Fprintf(pm.logWriter, " Warning: IPFS %s did not become ready: %v\n", ipfsNode.name, err)
}
}
// Read cluster secret to ensure all nodes use the same PSK
secretPath := filepath.Join(pm.oramaDir, "cluster-secret")
clusterSecret, err := os.ReadFile(secretPath)
if err != nil {
return fmt.Errorf("failed to read cluster secret: %w", err)
}
clusterSecretHex := strings.TrimSpace(string(clusterSecret))
// Phase 1: Initialize and start bootstrap IPFS Cluster, then read its identity
bootstrapMultiaddr := ""
{
node := nodes[0] // bootstrap
// Always clean stale cluster state to ensure fresh initialization with correct secret
if err := pm.cleanClusterState(node.clusterPath); err != nil {
fmt.Fprintf(pm.logWriter, " Warning: failed to clean cluster state for %s: %v\n", node.name, err)
}
os.MkdirAll(node.clusterPath, 0755)
fmt.Fprintf(pm.logWriter, " Initializing IPFS Cluster (%s)...\n", node.name)
cmd := exec.CommandContext(ctx, "ipfs-cluster-service", "init", "--force")
cmd.Env = append(os.Environ(),
fmt.Sprintf("IPFS_CLUSTER_PATH=%s", node.clusterPath),
fmt.Sprintf("CLUSTER_SECRET=%s", clusterSecretHex),
)
if output, err := cmd.CombinedOutput(); err != nil {
fmt.Fprintf(pm.logWriter, " Warning: ipfs-cluster-service init failed: %v (output: %s)\n", err, string(output))
}
// Ensure correct ports in service.json BEFORE starting daemon
// This is critical: it sets the cluster listen port to clusterPort, not the default
if err := pm.ensureIPFSClusterPorts(node.clusterPath, node.restAPIPort, node.clusterPort); err != nil {
fmt.Fprintf(pm.logWriter, " Warning: failed to update IPFS Cluster config for %s: %v\n", node.name, err)
}
// Verify the config was written correctly (debug: read it back)
serviceJSONPath := filepath.Join(node.clusterPath, "service.json")
if data, err := os.ReadFile(serviceJSONPath); err == nil {
var verifyConfig map[string]interface{}
if err := json.Unmarshal(data, &verifyConfig); err == nil {
if cluster, ok := verifyConfig["cluster"].(map[string]interface{}); ok {
if listenAddrs, ok := cluster["listen_multiaddress"].([]interface{}); ok {
fmt.Fprintf(pm.logWriter, " Config verified: %s cluster listening on %v\n", node.name, listenAddrs)
}
}
}
}
// Start bootstrap cluster service
pidPath := filepath.Join(pm.pidsDir, fmt.Sprintf("ipfs-cluster-%s.pid", node.name))
logPath := filepath.Join(pm.oramaDir, "logs", fmt.Sprintf("ipfs-cluster-%s.log", node.name))
cmd = exec.CommandContext(ctx, "ipfs-cluster-service", "daemon")
cmd.Env = append(os.Environ(), fmt.Sprintf("IPFS_CLUSTER_PATH=%s", node.clusterPath))
logFile, _ := os.Create(logPath)
cmd.Stdout = logFile
cmd.Stderr = logFile
if err := cmd.Start(); err != nil {
fmt.Fprintf(pm.logWriter, " ⚠️ Failed to start ipfs-cluster-%s: %v\n", node.name, err)
return err
}
os.WriteFile(pidPath, []byte(fmt.Sprintf("%d", cmd.Process.Pid)), 0644)
fmt.Fprintf(pm.logWriter, "✓ IPFS Cluster (%s) started (PID: %d, API: %d)\n", node.name, cmd.Process.Pid, node.restAPIPort)
// Wait for bootstrap to be ready and read its identity
if err := pm.waitClusterReady(ctx, node.name, node.restAPIPort); err != nil {
fmt.Fprintf(pm.logWriter, " Warning: IPFS Cluster %s did not become ready: %v\n", node.name, err)
}
// Add a brief delay to allow identity.json to be written
time.Sleep(500 * time.Millisecond)
// Read bootstrap peer ID for follower nodes to join
peerID, err := pm.waitForClusterPeerID(ctx, filepath.Join(node.clusterPath, "identity.json"))
if err != nil {
fmt.Fprintf(pm.logWriter, " Warning: failed to read bootstrap peer ID: %v\n", err)
} else {
bootstrapMultiaddr = fmt.Sprintf("/ip4/127.0.0.1/tcp/%d/p2p/%s", node.clusterPort, peerID)
fmt.Fprintf(pm.logWriter, " Bootstrap multiaddress: %s\n", bootstrapMultiaddr)
}
}
// Phase 2: Initialize and start follower IPFS Cluster nodes with bootstrap flag
for i := 1; i < len(nodes); i++ {
node := nodes[i]
// Always clean stale cluster state to ensure fresh initialization with correct secret
if err := pm.cleanClusterState(node.clusterPath); err != nil {
fmt.Fprintf(pm.logWriter, " Warning: failed to clean cluster state for %s: %v\n", node.name, err)
}
os.MkdirAll(node.clusterPath, 0755)
fmt.Fprintf(pm.logWriter, " Initializing IPFS Cluster (%s)...\n", node.name)
cmd := exec.CommandContext(ctx, "ipfs-cluster-service", "init", "--force")
cmd.Env = append(os.Environ(),
fmt.Sprintf("IPFS_CLUSTER_PATH=%s", node.clusterPath),
fmt.Sprintf("CLUSTER_SECRET=%s", clusterSecretHex),
)
if output, err := cmd.CombinedOutput(); err != nil {
fmt.Fprintf(pm.logWriter, " Warning: ipfs-cluster-service init failed for %s: %v (output: %s)\n", node.name, err, string(output))
}
// Ensure correct ports in service.json BEFORE starting daemon
if err := pm.ensureIPFSClusterPorts(node.clusterPath, node.restAPIPort, node.clusterPort); err != nil {
fmt.Fprintf(pm.logWriter, " Warning: failed to update IPFS Cluster config for %s: %v\n", node.name, err)
}
// Verify the config was written correctly (debug: read it back)
serviceJSONPath := filepath.Join(node.clusterPath, "service.json")
if data, err := os.ReadFile(serviceJSONPath); err == nil {
var verifyConfig map[string]interface{}
if err := json.Unmarshal(data, &verifyConfig); err == nil {
if cluster, ok := verifyConfig["cluster"].(map[string]interface{}); ok {
if listenAddrs, ok := cluster["listen_multiaddress"].([]interface{}); ok {
fmt.Fprintf(pm.logWriter, " Config verified: %s cluster listening on %v\n", node.name, listenAddrs)
}
}
}
}
// Start follower cluster service with bootstrap flag
pidPath := filepath.Join(pm.pidsDir, fmt.Sprintf("ipfs-cluster-%s.pid", node.name))
logPath := filepath.Join(pm.oramaDir, "logs", fmt.Sprintf("ipfs-cluster-%s.log", node.name))
args := []string{"daemon"}
if bootstrapMultiaddr != "" {
args = append(args, "--bootstrap", bootstrapMultiaddr)
}
cmd = exec.CommandContext(ctx, "ipfs-cluster-service", args...)
cmd.Env = append(os.Environ(), fmt.Sprintf("IPFS_CLUSTER_PATH=%s", node.clusterPath))
logFile, _ := os.Create(logPath)
cmd.Stdout = logFile
cmd.Stderr = logFile
if err := cmd.Start(); err != nil {
fmt.Fprintf(pm.logWriter, " ⚠️ Failed to start ipfs-cluster-%s: %v\n", node.name, err)
continue
}
os.WriteFile(pidPath, []byte(fmt.Sprintf("%d", cmd.Process.Pid)), 0644)
fmt.Fprintf(pm.logWriter, "✓ IPFS Cluster (%s) started (PID: %d, API: %d)\n", node.name, cmd.Process.Pid, node.restAPIPort)
// Wait for follower node to connect to the bootstrap peer
if err := pm.waitClusterReady(ctx, node.name, node.restAPIPort); err != nil {
fmt.Fprintf(pm.logWriter, " Warning: IPFS Cluster %s did not become ready: %v\n", node.name, err)
}
}
// Phase 3: Wait for all cluster peers to discover each other
fmt.Fprintf(pm.logWriter, " Waiting for IPFS Cluster peers to form...\n")
if err := pm.waitClusterFormed(ctx, nodes[0].restAPIPort); err != nil {
fmt.Fprintf(pm.logWriter, " Warning: IPFS Cluster did not form fully: %v\n", err)
}
time.Sleep(1 * time.Second)
return nil
}
// waitForClusterPeerID polls the identity.json file until it appears and extracts the peer ID
func (pm *ProcessManager) waitForClusterPeerID(ctx context.Context, identityPath string) (string, error) {
maxRetries := 30
retryInterval := 500 * time.Millisecond
for attempt := 0; attempt < maxRetries; attempt++ {
data, err := os.ReadFile(identityPath)
if err == nil {
var identity map[string]interface{}
if err := json.Unmarshal(data, &identity); err == nil {
if id, ok := identity["id"].(string); ok {
return id, nil
}
}
}
select {
case <-time.After(retryInterval):
continue
case <-ctx.Done():
return "", ctx.Err()
}
}
return "", fmt.Errorf("could not read cluster peer ID after %d seconds", (maxRetries * int(retryInterval.Milliseconds()) / 1000))
}
// waitClusterReady polls the cluster REST API until it's ready
func (pm *ProcessManager) waitClusterReady(ctx context.Context, name string, restAPIPort int) error {
maxRetries := 30
retryInterval := 500 * time.Millisecond
for attempt := 0; attempt < maxRetries; attempt++ {
httpURL := fmt.Sprintf("http://127.0.0.1:%d/peers", restAPIPort)
resp, err := http.Get(httpURL)
if err == nil && resp.StatusCode == 200 {
resp.Body.Close()
return nil
}
if resp != nil {
resp.Body.Close()
}
select {
case <-time.After(retryInterval):
continue
case <-ctx.Done():
return ctx.Err()
}
}
return fmt.Errorf("IPFS Cluster %s did not become ready after %d seconds", name, (maxRetries * int(retryInterval.Seconds())))
}
// waitClusterFormed waits for all cluster peers to be visible from the bootstrap node
func (pm *ProcessManager) waitClusterFormed(ctx context.Context, bootstrapRestAPIPort int) error {
maxRetries := 30
retryInterval := 1 * time.Second
requiredPeers := 3 // bootstrap, node2, node3
for attempt := 0; attempt < maxRetries; attempt++ {
httpURL := fmt.Sprintf("http://127.0.0.1:%d/peers", bootstrapRestAPIPort)
resp, err := http.Get(httpURL)
if err == nil && resp.StatusCode == 200 {
// The /peers endpoint returns NDJSON (newline-delimited JSON), not a JSON array
// We need to stream-read each peer object
dec := json.NewDecoder(resp.Body)
peerCount := 0
for {
var peer interface{}
err := dec.Decode(&peer)
if err != nil {
if err == io.EOF {
break
}
break // Stop on parse error
}
peerCount++
}
resp.Body.Close()
if peerCount >= requiredPeers {
return nil // All peers have formed
}
}
if resp != nil {
resp.Body.Close()
}
select {
case <-time.After(retryInterval):
continue
case <-ctx.Done():
return ctx.Err()
}
}
return fmt.Errorf("IPFS Cluster did not form fully after %d seconds", (maxRetries * int(retryInterval.Seconds())))
}
// cleanClusterState removes stale cluster state files to ensure fresh initialization
// This prevents PSK (private network key) mismatches when cluster secret changes
func (pm *ProcessManager) cleanClusterState(clusterPath string) error {
// Remove pebble datastore (contains persisted PSK state)
pebblePath := filepath.Join(clusterPath, "pebble")
if err := os.RemoveAll(pebblePath); err != nil && !os.IsNotExist(err) {
return fmt.Errorf("failed to remove pebble directory: %w", err)
}
// Remove peerstore (contains peer addresses and metadata)
peerstorePath := filepath.Join(clusterPath, "peerstore")
if err := os.Remove(peerstorePath); err != nil && !os.IsNotExist(err) {
return fmt.Errorf("failed to remove peerstore: %w", err)
}
// Remove service.json (will be regenerated with correct ports and secret)
serviceJSONPath := filepath.Join(clusterPath, "service.json")
if err := os.Remove(serviceJSONPath); err != nil && !os.IsNotExist(err) {
return fmt.Errorf("failed to remove service.json: %w", err)
}
// Remove cluster.lock if it exists (from previous run)
lockPath := filepath.Join(clusterPath, "cluster.lock")
if err := os.Remove(lockPath); err != nil && !os.IsNotExist(err) {
return fmt.Errorf("failed to remove cluster.lock: %w", err)
}
// Note: We keep identity.json as it's tied to the node's peer ID
// The secret will be updated via CLUSTER_SECRET env var during init
return nil
}
// ensureIPFSClusterPorts updates service.json with correct per-node ports and IPFS connector settings
func (pm *ProcessManager) ensureIPFSClusterPorts(clusterPath string, restAPIPort int, clusterPort int) error {
serviceJSONPath := filepath.Join(clusterPath, "service.json")
// Read existing config
data, err := os.ReadFile(serviceJSONPath)
if err != nil {
return fmt.Errorf("failed to read service.json: %w", err)
}
var config map[string]interface{}
if err := json.Unmarshal(data, &config); err != nil {
return fmt.Errorf("failed to unmarshal service.json: %w", err)
}
// Calculate unique ports for this node based on restAPIPort offset
// bootstrap=9094 -> proxy=9095, pinsvc=9097, cluster=9096
// node2=9104 -> proxy=9105, pinsvc=9107, cluster=9106
// node3=9114 -> proxy=9115, pinsvc=9117, cluster=9116
portOffset := restAPIPort - 9094
proxyPort := 9095 + portOffset
pinsvcPort := 9097 + portOffset
// Infer IPFS port from REST API port
// 9094 -> 4501 (bootstrap), 9104 -> 4502 (node2), 9114 -> 4503 (node3)
ipfsPort := 4501 + (portOffset / 10)
// Update API settings
if api, ok := config["api"].(map[string]interface{}); ok {
// Update REST API listen address
if restapi, ok := api["restapi"].(map[string]interface{}); ok {
restapi["http_listen_multiaddress"] = fmt.Sprintf("/ip4/0.0.0.0/tcp/%d", restAPIPort)
}
// Update IPFS Proxy settings
if proxy, ok := api["ipfsproxy"].(map[string]interface{}); ok {
proxy["listen_multiaddress"] = fmt.Sprintf("/ip4/127.0.0.1/tcp/%d", proxyPort)
proxy["node_multiaddress"] = fmt.Sprintf("/ip4/127.0.0.1/tcp/%d", ipfsPort)
}
// Update Pinning Service API port
if pinsvc, ok := api["pinsvcapi"].(map[string]interface{}); ok {
pinsvc["http_listen_multiaddress"] = fmt.Sprintf("/ip4/127.0.0.1/tcp/%d", pinsvcPort)
}
}
// Update cluster listen multiaddress to match the correct port
// Replace all old listen addresses with new ones for the correct port
if cluster, ok := config["cluster"].(map[string]interface{}); ok {
listenAddrs := []string{
fmt.Sprintf("/ip4/0.0.0.0/tcp/%d", clusterPort),
fmt.Sprintf("/ip4/127.0.0.1/tcp/%d", clusterPort),
}
cluster["listen_multiaddress"] = listenAddrs
}
// Update IPFS connector settings to point to correct IPFS API port
if connector, ok := config["ipfs_connector"].(map[string]interface{}); ok {
if ipfshttp, ok := connector["ipfshttp"].(map[string]interface{}); ok {
ipfshttp["node_multiaddress"] = fmt.Sprintf("/ip4/127.0.0.1/tcp/%d", ipfsPort)
}
}
// Write updated config
updatedData, err := json.MarshalIndent(config, "", " ")
if err != nil {
return fmt.Errorf("failed to marshal updated config: %w", err)
}
if err := os.WriteFile(serviceJSONPath, updatedData, 0644); err != nil {
return fmt.Errorf("failed to write service.json: %w", err)
}
return nil
}
func (pm *ProcessManager) startOlric(ctx context.Context) error {
pidPath := filepath.Join(pm.pidsDir, "olric.pid")
logPath := filepath.Join(pm.oramaDir, "logs", "olric.log")
configPath := filepath.Join(pm.oramaDir, "olric-config.yaml")
cmd := exec.CommandContext(ctx, "olric-server")
cmd.Env = append(os.Environ(), fmt.Sprintf("OLRIC_SERVER_CONFIG=%s", configPath))
logFile, _ := os.Create(logPath)
cmd.Stdout = logFile
cmd.Stderr = logFile
if err := cmd.Start(); err != nil {
return fmt.Errorf("failed to start olric: %w", err)
}
os.WriteFile(pidPath, []byte(fmt.Sprintf("%d", cmd.Process.Pid)), 0644)
fmt.Fprintf(pm.logWriter, "✓ Olric started (PID: %d)\n", cmd.Process.Pid)
time.Sleep(1 * time.Second)
return nil
}
func (pm *ProcessManager) startAnon(ctx context.Context) error {
if runtime.GOOS != "darwin" {
return nil // Skip on non-macOS for now
}
pidPath := filepath.Join(pm.pidsDir, "anon.pid")
logPath := filepath.Join(pm.oramaDir, "logs", "anon.log")
cmd := exec.CommandContext(ctx, "npx", "anyone-client")
logFile, _ := os.Create(logPath)
cmd.Stdout = logFile
cmd.Stderr = logFile
if err := cmd.Start(); err != nil {
fmt.Fprintf(pm.logWriter, " ⚠️ Failed to start Anon: %v\n", err)
return nil
}
os.WriteFile(pidPath, []byte(fmt.Sprintf("%d", cmd.Process.Pid)), 0644)
fmt.Fprintf(pm.logWriter, "✓ Anon proxy started (PID: %d, SOCKS: 9050)\n", cmd.Process.Pid)
return nil
}
func (pm *ProcessManager) startNode(name, configFile, logPath string) error {
pidPath := filepath.Join(pm.pidsDir, fmt.Sprintf("%s.pid", name))
cmd := exec.Command("./bin/orama-node", "--config", configFile)
logFile, _ := os.Create(logPath)
cmd.Stdout = logFile
cmd.Stderr = logFile
if err := cmd.Start(); err != nil {
return fmt.Errorf("failed to start %s: %w", name, err)
}
os.WriteFile(pidPath, []byte(fmt.Sprintf("%d", cmd.Process.Pid)), 0644)
fmt.Fprintf(pm.logWriter, "✓ %s started (PID: %d)\n", strings.Title(name), cmd.Process.Pid)
time.Sleep(1 * time.Second)
return nil
}
func (pm *ProcessManager) startGateway(ctx context.Context) error {
pidPath := filepath.Join(pm.pidsDir, "gateway.pid")
logPath := filepath.Join(pm.oramaDir, "logs", "gateway.log")
cmd := exec.Command("./bin/gateway", "--config", "gateway.yaml")
logFile, _ := os.Create(logPath)
cmd.Stdout = logFile
cmd.Stderr = logFile
if err := cmd.Start(); err != nil {
return fmt.Errorf("failed to start gateway: %w", err)
}
os.WriteFile(pidPath, []byte(fmt.Sprintf("%d", cmd.Process.Pid)), 0644)
fmt.Fprintf(pm.logWriter, "✓ Gateway started (PID: %d, listen: 6001)\n", cmd.Process.Pid)
return nil
}
// stopProcess terminates a managed process and its children
func (pm *ProcessManager) stopProcess(name string) error {
pidPath := filepath.Join(pm.pidsDir, fmt.Sprintf("%s.pid", name))
pidBytes, err := os.ReadFile(pidPath)
if err != nil {
return nil // Process not running or PID not found
}
pid, err := strconv.Atoi(strings.TrimSpace(string(pidBytes)))
if err != nil {
os.Remove(pidPath)
return nil
}
// Check if process exists before trying to kill
if !checkProcessRunning(pid) {
os.Remove(pidPath)
fmt.Fprintf(pm.logWriter, "✓ %s (not running)\n", name)
return nil
}
proc, err := os.FindProcess(pid)
if err != nil {
os.Remove(pidPath)
return nil
}
// Try graceful shutdown first (SIGTERM)
proc.Signal(os.Interrupt)
// Wait up to 2 seconds for graceful shutdown
gracefulShutdown := false
for i := 0; i < 20; i++ {
time.Sleep(100 * time.Millisecond)
if !checkProcessRunning(pid) {
gracefulShutdown = true
break
}
}
// Force kill if still running after graceful attempt
if !gracefulShutdown && checkProcessRunning(pid) {
proc.Signal(os.Kill)
time.Sleep(200 * time.Millisecond)
// Kill any child processes (platform-specific)
if runtime.GOOS != "windows" {
exec.Command("pkill", "-9", "-P", fmt.Sprintf("%d", pid)).Run()
}
// Final force kill attempt if somehow still alive
if checkProcessRunning(pid) {
exec.Command("kill", "-9", fmt.Sprintf("%d", pid)).Run()
time.Sleep(100 * time.Millisecond)
}
}
os.Remove(pidPath)
if gracefulShutdown {
fmt.Fprintf(pm.logWriter, "✓ %s stopped gracefully\n", name)
} else {
fmt.Fprintf(pm.logWriter, "✓ %s stopped (forced)\n", name)
}
return nil
}
// checkProcessRunning checks if a process with given PID is running
func checkProcessRunning(pid int) bool {
proc, err := os.FindProcess(pid)
if err != nil {
return false
}
// Send signal 0 to check if process exists (doesn't actually send signal)
err = proc.Signal(os.Signal(nil))
return err == nil
}

30
pkg/gateway/jwt.go → pkg/gateway/auth/jwt.go
View File

@ -1,4 +1,4 @@
package gateway
package auth
import (
"crypto"
@ -13,13 +13,13 @@ import (
"time"
)
func (g *Gateway) jwksHandler(w http.ResponseWriter, r *http.Request) {
func (s *Service) JWKSHandler(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/json")
if g.signingKey == nil {
if s.signingKey == nil {
_ = json.NewEncoder(w).Encode(map[string]any{"keys": []any{}})
return
}
pub := g.signingKey.Public().(*rsa.PublicKey)
pub := s.signingKey.Public().(*rsa.PublicKey)
n := pub.N.Bytes()
// Encode exponent as big-endian bytes
eVal := pub.E
@ -35,7 +35,7 @@ func (g *Gateway) jwksHandler(w http.ResponseWriter, r *http.Request) {
"kty": "RSA",
"use": "sig",
"alg": "RS256",
"kid": g.keyID,
"kid": s.keyID,
"n": base64.RawURLEncoding.EncodeToString(n),
"e": base64.RawURLEncoding.EncodeToString(eb),
}
@ -49,7 +49,7 @@ type jwtHeader struct {
Kid string `json:"kid"`
}
type jwtClaims struct {
type JWTClaims struct {
Iss string `json:"iss"`
Sub string `json:"sub"`
Aud string `json:"aud"`
@ -59,9 +59,9 @@ type jwtClaims struct {
Namespace string `json:"namespace"`
}
// parseAndVerifyJWT verifies an RS256 JWT created by this gateway and returns claims
func (g *Gateway) parseAndVerifyJWT(token string) (*jwtClaims, error) {
if g.signingKey == nil {
// ParseAndVerifyJWT verifies an RS256 JWT created by this gateway and returns claims
func (s *Service) ParseAndVerifyJWT(token string) (*JWTClaims, error) {
if s.signingKey == nil {
return nil, errors.New("signing key unavailable")
}
parts := strings.Split(token, ".")
@ -90,12 +90,12 @@ func (g *Gateway) parseAndVerifyJWT(token string) (*jwtClaims, error) {
// Verify signature
signingInput := parts[0] + "." + parts[1]
sum := sha256.Sum256([]byte(signingInput))
pub := g.signingKey.Public().(*rsa.PublicKey)
pub := s.signingKey.Public().(*rsa.PublicKey)
if err := rsa.VerifyPKCS1v15(pub, crypto.SHA256, sum[:], sb); err != nil {
return nil, errors.New("invalid signature")
}
// Parse claims
var claims jwtClaims
var claims JWTClaims
if err := json.Unmarshal(pb, &claims); err != nil {
return nil, errors.New("invalid claims json")
}
@ -122,14 +122,14 @@ func (g *Gateway) parseAndVerifyJWT(token string) (*jwtClaims, error) {
return &claims, nil
}
func (g *Gateway) generateJWT(ns, subject string, ttl time.Duration) (string, int64, error) {
if g.signingKey == nil {
func (s *Service) GenerateJWT(ns, subject string, ttl time.Duration) (string, int64, error) {
if s.signingKey == nil {
return "", 0, errors.New("signing key unavailable")
}
header := map[string]string{
"alg": "RS256",
"typ": "JWT",
"kid": g.keyID,
"kid": s.keyID,
}
hb, _ := json.Marshal(header)
now := time.Now().UTC()
@ -148,7 +148,7 @@ func (g *Gateway) generateJWT(ns, subject string, ttl time.Duration) (string, in
pb64 := base64.RawURLEncoding.EncodeToString(pb)
signingInput := hb64 + "." + pb64
sum := sha256.Sum256([]byte(signingInput))
sig, err := rsa.SignPKCS1v15(rand.Reader, g.signingKey, crypto.SHA256, sum[:])
sig, err := rsa.SignPKCS1v15(rand.Reader, s.signingKey, crypto.SHA256, sum[:])
if err != nil {
return "", 0, err
}

391
pkg/gateway/auth/service.go Normal file
View File

@ -0,0 +1,391 @@
package auth
import (
"context"
"crypto/ed25519"
"crypto/rand"
"crypto/rsa"
"crypto/sha256"
"crypto/x509"
"encoding/base64"
"encoding/hex"
"encoding/json"
"encoding/pem"
"fmt"
"math/big"
"strconv"
"strings"
"time"
"github.com/DeBrosOfficial/network/pkg/client"
"github.com/DeBrosOfficial/network/pkg/logging"
ethcrypto "github.com/ethereum/go-ethereum/crypto"
)
// Service handles authentication business logic
type Service struct {
logger *logging.ColoredLogger
orm client.NetworkClient
signingKey *rsa.PrivateKey
keyID string
defaultNS string
}
func NewService(logger *logging.ColoredLogger, orm client.NetworkClient, signingKeyPEM string, defaultNS string) (*Service, error) {
s := &Service{
logger: logger,
orm: orm,
defaultNS: defaultNS,
}
if signingKeyPEM != "" {
block, _ := pem.Decode([]byte(signingKeyPEM))
if block == nil {
return nil, fmt.Errorf("failed to parse signing key PEM")
}
key, err := x509.ParsePKCS1PrivateKey(block.Bytes)
if err != nil {
return nil, fmt.Errorf("failed to parse RSA private key: %w", err)
}
s.signingKey = key
// Generate a simple KID from the public key hash
pubBytes := x509.MarshalPKCS1PublicKey(&key.PublicKey)
sum := sha256.Sum256(pubBytes)
s.keyID = hex.EncodeToString(sum[:8])
}
return s, nil
}
// CreateNonce generates a new nonce and stores it in the database
func (s *Service) CreateNonce(ctx context.Context, wallet, purpose, namespace string) (string, error) {
// Generate a URL-safe random nonce (32 bytes)
buf := make([]byte, 32)
if _, err := rand.Read(buf); err != nil {
return "", fmt.Errorf("failed to generate nonce: %w", err)
}
nonce := base64.RawURLEncoding.EncodeToString(buf)
// Use internal context to bypass authentication for system operations
internalCtx := client.WithInternalAuth(ctx)
db := s.orm.Database()
if namespace == "" {
namespace = s.defaultNS
if namespace == "" {
namespace = "default"
}
}
// Ensure namespace exists
if _, err := db.Query(internalCtx, "INSERT OR IGNORE INTO namespaces(name) VALUES (?)", namespace); err != nil {
return "", fmt.Errorf("failed to ensure namespace: %w", err)
}
nsID, err := s.ResolveNamespaceID(ctx, namespace)
if err != nil {
return "", fmt.Errorf("failed to resolve namespace ID: %w", err)
}
// Store nonce with 5 minute expiry
walletLower := strings.ToLower(strings.TrimSpace(wallet))
if _, err := db.Query(internalCtx,
"INSERT INTO nonces(namespace_id, wallet, nonce, purpose, expires_at) VALUES (?, ?, ?, ?, datetime('now', '+5 minutes'))",
nsID, walletLower, nonce, purpose,
); err != nil {
return "", fmt.Errorf("failed to store nonce: %w", err)
}
return nonce, nil
}
// VerifySignature verifies a wallet signature for a given nonce
func (s *Service) VerifySignature(ctx context.Context, wallet, nonce, signature, chainType string) (bool, error) {
chainType = strings.ToUpper(strings.TrimSpace(chainType))
if chainType == "" {
chainType = "ETH"
}
switch chainType {
case "ETH":
return s.verifyEthSignature(wallet, nonce, signature)
case "SOL":
return s.verifySolSignature(wallet, nonce, signature)
default:
return false, fmt.Errorf("unsupported chain type: %s", chainType)
}
}
func (s *Service) verifyEthSignature(wallet, nonce, signature string) (bool, error) {
msg := []byte(nonce)
prefix := []byte("\x19Ethereum Signed Message:\n" + strconv.Itoa(len(msg)))
hash := ethcrypto.Keccak256(prefix, msg)
sigHex := strings.TrimSpace(signature)
if strings.HasPrefix(sigHex, "0x") || strings.HasPrefix(sigHex, "0X") {
sigHex = sigHex[2:]
}
sig, err := hex.DecodeString(sigHex)
if err != nil || len(sig) != 65 {
return false, fmt.Errorf("invalid signature format")
}
if sig[64] >= 27 {
sig[64] -= 27
}
pub, err := ethcrypto.SigToPub(hash, sig)
if err != nil {
return false, fmt.Errorf("signature recovery failed: %w", err)
}
addr := ethcrypto.PubkeyToAddress(*pub).Hex()
want := strings.ToLower(strings.TrimPrefix(strings.TrimPrefix(wallet, "0x"), "0X"))
got := strings.ToLower(strings.TrimPrefix(strings.TrimPrefix(addr, "0x"), "0X"))
return got == want, nil
}
func (s *Service) verifySolSignature(wallet, nonce, signature string) (bool, error) {
sig, err := base64.StdEncoding.DecodeString(signature)
if err != nil {
return false, fmt.Errorf("invalid base64 signature: %w", err)
}
if len(sig) != 64 {
return false, fmt.Errorf("invalid signature length: expected 64 bytes, got %d", len(sig))
}
pubKeyBytes, err := s.Base58Decode(wallet)
if err != nil {
return false, fmt.Errorf("invalid wallet address: %w", err)
}
if len(pubKeyBytes) != 32 {
return false, fmt.Errorf("invalid public key length: expected 32 bytes, got %d", len(pubKeyBytes))
}
message := []byte(nonce)
return ed25519.Verify(ed25519.PublicKey(pubKeyBytes), message, sig), nil
}
// IssueTokens generates access and refresh tokens for a verified wallet
func (s *Service) IssueTokens(ctx context.Context, wallet, namespace string) (string, string, int64, error) {
if s.signingKey == nil {
return "", "", 0, fmt.Errorf("signing key unavailable")
}
// Issue access token (15m)
token, expUnix, err := s.GenerateJWT(namespace, wallet, 15*time.Minute)
if err != nil {
return "", "", 0, fmt.Errorf("failed to generate JWT: %w", err)
}
// Create refresh token (30d)
rbuf := make([]byte, 32)
if _, err := rand.Read(rbuf); err != nil {
return "", "", 0, fmt.Errorf("failed to generate refresh token: %w", err)
}
refresh := base64.RawURLEncoding.EncodeToString(rbuf)
nsID, err := s.ResolveNamespaceID(ctx, namespace)
if err != nil {
return "", "", 0, fmt.Errorf("failed to resolve namespace ID: %w", err)
}
internalCtx := client.WithInternalAuth(ctx)
db := s.orm.Database()
if _, err := db.Query(internalCtx,
"INSERT INTO refresh_tokens(namespace_id, subject, token, audience, expires_at) VALUES (?, ?, ?, ?, datetime('now', '+30 days'))",
nsID, wallet, refresh, "gateway",
); err != nil {
return "", "", 0, fmt.Errorf("failed to store refresh token: %w", err)
}
return token, refresh, expUnix, nil
}
// RefreshToken validates a refresh token and issues a new access token
func (s *Service) RefreshToken(ctx context.Context, refreshToken, namespace string) (string, string, int64, error) {
internalCtx := client.WithInternalAuth(ctx)
db := s.orm.Database()
nsID, err := s.ResolveNamespaceID(ctx, namespace)
if err != nil {
return "", "", 0, err
}
q := "SELECT subject FROM refresh_tokens WHERE namespace_id = ? AND token = ? AND revoked_at IS NULL AND (expires_at IS NULL OR expires_at > datetime('now')) LIMIT 1"
res, err := db.Query(internalCtx, q, nsID, refreshToken)
if err != nil || res == nil || res.Count == 0 {
return "", "", 0, fmt.Errorf("invalid or expired refresh token")
}
subject := ""
if len(res.Rows) > 0 && len(res.Rows[0]) > 0 {
if val, ok := res.Rows[0][0].(string); ok {
subject = val
} else {
b, _ := json.Marshal(res.Rows[0][0])
_ = json.Unmarshal(b, &subject)
}
}
token, expUnix, err := s.GenerateJWT(namespace, subject, 15*time.Minute)
if err != nil {
return "", "", 0, err
}
return token, subject, expUnix, nil
}
// RevokeToken revokes a specific refresh token or all tokens for a subject
func (s *Service) RevokeToken(ctx context.Context, namespace, token string, all bool, subject string) error {
internalCtx := client.WithInternalAuth(ctx)
db := s.orm.Database()
nsID, err := s.ResolveNamespaceID(ctx, namespace)
if err != nil {
return err
}
if token != "" {
_, err := db.Query(internalCtx, "UPDATE refresh_tokens SET revoked_at = datetime('now') WHERE namespace_id = ? AND token = ? AND revoked_at IS NULL", nsID, token)
return err
}
if all && subject != "" {
_, err := db.Query(internalCtx, "UPDATE refresh_tokens SET revoked_at = datetime('now') WHERE namespace_id = ? AND subject = ? AND revoked_at IS NULL", nsID, subject)
return err
}
return fmt.Errorf("nothing to revoke")
}
// RegisterApp registers a new client application
func (s *Service) RegisterApp(ctx context.Context, wallet, namespace, name, publicKey string) (string, error) {
internalCtx := client.WithInternalAuth(ctx)
db := s.orm.Database()
nsID, err := s.ResolveNamespaceID(ctx, namespace)
if err != nil {
return "", err
}
// Generate client app_id
buf := make([]byte, 12)
if _, err := rand.Read(buf); err != nil {
return "", fmt.Errorf("failed to generate app id: %w", err)
}
appID := "app_" + base64.RawURLEncoding.EncodeToString(buf)
// Persist app
if _, err := db.Query(internalCtx, "INSERT INTO apps(namespace_id, app_id, name, public_key) VALUES (?, ?, ?, ?)", nsID, appID, name, publicKey); err != nil {
return "", err
}
// Record ownership
_, _ = db.Query(internalCtx, "INSERT OR IGNORE INTO namespace_ownership(namespace_id, owner_type, owner_id) VALUES (?, ?, ?)", nsID, "wallet", wallet)
return appID, nil
}
// GetOrCreateAPIKey returns an existing API key or creates a new one for a wallet in a namespace
func (s *Service) GetOrCreateAPIKey(ctx context.Context, wallet, namespace string) (string, error) {
internalCtx := client.WithInternalAuth(ctx)
db := s.orm.Database()
nsID, err := s.ResolveNamespaceID(ctx, namespace)
if err != nil {
return "", err
}
// Try existing linkage
var apiKey string
r1, err := db.Query(internalCtx,
"SELECT api_keys.key FROM wallet_api_keys JOIN api_keys ON wallet_api_keys.api_key_id = api_keys.id WHERE wallet_api_keys.namespace_id = ? AND LOWER(wallet_api_keys.wallet) = LOWER(?) LIMIT 1",
nsID, wallet,
)
if err == nil && r1 != nil && r1.Count > 0 && len(r1.Rows) > 0 && len(r1.Rows[0]) > 0 {
if val, ok := r1.Rows[0][0].(string); ok {
apiKey = val
}
}
if apiKey != "" {
return apiKey, nil
}
// Create new API key
buf := make([]byte, 18)
if _, err := rand.Read(buf); err != nil {
return "", fmt.Errorf("failed to generate api key: %w", err)
}
apiKey = "ak_" + base64.RawURLEncoding.EncodeToString(buf) + ":" + namespace
if _, err := db.Query(internalCtx, "INSERT INTO api_keys(key, name, namespace_id) VALUES (?, ?, ?)", apiKey, "", nsID); err != nil {
return "", fmt.Errorf("failed to store api key: %w", err)
}
// Link wallet -> api_key
rid, err := db.Query(internalCtx, "SELECT id FROM api_keys WHERE key = ? LIMIT 1", apiKey)
if err == nil && rid != nil && rid.Count > 0 && len(rid.Rows) > 0 && len(rid.Rows[0]) > 0 {
apiKeyID := rid.Rows[0][0]
_, _ = db.Query(internalCtx, "INSERT OR IGNORE INTO wallet_api_keys(namespace_id, wallet, api_key_id) VALUES (?, ?, ?)", nsID, strings.ToLower(wallet), apiKeyID)
}
// Record ownerships
_, _ = db.Query(internalCtx, "INSERT OR IGNORE INTO namespace_ownership(namespace_id, owner_type, owner_id) VALUES (?, 'api_key', ?)", nsID, apiKey)
_, _ = db.Query(internalCtx, "INSERT OR IGNORE INTO namespace_ownership(namespace_id, owner_type, owner_id) VALUES (?, 'wallet', ?)", nsID, wallet)
return apiKey, nil
}
// ResolveNamespaceID ensures the given namespace exists and returns its primary key ID.
func (s *Service) ResolveNamespaceID(ctx context.Context, ns string) (interface{}, error) {
if s.orm == nil {
return nil, fmt.Errorf("client not initialized")
}
ns = strings.TrimSpace(ns)
if ns == "" {
ns = "default"
}
internalCtx := client.WithInternalAuth(ctx)
db := s.orm.Database()
if _, err := db.Query(internalCtx, "INSERT OR IGNORE INTO namespaces(name) VALUES (?)", ns); err != nil {
return nil, err
}
res, err := db.Query(internalCtx, "SELECT id FROM namespaces WHERE name = ? LIMIT 1", ns)
if err != nil {
return nil, err
}
if res == nil || res.Count == 0 || len(res.Rows) == 0 || len(res.Rows[0]) == 0 {
return nil, fmt.Errorf("failed to resolve namespace")
}
return res.Rows[0][0], nil
}
// Base58Decode decodes a base58-encoded string
func (s *Service) Base58Decode(input string) ([]byte, error) {
const alphabet = "123456789ABCDEFGHJKLMNPQRSTUVWXYZabcdefghijkmnopqrstuvwxyz"
answer := big.NewInt(0)
j := big.NewInt(1)
for i := len(input) - 1; i >= 0; i-- {
tmp := strings.IndexByte(alphabet, input[i])
if tmp == -1 {
return nil, fmt.Errorf("invalid base58 character")
}
idx := big.NewInt(int64(tmp))
tmp1 := new(big.Int)
tmp1.Mul(idx, j)
answer.Add(answer, tmp1)
j.Mul(j, big.NewInt(58))
}
// Handle leading zeros
res := answer.Bytes()
for i := 0; i < len(input) && input[i] == alphabet[0]; i++ {
res = append([]byte{0}, res...)
}
return res, nil
}

697
pkg/gateway/auth_handlers.go
View File

@ -1,20 +1,14 @@
package gateway
import (
"crypto/ed25519"
"crypto/rand"
"encoding/base64"
"encoding/hex"
"encoding/json"
"fmt"
"math/big"
"net/http"
"strconv"
"strings"
"time"
"github.com/DeBrosOfficial/network/pkg/client"
ethcrypto "github.com/ethereum/go-ethereum/crypto"
"github.com/DeBrosOfficial/network/pkg/gateway/auth"
)
func (g *Gateway) whoamiHandler(w http.ResponseWriter, r *http.Request) {
@ -29,7 +23,7 @@ func (g *Gateway) whoamiHandler(w http.ResponseWriter, r *http.Request) {
// Prefer JWT if present
if v := ctx.Value(ctxKeyJWT); v != nil {
if claims, ok := v.(*jwtClaims); ok && claims != nil {
if claims, ok := v.(*auth.JWTClaims); ok && claims != nil {
writeJSON(w, http.StatusOK, map[string]any{
"authenticated": true,
"method": "jwt",
@ -61,8 +55,8 @@ func (g *Gateway) whoamiHandler(w http.ResponseWriter, r *http.Request) {
}
func (g *Gateway) challengeHandler(w http.ResponseWriter, r *http.Request) {
if g.client == nil {
writeError(w, http.StatusServiceUnavailable, "client not initialized")
if g.authService == nil {
writeError(w, http.StatusServiceUnavailable, "auth service not initialized")
return
}
if r.Method != http.MethodPost {
@ -82,51 +76,16 @@ func (g *Gateway) challengeHandler(w http.ResponseWriter, r *http.Request) {
writeError(w, http.StatusBadRequest, "wallet is required")
return
}
ns := strings.TrimSpace(req.Namespace)
if ns == "" {
ns = strings.TrimSpace(g.cfg.ClientNamespace)
if ns == "" {
ns = "default"
}
}
// Generate a URL-safe random nonce (32 bytes)
buf := make([]byte, 32)
if _, err := rand.Read(buf); err != nil {
writeError(w, http.StatusInternalServerError, "failed to generate nonce")
return
}
nonce := base64.RawURLEncoding.EncodeToString(buf)
// Insert namespace if missing, fetch id
ctx := r.Context()
// Use internal context to bypass authentication for system operations
internalCtx := client.WithInternalAuth(ctx)
db := g.client.Database()
if _, err := db.Query(internalCtx, "INSERT OR IGNORE INTO namespaces(name) VALUES (?)", ns); err != nil {
writeError(w, http.StatusInternalServerError, err.Error())
return
}
nres, err := db.Query(internalCtx, "SELECT id FROM namespaces WHERE name = ? LIMIT 1", ns)
if err != nil || nres == nil || nres.Count == 0 || len(nres.Rows) == 0 || len(nres.Rows[0]) == 0 {
writeError(w, http.StatusInternalServerError, "failed to resolve namespace")
return
}
nsID := nres.Rows[0][0]
// Store nonce with 5 minute expiry
// Normalize wallet address to lowercase for case-insensitive comparison
walletLower := strings.ToLower(strings.TrimSpace(req.Wallet))
if _, err := db.Query(internalCtx,
"INSERT INTO nonces(namespace_id, wallet, nonce, purpose, expires_at) VALUES (?, ?, ?, ?, datetime('now', '+5 minutes'))",
nsID, walletLower, nonce, req.Purpose,
); err != nil {
nonce, err := g.authService.CreateNonce(r.Context(), req.Wallet, req.Purpose, req.Namespace)
if err != nil {
writeError(w, http.StatusInternalServerError, err.Error())
return
}
writeJSON(w, http.StatusOK, map[string]any{
"wallet": req.Wallet,
"namespace": ns,
"namespace": req.Namespace,
"nonce": nonce,
"purpose": req.Purpose,
"expires_at": time.Now().Add(5 * time.Minute).UTC().Format(time.RFC3339Nano),
@ -134,8 +93,8 @@ func (g *Gateway) challengeHandler(w http.ResponseWriter, r *http.Request) {
}
func (g *Gateway) verifyHandler(w http.ResponseWriter, r *http.Request) {
if g.client == nil {
writeError(w, http.StatusServiceUnavailable, "client not initialized")
if g.authService == nil {
writeError(w, http.StatusServiceUnavailable, "auth service not initialized")
return
}
if r.Method != http.MethodPost {
@ -147,7 +106,7 @@ func (g *Gateway) verifyHandler(w http.ResponseWriter, r *http.Request) {
Nonce string `json:"nonce"`
Signature string `json:"signature"`
Namespace string `json:"namespace"`
ChainType string `json:"chain_type"` // "ETH" or "SOL", defaults to "ETH"
ChainType string `json:"chain_type"`
}
if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
writeError(w, http.StatusBadRequest, "invalid json body")
@ -157,185 +116,30 @@ func (g *Gateway) verifyHandler(w http.ResponseWriter, r *http.Request) {
writeError(w, http.StatusBadRequest, "wallet, nonce and signature are required")
return
}
ns := strings.TrimSpace(req.Namespace)
if ns == "" {
ns = strings.TrimSpace(g.cfg.ClientNamespace)
if ns == "" {
ns = "default"
}
}
ctx := r.Context()
// Use internal context to bypass authentication for system operations
internalCtx := client.WithInternalAuth(ctx)
verified, err := g.authService.VerifySignature(ctx, req.Wallet, req.Nonce, req.Signature, req.ChainType)
if err != nil || !verified {
writeError(w, http.StatusUnauthorized, "signature verification failed")
return
}
// Mark nonce used
nsID, _ := g.authService.ResolveNamespaceID(ctx, req.Namespace)
db := g.client.Database()
nsID, err := g.resolveNamespaceID(ctx, ns)
_, _ = db.Query(client.WithInternalAuth(ctx), "UPDATE nonces SET used_at = datetime('now') WHERE namespace_id = ? AND wallet = ? AND nonce = ?", nsID, strings.ToLower(req.Wallet), req.Nonce)
token, refresh, expUnix, err := g.authService.IssueTokens(ctx, req.Wallet, req.Namespace)
if err != nil {
writeError(w, http.StatusInternalServerError, err.Error())
return
}
// Normalize wallet address to lowercase for case-insensitive comparison
walletLower := strings.ToLower(strings.TrimSpace(req.Wallet))
q := "SELECT id FROM nonces WHERE namespace_id = ? AND LOWER(wallet) = LOWER(?) AND nonce = ? AND used_at IS NULL AND (expires_at IS NULL OR expires_at > datetime('now')) LIMIT 1"
nres, err := db.Query(internalCtx, q, nsID, walletLower, req.Nonce)
if err != nil || nres == nil || nres.Count == 0 {
writeError(w, http.StatusBadRequest, "invalid or expired nonce")
return
}
nonceID := nres.Rows[0][0]
// Determine chain type (default to ETH for backward compatibility)
chainType := strings.ToUpper(strings.TrimSpace(req.ChainType))
if chainType == "" {
chainType = "ETH"
}
// Verify signature based on chain type
var verified bool
var verifyErr error
switch chainType {
case "ETH":
// EVM personal_sign verification of the nonce
msg := []byte(req.Nonce)
prefix := []byte("\x19Ethereum Signed Message:\n" + strconv.Itoa(len(msg)))
hash := ethcrypto.Keccak256(prefix, msg)
// Decode signature (expects 65-byte r||s||v, hex with optional 0x)
sigHex := strings.TrimSpace(req.Signature)
if strings.HasPrefix(sigHex, "0x") || strings.HasPrefix(sigHex, "0X") {
sigHex = sigHex[2:]
}
sig, err := hex.DecodeString(sigHex)
if err != nil || len(sig) != 65 {
writeError(w, http.StatusBadRequest, "invalid signature format")
return
}
// Normalize V to 0/1 as expected by geth
if sig[64] >= 27 {
sig[64] -= 27
}
pub, err := ethcrypto.SigToPub(hash, sig)
if err != nil {
writeError(w, http.StatusUnauthorized, "signature recovery failed")
return
}
addr := ethcrypto.PubkeyToAddress(*pub).Hex()
want := strings.ToLower(strings.TrimPrefix(strings.TrimPrefix(req.Wallet, "0x"), "0X"))
got := strings.ToLower(strings.TrimPrefix(strings.TrimPrefix(addr, "0x"), "0X"))
if got != want {
writeError(w, http.StatusUnauthorized, "signature does not match wallet")
return
}
verified = true
case "SOL":
// Solana uses Ed25519 signatures
// Signature is base64-encoded, public key is the wallet address (base58)
// Decode base64 signature (Solana signatures are 64 bytes)
sig, err := base64.StdEncoding.DecodeString(req.Signature)
if err != nil {
writeError(w, http.StatusBadRequest, fmt.Sprintf("invalid base64 signature: %v", err))
return
}
if len(sig) != 64 {
writeError(w, http.StatusBadRequest, fmt.Sprintf("invalid signature length: expected 64 bytes, got %d", len(sig)))
return
}
// Decode base58 public key (Solana wallet address)
pubKeyBytes, err := base58Decode(req.Wallet)
if err != nil {
writeError(w, http.StatusBadRequest, fmt.Sprintf("invalid wallet address: %v", err))
return
}
if len(pubKeyBytes) != 32 {
writeError(w, http.StatusBadRequest, fmt.Sprintf("invalid public key length: expected 32 bytes, got %d", len(pubKeyBytes)))
return
}
// Verify Ed25519 signature
message := []byte(req.Nonce)
if !ed25519.Verify(ed25519.PublicKey(pubKeyBytes), message, sig) {
writeError(w, http.StatusUnauthorized, "signature verification failed")
return
}
verified = true
default:
writeError(w, http.StatusBadRequest, fmt.Sprintf("unsupported chain type: %s (must be ETH or SOL)", chainType))
return
}
if !verified {
writeError(w, http.StatusUnauthorized, fmt.Sprintf("signature verification failed: %v", verifyErr))
return
}
// Mark nonce used now (after successful verification)
if _, err := db.Query(internalCtx, "UPDATE nonces SET used_at = datetime('now') WHERE id = ?", nonceID); err != nil {
writeError(w, http.StatusInternalServerError, err.Error())
return
}
if g.signingKey == nil {
writeError(w, http.StatusServiceUnavailable, "signing key unavailable")
return
}
// Issue access token (15m) and a refresh token (30d)
token, expUnix, err := g.generateJWT(ns, req.Wallet, 15*time.Minute)
apiKey, err := g.authService.GetOrCreateAPIKey(ctx, req.Wallet, req.Namespace)
if err != nil {
writeError(w, http.StatusInternalServerError, err.Error())
return
}
// create refresh token
rbuf := make([]byte, 32)
if _, err := rand.Read(rbuf); err != nil {
writeError(w, http.StatusInternalServerError, "failed to generate refresh token")
return
}
refresh := base64.RawURLEncoding.EncodeToString(rbuf)
if _, err := db.Query(internalCtx, "INSERT INTO refresh_tokens(namespace_id, subject, token, audience, expires_at) VALUES (?, ?, ?, ?, datetime('now', '+30 days'))", nsID, req.Wallet, refresh, "gateway"); err != nil {
writeError(w, http.StatusInternalServerError, err.Error())
return
}
// Ensure API key exists for this (namespace, wallet) and record ownerships
// This is done automatically after successful verification; no second nonce needed
var apiKey string
// Try existing linkage
r1, err := db.Query(internalCtx,
"SELECT api_keys.key FROM wallet_api_keys JOIN api_keys ON wallet_api_keys.api_key_id = api_keys.id WHERE wallet_api_keys.namespace_id = ? AND LOWER(wallet_api_keys.wallet) = LOWER(?) LIMIT 1",
nsID, req.Wallet,
)
if err == nil && r1 != nil && r1.Count > 0 && len(r1.Rows) > 0 && len(r1.Rows[0]) > 0 {
if s, ok := r1.Rows[0][0].(string); ok {
apiKey = s
} else {
b, _ := json.Marshal(r1.Rows[0][0])
_ = json.Unmarshal(b, &apiKey)
}
}
if strings.TrimSpace(apiKey) == "" {
// Create new API key with format ak_<random>:<namespace>
buf := make([]byte, 18)
if _, err := rand.Read(buf); err == nil {
apiKey = "ak_" + base64.RawURLEncoding.EncodeToString(buf) + ":" + ns
if _, err := db.Query(internalCtx, "INSERT INTO api_keys(key, name, namespace_id) VALUES (?, ?, ?)", apiKey, "", nsID); err == nil {
// Link wallet -> api_key
rid, err := db.Query(internalCtx, "SELECT id FROM api_keys WHERE key = ? LIMIT 1", apiKey)
if err == nil && rid != nil && rid.Count > 0 && len(rid.Rows) > 0 && len(rid.Rows[0]) > 0 {
apiKeyID := rid.Rows[0][0]
_, _ = db.Query(internalCtx, "INSERT OR IGNORE INTO wallet_api_keys(namespace_id, wallet, api_key_id) VALUES (?, ?, ?)", nsID, strings.ToLower(req.Wallet), apiKeyID)
}
}
}
}
// Record ownerships (best-effort)
_, _ = db.Query(internalCtx, "INSERT OR IGNORE INTO namespace_ownership(namespace_id, owner_type, owner_id) VALUES (?, 'api_key', ?)", nsID, apiKey)
_, _ = db.Query(internalCtx, "INSERT OR IGNORE INTO namespace_ownership(namespace_id, owner_type, owner_id) VALUES (?, 'wallet', ?)", nsID, req.Wallet)
writeJSON(w, http.StatusOK, map[string]any{
"access_token": token,
@ -343,23 +147,16 @@ func (g *Gateway) verifyHandler(w http.ResponseWriter, r *http.Request) {
"expires_in": int(expUnix - time.Now().Unix()),
"refresh_token": refresh,
"subject": req.Wallet,
"namespace": ns,
"namespace": req.Namespace,
"api_key": apiKey,
"nonce": req.Nonce,
"signature_verified": true,
})
}
// issueAPIKeyHandler creates or returns an API key for a verified wallet in a namespace.
// Requires: POST { wallet, nonce, signature, namespace }
// Behavior:
// - Validates nonce and signature like verifyHandler
// - Ensures namespace exists
// - If an API key already exists for (namespace, wallet), returns it; else creates one
// - Records namespace ownership mapping for the wallet and api_key
func (g *Gateway) issueAPIKeyHandler(w http.ResponseWriter, r *http.Request) {
if g.client == nil {
writeError(w, http.StatusServiceUnavailable, "client not initialized")
if g.authService == nil {
writeError(w, http.StatusServiceUnavailable, "auth service not initialized")
return
}
if r.Method != http.MethodPost {
@ -371,6 +168,7 @@ func (g *Gateway) issueAPIKeyHandler(w http.ResponseWriter, r *http.Request) {
Nonce string `json:"nonce"`
Signature string `json:"signature"`
Namespace string `json:"namespace"`
ChainType string `json:"chain_type"`
Plan string `json:"plan"`
}
if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
@ -381,110 +179,33 @@ func (g *Gateway) issueAPIKeyHandler(w http.ResponseWriter, r *http.Request) {
writeError(w, http.StatusBadRequest, "wallet, nonce and signature are required")
return
}
ns := strings.TrimSpace(req.Namespace)
if ns == "" {
ns = strings.TrimSpace(g.cfg.ClientNamespace)
if ns == "" {
ns = "default"
}
}
ctx := r.Context()
// Use internal context to bypass authentication for system operations
internalCtx := client.WithInternalAuth(ctx)
db := g.client.Database()
// Resolve namespace id
nsID, err := g.resolveNamespaceID(ctx, ns)
if err != nil {
writeError(w, http.StatusInternalServerError, err.Error())
return
}
// Validate nonce exists and not used/expired
// Normalize wallet address to lowercase for case-insensitive comparison
walletLower := strings.ToLower(strings.TrimSpace(req.Wallet))
q := "SELECT id FROM nonces WHERE namespace_id = ? AND LOWER(wallet) = LOWER(?) AND nonce = ? AND used_at IS NULL AND (expires_at IS NULL OR expires_at > datetime('now')) LIMIT 1"
nres, err := db.Query(internalCtx, q, nsID, walletLower, req.Nonce)
if err != nil || nres == nil || nres.Count == 0 {
writeError(w, http.StatusBadRequest, "invalid or expired nonce")
return
}
nonceID := nres.Rows[0][0]
// Verify signature like verifyHandler
msg := []byte(req.Nonce)
prefix := []byte("\x19Ethereum Signed Message:\n" + strconv.Itoa(len(msg)))
hash := ethcrypto.Keccak256(prefix, msg)
sigHex := strings.TrimSpace(req.Signature)
if strings.HasPrefix(sigHex, "0x") || strings.HasPrefix(sigHex, "0X") {
sigHex = sigHex[2:]
}
sig, err := hex.DecodeString(sigHex)
if err != nil || len(sig) != 65 {
writeError(w, http.StatusBadRequest, "invalid signature format")
return
}
if sig[64] >= 27 {
sig[64] -= 27
}
pub, err := ethcrypto.SigToPub(hash, sig)
if err != nil {
writeError(w, http.StatusUnauthorized, "signature recovery failed")
return
}
addr := ethcrypto.PubkeyToAddress(*pub).Hex()
want := strings.ToLower(strings.TrimPrefix(strings.TrimPrefix(req.Wallet, "0x"), "0X"))
got := strings.ToLower(strings.TrimPrefix(strings.TrimPrefix(addr, "0x"), "0X"))
if got != want {
writeError(w, http.StatusUnauthorized, "signature does not match wallet")
verified, err := g.authService.VerifySignature(ctx, req.Wallet, req.Nonce, req.Signature, req.ChainType)
if err != nil || !verified {
writeError(w, http.StatusUnauthorized, "signature verification failed")
return
}
// Mark nonce used
if _, err := db.Query(internalCtx, "UPDATE nonces SET used_at = datetime('now') WHERE id = ?", nonceID); err != nil {
nsID, _ := g.authService.ResolveNamespaceID(ctx, req.Namespace)
db := g.client.Database()
_, _ = db.Query(client.WithInternalAuth(ctx), "UPDATE nonces SET used_at = datetime('now') WHERE namespace_id = ? AND wallet = ? AND nonce = ?", nsID, strings.ToLower(req.Wallet), req.Nonce)
apiKey, err := g.authService.GetOrCreateAPIKey(ctx, req.Wallet, req.Namespace)
if err != nil {
writeError(w, http.StatusInternalServerError, err.Error())
return
}
// Check if api key exists for (namespace, wallet) via linkage table
var apiKey string
r1, err := db.Query(internalCtx, "SELECT api_keys.key FROM wallet_api_keys JOIN api_keys ON wallet_api_keys.api_key_id = api_keys.id WHERE wallet_api_keys.namespace_id = ? AND LOWER(wallet_api_keys.wallet) = LOWER(?) LIMIT 1", nsID, req.Wallet)
if err == nil && r1 != nil && r1.Count > 0 && len(r1.Rows) > 0 && len(r1.Rows[0]) > 0 {
if s, ok := r1.Rows[0][0].(string); ok {
apiKey = s
} else {
b, _ := json.Marshal(r1.Rows[0][0])
_ = json.Unmarshal(b, &apiKey)
}
}
if strings.TrimSpace(apiKey) == "" {
// Create new API key with format ak_<random>:<namespace>
buf := make([]byte, 18)
if _, err := rand.Read(buf); err != nil {
writeError(w, http.StatusInternalServerError, "failed to generate api key")
return
}
apiKey = "ak_" + base64.RawURLEncoding.EncodeToString(buf) + ":" + ns
if _, err := db.Query(internalCtx, "INSERT INTO api_keys(key, name, namespace_id) VALUES (?, ?, ?)", apiKey, "", nsID); err != nil {
writeError(w, http.StatusInternalServerError, err.Error())
return
}
// Create linkage
// Find api_key id
rid, err := db.Query(internalCtx, "SELECT id FROM api_keys WHERE key = ? LIMIT 1", apiKey)
if err == nil && rid != nil && rid.Count > 0 && len(rid.Rows) > 0 && len(rid.Rows[0]) > 0 {
apiKeyID := rid.Rows[0][0]
_, _ = db.Query(internalCtx, "INSERT OR IGNORE INTO wallet_api_keys(namespace_id, wallet, api_key_id) VALUES (?, ?, ?)", nsID, strings.ToLower(req.Wallet), apiKeyID)
}
}
// Record ownerships (best-effort)
_, _ = db.Query(internalCtx, "INSERT OR IGNORE INTO namespace_ownership(namespace_id, owner_type, owner_id) VALUES (?, 'api_key', ?)", nsID, apiKey)
_, _ = db.Query(internalCtx, "INSERT OR IGNORE INTO namespace_ownership(namespace_id, owner_type, owner_id) VALUES (?, 'wallet', ?)", nsID, req.Wallet)
writeJSON(w, http.StatusOK, map[string]any{
"api_key": apiKey,
"namespace": ns,
"namespace": req.Namespace,
"plan": func() string {
if strings.TrimSpace(req.Plan) == "" {
return "free"
} else {
return req.Plan
}
return req.Plan
}(),
"wallet": strings.ToLower(strings.TrimPrefix(strings.TrimPrefix(req.Wallet, "0x"), "0X")),
})
@ -494,8 +215,8 @@ func (g *Gateway) issueAPIKeyHandler(w http.ResponseWriter, r *http.Request) {
// Requires Authorization header with API key (Bearer or ApiKey or X-API-Key header).
// Returns a JWT bound to the namespace derived from the API key record.
func (g *Gateway) apiKeyToJWTHandler(w http.ResponseWriter, r *http.Request) {
if g.client == nil {
writeError(w, http.StatusServiceUnavailable, "client not initialized")
if g.authService == nil {
writeError(w, http.StatusServiceUnavailable, "auth service not initialized")
return
}
if r.Method != http.MethodPost {
@ -507,10 +228,10 @@ func (g *Gateway) apiKeyToJWTHandler(w http.ResponseWriter, r *http.Request) {
writeError(w, http.StatusUnauthorized, "missing API key")
return
}
// Validate and get namespace
db := g.client.Database()
ctx := r.Context()
// Use internal context to bypass authentication for system operations
internalCtx := client.WithInternalAuth(ctx)
q := "SELECT namespaces.name FROM api_keys JOIN namespaces ON api_keys.namespace_id = namespaces.id WHERE api_keys.key = ? LIMIT 1"
res, err := db.Query(internalCtx, q, key)
@ -518,28 +239,18 @@ func (g *Gateway) apiKeyToJWTHandler(w http.ResponseWriter, r *http.Request) {
writeError(w, http.StatusUnauthorized, "invalid API key")
return
}
var ns string
if s, ok := res.Rows[0][0].(string); ok {
ns = s
} else {
b, _ := json.Marshal(res.Rows[0][0])
_ = json.Unmarshal(b, &ns)
}
ns = strings.TrimSpace(ns)
if ns == "" {
writeError(w, http.StatusUnauthorized, "invalid API key")
return
}
if g.signingKey == nil {
writeError(w, http.StatusServiceUnavailable, "signing key unavailable")
return
}
// Subject is the API key string for now
token, expUnix, err := g.generateJWT(ns, key, 15*time.Minute)
token, expUnix, err := g.authService.GenerateJWT(ns, key, 15*time.Minute)
if err != nil {
writeError(w, http.StatusInternalServerError, err.Error())
return
}
writeJSON(w, http.StatusOK, map[string]any{
"access_token": token,
"token_type": "Bearer",
@ -549,8 +260,8 @@ func (g *Gateway) apiKeyToJWTHandler(w http.ResponseWriter, r *http.Request) {
}
func (g *Gateway) registerHandler(w http.ResponseWriter, r *http.Request) {
if g.client == nil {
writeError(w, http.StatusServiceUnavailable, "client not initialized")
if g.authService == nil {
writeError(w, http.StatusServiceUnavailable, "auth service not initialized")
return
}
if r.Method != http.MethodPost {
@ -562,6 +273,7 @@ func (g *Gateway) registerHandler(w http.ResponseWriter, r *http.Request) {
Nonce string `json:"nonce"`
Signature string `json:"signature"`
Namespace string `json:"namespace"`
ChainType string `json:"chain_type"`
Name string `json:"name"`
}
if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
@ -572,106 +284,45 @@ func (g *Gateway) registerHandler(w http.ResponseWriter, r *http.Request) {
writeError(w, http.StatusBadRequest, "wallet, nonce and signature are required")
return
}
ns := strings.TrimSpace(req.Namespace)
if ns == "" {
ns = strings.TrimSpace(g.cfg.ClientNamespace)
if ns == "" {
ns = "default"
}
}
ctx := r.Context()
// Use internal context to bypass authentication for system operations
internalCtx := client.WithInternalAuth(ctx)
verified, err := g.authService.VerifySignature(ctx, req.Wallet, req.Nonce, req.Signature, req.ChainType)
if err != nil || !verified {
writeError(w, http.StatusUnauthorized, "signature verification failed")
return
}
// Mark nonce used
nsID, _ := g.authService.ResolveNamespaceID(ctx, req.Namespace)
db := g.client.Database()
nsID, err := g.resolveNamespaceID(ctx, ns)
_, _ = db.Query(client.WithInternalAuth(ctx), "UPDATE nonces SET used_at = datetime('now') WHERE namespace_id = ? AND wallet = ? AND nonce = ?", nsID, strings.ToLower(req.Wallet), req.Nonce)
// In a real app we'd derive the public key from the signature, but for simplicity here
// we just use a placeholder or expect it in the request if needed.
// For Ethereum, we can recover it.
publicKey := "recovered-pk"
appID, err := g.authService.RegisterApp(ctx, req.Wallet, req.Namespace, req.Name, publicKey)
if err != nil {
writeError(w, http.StatusInternalServerError, err.Error())
return
}
// Validate nonce
q := "SELECT id FROM nonces WHERE namespace_id = ? AND wallet = ? AND nonce = ? AND used_at IS NULL AND (expires_at IS NULL OR expires_at > datetime('now')) LIMIT 1"
nres, err := db.Query(internalCtx, q, nsID, req.Wallet, req.Nonce)
if err != nil || nres == nil || nres.Count == 0 || len(nres.Rows) == 0 || len(nres.Rows[0]) == 0 {
writeError(w, http.StatusBadRequest, "invalid or expired nonce")
return
}
nonceID := nres.Rows[0][0]
// EVM personal_sign verification of the nonce
msg := []byte(req.Nonce)
prefix := []byte("\x19Ethereum Signed Message:\n" + strconv.Itoa(len(msg)))
hash := ethcrypto.Keccak256(prefix, msg)
// Decode signature (expects 65-byte r||s||v, hex with optional 0x)
sigHex := strings.TrimSpace(req.Signature)
if strings.HasPrefix(sigHex, "0x") || strings.HasPrefix(sigHex, "0X") {
sigHex = sigHex[2:]
}
sig, err := hex.DecodeString(sigHex)
if err != nil || len(sig) != 65 {
writeError(w, http.StatusBadRequest, "invalid signature format")
return
}
// Normalize V to 0/1 as expected by geth
if sig[64] >= 27 {
sig[64] -= 27
}
pub, err := ethcrypto.SigToPub(hash, sig)
if err != nil {
writeError(w, http.StatusUnauthorized, "signature recovery failed")
return
}
addr := ethcrypto.PubkeyToAddress(*pub).Hex()
want := strings.ToLower(strings.TrimPrefix(strings.TrimPrefix(req.Wallet, "0x"), "0X"))
got := strings.ToLower(strings.TrimPrefix(strings.TrimPrefix(addr, "0x"), "0X"))
if got != want {
writeError(w, http.StatusUnauthorized, "signature does not match wallet")
return
}
// Mark nonce used now (after successful verification)
if _, err := db.Query(internalCtx, "UPDATE nonces SET used_at = datetime('now') WHERE id = ?", nonceID); err != nil {
writeError(w, http.StatusInternalServerError, err.Error())
return
}
// Derive public key (uncompressed) hex
pubBytes := ethcrypto.FromECDSAPub(pub)
pubHex := "0x" + hex.EncodeToString(pubBytes)
// Generate client app_id
buf := make([]byte, 12)
if _, err := rand.Read(buf); err != nil {
writeError(w, http.StatusInternalServerError, "failed to generate app id")
return
}
appID := "app_" + base64.RawURLEncoding.EncodeToString(buf)
// Persist app
if _, err := db.Query(internalCtx, "INSERT INTO apps(namespace_id, app_id, name, public_key) VALUES (?, ?, ?, ?)", nsID, appID, req.Name, pubHex); err != nil {
writeError(w, http.StatusInternalServerError, err.Error())
return
}
// Record namespace ownership by wallet (best-effort)
_, _ = db.Query(internalCtx, "INSERT OR IGNORE INTO namespace_ownership(namespace_id, owner_type, owner_id) VALUES (?, ?, ?)", nsID, "wallet", req.Wallet)
writeJSON(w, http.StatusCreated, map[string]any{
"client_id": appID,
"app": map[string]any{
"app_id": appID,
"name": req.Name,
"public_key": pubHex,
"namespace": ns,
"wallet": strings.ToLower(strings.TrimPrefix(strings.TrimPrefix(req.Wallet, "0x"), "0X")),
"app_id": appID,
"name": req.Name,
"namespace": req.Namespace,
"wallet": strings.ToLower(req.Wallet),
},
"signature_verified": true,
})
}
func (g *Gateway) refreshHandler(w http.ResponseWriter, r *http.Request) {
if g.client == nil {
writeError(w, http.StatusServiceUnavailable, "client not initialized")
if g.authService == nil {
writeError(w, http.StatusServiceUnavailable, "auth service not initialized")
return
}
if r.Method != http.MethodPost {
@ -690,54 +341,20 @@ func (g *Gateway) refreshHandler(w http.ResponseWriter, r *http.Request) {
writeError(w, http.StatusBadRequest, "refresh_token is required")
return
}
ns := strings.TrimSpace(req.Namespace)
if ns == "" {
ns = strings.TrimSpace(g.cfg.ClientNamespace)
if ns == "" {
ns = "default"
}
}
ctx := r.Context()
// Use internal context to bypass authentication for system operations
internalCtx := client.WithInternalAuth(ctx)
db := g.client.Database()
nsID, err := g.resolveNamespaceID(ctx, ns)
token, subject, expUnix, err := g.authService.RefreshToken(r.Context(), req.RefreshToken, req.Namespace)
if err != nil {
writeError(w, http.StatusInternalServerError, err.Error())
return
}
q := "SELECT subject FROM refresh_tokens WHERE namespace_id = ? AND token = ? AND revoked_at IS NULL AND (expires_at IS NULL OR expires_at > datetime('now')) LIMIT 1"
rres, err := db.Query(internalCtx, q, nsID, req.RefreshToken)
if err != nil || rres == nil || rres.Count == 0 {
writeError(w, http.StatusUnauthorized, "invalid or expired refresh token")
return
}
subject := ""
if len(rres.Rows) > 0 && len(rres.Rows[0]) > 0 {
if s, ok := rres.Rows[0][0].(string); ok {
subject = s
} else {
// fallback: format via json
b, _ := json.Marshal(rres.Rows[0][0])
_ = json.Unmarshal(b, &subject)
}
}
if g.signingKey == nil {
writeError(w, http.StatusServiceUnavailable, "signing key unavailable")
return
}
token, expUnix, err := g.generateJWT(ns, subject, 15*time.Minute)
if err != nil {
writeError(w, http.StatusInternalServerError, err.Error())
writeError(w, http.StatusUnauthorized, err.Error())
return
}
writeJSON(w, http.StatusOK, map[string]any{
"access_token": token,
"token_type": "Bearer",
"expires_in": int(expUnix - time.Now().Unix()),
"refresh_token": req.RefreshToken,
"subject": subject,
"namespace": ns,
"namespace": req.Namespace,
})
}
@ -1064,8 +681,8 @@ func (g *Gateway) loginPageHandler(w http.ResponseWriter, r *http.Request) {
// be revoked. If all=true is provided (and the request is authenticated via JWT),
// all tokens for the JWT subject within the namespace are revoked.
func (g *Gateway) logoutHandler(w http.ResponseWriter, r *http.Request) {
if g.client == nil {
writeError(w, http.StatusServiceUnavailable, "client not initialized")
if g.authService == nil {
writeError(w, http.StatusServiceUnavailable, "auth service not initialized")
return
}
if r.Method != http.MethodPost {
@ -1081,38 +698,12 @@ func (g *Gateway) logoutHandler(w http.ResponseWriter, r *http.Request) {
writeError(w, http.StatusBadRequest, "invalid json body")
return
}
ns := strings.TrimSpace(req.Namespace)
if ns == "" {
ns = strings.TrimSpace(g.cfg.ClientNamespace)
if ns == "" {
ns = "default"
}
}
ctx := r.Context()
// Use internal context to bypass authentication for system operations
internalCtx := client.WithInternalAuth(ctx)
db := g.client.Database()
nsID, err := g.resolveNamespaceID(ctx, ns)
if err != nil {
writeError(w, http.StatusInternalServerError, err.Error())
return
}
if strings.TrimSpace(req.RefreshToken) != "" {
// Revoke specific token
if _, err := db.Query(internalCtx, "UPDATE refresh_tokens SET revoked_at = datetime('now') WHERE namespace_id = ? AND token = ? AND revoked_at IS NULL", nsID, req.RefreshToken); err != nil {
writeError(w, http.StatusInternalServerError, err.Error())
return
}
writeJSON(w, http.StatusOK, map[string]any{"status": "ok", "revoked": 1})
return
}
var subject string
if req.All {
// Require JWT to identify subject
var subject string
if v := ctx.Value(ctxKeyJWT); v != nil {
if claims, ok := v.(*jwtClaims); ok && claims != nil {
if claims, ok := v.(*auth.JWTClaims); ok && claims != nil {
subject = strings.TrimSpace(claims.Sub)
}
}
@ -1120,23 +711,19 @@ func (g *Gateway) logoutHandler(w http.ResponseWriter, r *http.Request) {
writeError(w, http.StatusUnauthorized, "jwt required for all=true")
return
}
if _, err := db.Query(internalCtx, "UPDATE refresh_tokens SET revoked_at = datetime('now') WHERE namespace_id = ? AND subject = ? AND revoked_at IS NULL", nsID, subject); err != nil {
writeError(w, http.StatusInternalServerError, err.Error())
return
}
writeJSON(w, http.StatusOK, map[string]any{"status": "ok", "revoked": "all"})
}
if err := g.authService.RevokeToken(ctx, req.Namespace, req.RefreshToken, req.All, subject); err != nil {
writeError(w, http.StatusInternalServerError, err.Error())
return
}
writeError(w, http.StatusBadRequest, "nothing to revoke: provide refresh_token or all=true")
writeJSON(w, http.StatusOK, map[string]any{"status": "ok"})
}
// simpleAPIKeyHandler creates an API key directly from a wallet address without signature verification
// This is a simplified flow for development/testing
// Requires: POST { wallet, namespace }
func (g *Gateway) simpleAPIKeyHandler(w http.ResponseWriter, r *http.Request) {
if g.client == nil {
writeError(w, http.StatusServiceUnavailable, "client not initialized")
if g.authService == nil {
writeError(w, http.StatusServiceUnavailable, "auth service not initialized")
return
}
if r.Method != http.MethodPost {
@ -1159,114 +746,16 @@ func (g *Gateway) simpleAPIKeyHandler(w http.ResponseWriter, r *http.Request) {
return
}
ns := strings.TrimSpace(req.Namespace)
if ns == "" {
ns = strings.TrimSpace(g.cfg.ClientNamespace)
if ns == "" {
ns = "default"
}
}
ctx := r.Context()
internalCtx := client.WithInternalAuth(ctx)
db := g.client.Database()
// Resolve or create namespace
if _, err := db.Query(internalCtx, "INSERT OR IGNORE INTO namespaces(name) VALUES (?)", ns); err != nil {
apiKey, err := g.authService.GetOrCreateAPIKey(r.Context(), req.Wallet, req.Namespace)
if err != nil {
writeError(w, http.StatusInternalServerError, err.Error())
return
}
nres, err := db.Query(internalCtx, "SELECT id FROM namespaces WHERE name = ? LIMIT 1", ns)
if err != nil || nres == nil || nres.Count == 0 || len(nres.Rows) == 0 || len(nres.Rows[0]) == 0 {
writeError(w, http.StatusInternalServerError, "failed to resolve namespace")
return
}
nsID := nres.Rows[0][0]
// Check if api key already exists for (namespace, wallet)
var apiKey string
r1, err := db.Query(internalCtx,
"SELECT api_keys.key FROM wallet_api_keys JOIN api_keys ON wallet_api_keys.api_key_id = api_keys.id WHERE wallet_api_keys.namespace_id = ? AND LOWER(wallet_api_keys.wallet) = LOWER(?) LIMIT 1",
nsID, req.Wallet,
)
if err == nil && r1 != nil && r1.Count > 0 && len(r1.Rows) > 0 && len(r1.Rows[0]) > 0 {
if s, ok := r1.Rows[0][0].(string); ok {
apiKey = s
} else {
b, _ := json.Marshal(r1.Rows[0][0])
_ = json.Unmarshal(b, &apiKey)
}
}
// If no existing key, create a new one
if strings.TrimSpace(apiKey) == "" {
buf := make([]byte, 18)
if _, err := rand.Read(buf); err != nil {
writeError(w, http.StatusInternalServerError, "failed to generate api key")
return
}
apiKey = "ak_" + base64.RawURLEncoding.EncodeToString(buf) + ":" + ns
if _, err := db.Query(internalCtx, "INSERT INTO api_keys(key, name, namespace_id) VALUES (?, ?, ?)", apiKey, "", nsID); err != nil {
writeError(w, http.StatusInternalServerError, err.Error())
return
}
// Link wallet to api key
rid, err := db.Query(internalCtx, "SELECT id FROM api_keys WHERE key = ? LIMIT 1", apiKey)
if err == nil && rid != nil && rid.Count > 0 && len(rid.Rows) > 0 && len(rid.Rows[0]) > 0 {
apiKeyID := rid.Rows[0][0]
_, _ = db.Query(internalCtx, "INSERT OR IGNORE INTO wallet_api_keys(namespace_id, wallet, api_key_id) VALUES (?, ?, ?)", nsID, strings.ToLower(req.Wallet), apiKeyID)
}
}
// Record ownerships (best-effort)
_, _ = db.Query(internalCtx, "INSERT OR IGNORE INTO namespace_ownership(namespace_id, owner_type, owner_id) VALUES (?, 'api_key', ?)", nsID, apiKey)
_, _ = db.Query(internalCtx, "INSERT OR IGNORE INTO namespace_ownership(namespace_id, owner_type, owner_id) VALUES (?, 'wallet', ?)", nsID, req.Wallet)
writeJSON(w, http.StatusOK, map[string]any{
"api_key": apiKey,
"namespace": ns,
"namespace": req.Namespace,
"wallet": strings.ToLower(strings.TrimPrefix(strings.TrimPrefix(req.Wallet, "0x"), "0X")),
"created": time.Now().Format(time.RFC3339),
})
}
// base58Decode decodes a base58-encoded string (Bitcoin alphabet)
// Used for decoding Solana public keys (base58-encoded 32-byte ed25519 public keys)
func base58Decode(encoded string) ([]byte, error) {
const alphabet = "123456789ABCDEFGHJKLMNPQRSTUVWXYZabcdefghijkmnopqrstuvwxyz"
// Build reverse lookup map
lookup := make(map[rune]int)
for i, c := range alphabet {
lookup[c] = i
}
// Convert to big integer
num := big.NewInt(0)
base := big.NewInt(58)
for _, c := range encoded {
val, ok := lookup[c]
if !ok {
return nil, fmt.Errorf("invalid base58 character: %c", c)
}
num.Mul(num, base)
num.Add(num, big.NewInt(int64(val)))
}
// Convert to bytes
decoded := num.Bytes()
// Add leading zeros for each leading '1' in the input
for _, c := range encoded {
if c != '1' {
break
}
decoded = append([]byte{0}, decoded...)
}
return decoded, nil
}

47
pkg/gateway/gateway.go
View File

@ -4,12 +4,13 @@ import (
"context"
"crypto/rand"
"crypto/rsa"
"crypto/x509"
"database/sql"
"encoding/pem"
"fmt"
"net"
"os"
"path/filepath"
"strconv"
"strings"
"sync"
"time"
@ -21,6 +22,7 @@ import (
"github.com/DeBrosOfficial/network/pkg/olric"
"github.com/DeBrosOfficial/network/pkg/rqlite"
"github.com/DeBrosOfficial/network/pkg/serverless"
"github.com/DeBrosOfficial/network/pkg/gateway/auth"
"github.com/multiformats/go-multiaddr"
olriclib "github.com/olric-data/olric"
"go.uber.org/zap"
@ -68,8 +70,6 @@ type Gateway struct {
client client.NetworkClient
nodePeerID string // The node's actual peer ID from its identity file (overrides client's peer ID)
startedAt time.Time
signingKey *rsa.PrivateKey
keyID string
// rqlite SQL connection and HTTP ORM gateway
sqlDB *sql.DB
@ -93,6 +93,9 @@ type Gateway struct {
serverlessInvoker *serverless.Invoker
serverlessWSMgr *serverless.WSManager
serverlessHandlers *ServerlessHandlers
// Authentication service
authService *auth.Service
}
// localSubscriber represents a WebSocket subscriber for local message delivery
@ -139,16 +142,6 @@ func New(logger *logging.ColoredLogger, cfg *Config) (*Gateway, error) {
localSubscribers: make(map[string][]*localSubscriber),
}
logger.ComponentInfo(logging.ComponentGeneral, "Generating RSA signing key...")
// Generate local RSA signing key for JWKS/JWT (ephemeral for now)
if key, err := rsa.GenerateKey(rand.Reader, 2048); err == nil {
gw.signingKey = key
gw.keyID = "gw-" + strconv.FormatInt(time.Now().Unix(), 10)
logger.ComponentInfo(logging.ComponentGeneral, "RSA key generated successfully")
} else {
logger.ComponentWarn(logging.ComponentGeneral, "failed to generate RSA key; jwks will be empty", zap.Error(err))
}
logger.ComponentInfo(logging.ComponentGeneral, "Initializing RQLite ORM HTTP gateway...")
dsn := cfg.RQLiteDSN
if dsn == "" {
@ -362,14 +355,28 @@ func New(logger *logging.ColoredLogger, cfg *Config) (*Gateway, error) {
gw.serverlessInvoker = serverless.NewInvoker(engine, registry, hostFuncs, logger.Logger)
// Create HTTP handlers
gw.serverlessHandlers = NewServerlessHandlers(
gw.serverlessInvoker,
registry,
gw.serverlessWSMgr,
logger.Logger,
)
gw.serverlessHandlers = NewServerlessHandlers(
gw.serverlessInvoker,
registry,
gw.serverlessWSMgr,
logger.Logger,
)
logger.ComponentInfo(logging.ComponentGeneral, "Serverless function engine ready",
// Initialize auth service
// For now using ephemeral key, can be loaded from config later
key, _ := rsa.GenerateKey(rand.Reader, 2048)
keyPEM := pem.EncodeToMemory(&pem.Block{
Type: "RSA PRIVATE KEY",
Bytes: x509.MarshalPKCS1PrivateKey(key),
})
authService, err := auth.NewService(logger, c, string(keyPEM), cfg.ClientNamespace)
if err != nil {
logger.ComponentError(logging.ComponentGeneral, "failed to initialize auth service", zap.Error(err))
} else {
gw.authService = authService
}
logger.ComponentInfo(logging.ComponentGeneral, "Serverless function engine ready",
zap.Int("default_memory_mb", engineCfg.DefaultMemoryLimitMB),
zap.Int("default_timeout_sec", engineCfg.DefaultTimeoutSeconds),
zap.Int("module_cache_size", engineCfg.ModuleCacheSize),

36
pkg/gateway/jwt_test.go
View File

@ -3,22 +3,32 @@ package gateway
import (
"crypto/rand"
"crypto/rsa"
"crypto/x509"
"encoding/pem"
"testing"
"time"
"github.com/DeBrosOfficial/network/pkg/gateway/auth"
)
func TestJWTGenerateAndParse(t *testing.T) {
gw := &Gateway{}
key, _ := rsa.GenerateKey(rand.Reader, 2048)
gw.signingKey = key
gw.keyID = "kid"
keyPEM := pem.EncodeToMemory(&pem.Block{
Type: "RSA PRIVATE KEY",
Bytes: x509.MarshalPKCS1PrivateKey(key),
})
tok, exp, err := gw.generateJWT("ns1", "subj", time.Minute)
svc, err := auth.NewService(nil, nil, string(keyPEM), "default")
if err != nil {
t.Fatalf("failed to create service: %v", err)
}
tok, exp, err := svc.GenerateJWT("ns1", "subj", time.Minute)
if err != nil || exp <= 0 {
t.Fatalf("gen err=%v exp=%d", err, exp)
}
claims, err := gw.parseAndVerifyJWT(tok)
claims, err := svc.ParseAndVerifyJWT(tok)
if err != nil {
t.Fatalf("verify err: %v", err)
}
@ -28,17 +38,23 @@ func TestJWTGenerateAndParse(t *testing.T) {
}
func TestJWTExpired(t *testing.T) {
gw := &Gateway{}
key, _ := rsa.GenerateKey(rand.Reader, 2048)
gw.signingKey = key
gw.keyID = "kid"
keyPEM := pem.EncodeToMemory(&pem.Block{
Type: "RSA PRIVATE KEY",
Bytes: x509.MarshalPKCS1PrivateKey(key),
})
svc, err := auth.NewService(nil, nil, string(keyPEM), "default")
if err != nil {
t.Fatalf("failed to create service: %v", err)
}
// Use sufficiently negative TTL to bypass allowed clock skew
tok, _, err := gw.generateJWT("ns1", "subj", -2*time.Minute)
tok, _, err := svc.GenerateJWT("ns1", "subj", -2*time.Minute)
if err != nil {
t.Fatalf("gen err=%v", err)
}
if _, err := gw.parseAndVerifyJWT(tok); err == nil {
if _, err := svc.ParseAndVerifyJWT(tok); err == nil {
t.Fatalf("expected expired error")
}
}

5
pkg/gateway/middleware.go
View File

@ -10,6 +10,7 @@ import (
"time"
"github.com/DeBrosOfficial/network/pkg/client"
"github.com/DeBrosOfficial/network/pkg/gateway/auth"
"github.com/DeBrosOfficial/network/pkg/logging"
"go.uber.org/zap"
)
@ -74,7 +75,7 @@ func (g *Gateway) authMiddleware(next http.Handler) http.Handler {
if strings.HasPrefix(lower, "bearer ") {
tok := strings.TrimSpace(auth[len("Bearer "):])
if strings.Count(tok, ".") == 2 {
if claims, err := g.parseAndVerifyJWT(tok); err == nil {
if claims, err := g.authService.ParseAndVerifyJWT(tok); err == nil {
// Attach JWT claims and namespace to context
ctx := context.WithValue(r.Context(), ctxKeyJWT, claims)
if ns := strings.TrimSpace(claims.Namespace); ns != "" {
@ -235,7 +236,7 @@ func (g *Gateway) authorizationMiddleware(next http.Handler) http.Handler {
apiKeyFallback := ""
if v := ctx.Value(ctxKeyJWT); v != nil {
if claims, ok := v.(*jwtClaims); ok && claims != nil && strings.TrimSpace(claims.Sub) != "" {
if claims, ok := v.(*auth.JWTClaims); ok && claims != nil && strings.TrimSpace(claims.Sub) != "" {
// Determine subject type.
// If subject looks like an API key (e.g., ak_<random>:<namespace>),
// treat it as an API key owner; otherwise assume a wallet subject.

4
pkg/gateway/routes.go
View File

@ -14,8 +14,8 @@ func (g *Gateway) Routes() http.Handler {
mux.HandleFunc("/v1/status", g.statusHandler)
// auth endpoints
mux.HandleFunc("/v1/auth/jwks", g.jwksHandler)
mux.HandleFunc("/.well-known/jwks.json", g.jwksHandler)
mux.HandleFunc("/v1/auth/jwks", g.authService.JWKSHandler)
mux.HandleFunc("/.well-known/jwks.json", g.authService.JWKSHandler)
mux.HandleFunc("/v1/auth/login", g.loginPageHandler)
mux.HandleFunc("/v1/auth/challenge", g.challengeHandler)
mux.HandleFunc("/v1/auth/verify", g.verifyHandler)

14
pkg/installer/installer.go
View File

@ -17,6 +17,7 @@ import (
"github.com/charmbracelet/lipgloss"
"github.com/DeBrosOfficial/network/pkg/certutil"
"github.com/DeBrosOfficial/network/pkg/config"
"github.com/DeBrosOfficial/network/pkg/tlsutil"
)
@ -338,7 +339,7 @@ func (m *Model) handleEnter() (tea.Model, tea.Cmd) {
case StepSwarmKey:
swarmKey := strings.TrimSpace(m.textInput.Value())
if err := validateSwarmKey(swarmKey); err != nil {
if err := config.ValidateSwarmKey(swarmKey); err != nil {
m.err = err
return m, nil
}
@ -816,17 +817,6 @@ func validateClusterSecret(secret string) error {
return nil
}
func validateSwarmKey(key string) error {
if len(key) != 64 {
return fmt.Errorf("swarm key must be 64 hex characters")
}
keyRegex := regexp.MustCompile(`^[a-fA-F0-9]{64}$`)
if !keyRegex.MatchString(key) {
return fmt.Errorf("swarm key must be valid hexadecimal")
}
return nil
}
// ensureCertificatesForDomain generates self-signed certificates for the domain
func ensureCertificatesForDomain(domain string) error {
// Get home directory

1171
pkg/ipfs/cluster.go
View File

File diff suppressed because it is too large Load Diff

136
pkg/ipfs/cluster_config.go Normal file
View File

@ -0,0 +1,136 @@
package ipfs
import (
"encoding/json"
"fmt"
"os"
"path/filepath"
)
// ClusterServiceConfig represents the service.json configuration
type ClusterServiceConfig struct {
Cluster struct {
Peername string `json:"peername"`
Secret string `json:"secret"`
ListenMultiaddress []string `json:"listen_multiaddress"`
PeerAddresses []string `json:"peer_addresses"`
LeaveOnShutdown bool `json:"leave_on_shutdown"`
} `json:"cluster"`
Consensus struct {
CRDT struct {
ClusterName string `json:"cluster_name"`
TrustedPeers []string `json:"trusted_peers"`
Batching struct {
MaxBatchSize int `json:"max_batch_size"`
MaxBatchAge string `json:"max_batch_age"`
} `json:"batching"`
RepairInterval string `json:"repair_interval"`
} `json:"crdt"`
} `json:"consensus"`
API struct {
RestAPI struct {
HTTPListenMultiaddress string `json:"http_listen_multiaddress"`
} `json:"restapi"`
IPFSProxy struct {
ListenMultiaddress string `json:"listen_multiaddress"`
NodeMultiaddress string `json:"node_multiaddress"`
} `json:"ipfsproxy"`
PinSvcAPI struct {
HTTPListenMultiaddress string `json:"http_listen_multiaddress"`
} `json:"pinsvcapi"`
} `json:"api"`
IPFSConnector struct {
IPFSHTTP struct {
NodeMultiaddress string `json:"node_multiaddress"`
} `json:"ipfshttp"`
} `json:"ipfs_connector"`
Raw map[string]interface{} `json:"-"`
}
func (cm *ClusterConfigManager) loadOrCreateConfig(path string) (*ClusterServiceConfig, error) {
if _, err := os.Stat(path); os.IsNotExist(err) {
return cm.createTemplateConfig(), nil
}
data, err := os.ReadFile(path)
if err != nil {
return nil, fmt.Errorf("failed to read service.json: %w", err)
}
var cfg ClusterServiceConfig
if err := json.Unmarshal(data, &cfg); err != nil {
return nil, fmt.Errorf("failed to parse service.json: %w", err)
}
var raw map[string]interface{}
if err := json.Unmarshal(data, &raw); err != nil {
return nil, fmt.Errorf("failed to parse raw service.json: %w", err)
}
cfg.Raw = raw
return &cfg, nil
}
func (cm *ClusterConfigManager) saveConfig(path string, cfg *ClusterServiceConfig) error {
cm.updateNestedMap(cfg.Raw, "cluster", "peername", cfg.Cluster.Peername)
cm.updateNestedMap(cfg.Raw, "cluster", "secret", cfg.Cluster.Secret)
cm.updateNestedMap(cfg.Raw, "cluster", "listen_multiaddress", cfg.Cluster.ListenMultiaddress)
cm.updateNestedMap(cfg.Raw, "cluster", "peer_addresses", cfg.Cluster.PeerAddresses)
cm.updateNestedMap(cfg.Raw, "cluster", "leave_on_shutdown", cfg.Cluster.LeaveOnShutdown)
consensus := cm.ensureRequiredSection(cfg.Raw, "consensus")
crdt := cm.ensureRequiredSection(consensus, "crdt")
crdt["cluster_name"] = cfg.Consensus.CRDT.ClusterName
crdt["trusted_peers"] = cfg.Consensus.CRDT.TrustedPeers
crdt["repair_interval"] = cfg.Consensus.CRDT.RepairInterval
batching := cm.ensureRequiredSection(crdt, "batching")
batching["max_batch_size"] = cfg.Consensus.CRDT.Batching.MaxBatchSize
batching["max_batch_age"] = cfg.Consensus.CRDT.Batching.MaxBatchAge
api := cm.ensureRequiredSection(cfg.Raw, "api")
restapi := cm.ensureRequiredSection(api, "restapi")
restapi["http_listen_multiaddress"] = cfg.API.RestAPI.HTTPListenMultiaddress
ipfsproxy := cm.ensureRequiredSection(api, "ipfsproxy")
ipfsproxy["listen_multiaddress"] = cfg.API.IPFSProxy.ListenMultiaddress
ipfsproxy["node_multiaddress"] = cfg.API.IPFSProxy.NodeMultiaddress
pinsvcapi := cm.ensureRequiredSection(api, "pinsvcapi")
pinsvcapi["http_listen_multiaddress"] = cfg.API.PinSvcAPI.HTTPListenMultiaddress
ipfsConn := cm.ensureRequiredSection(cfg.Raw, "ipfs_connector")
ipfsHttp := cm.ensureRequiredSection(ipfsConn, "ipfshttp")
ipfsHttp["node_multiaddress"] = cfg.IPFSConnector.IPFSHTTP.NodeMultiaddress
data, err := json.MarshalIndent(cfg.Raw, "", " ")
if err != nil {
return fmt.Errorf("failed to marshal service.json: %w", err)
}
if err := os.MkdirAll(filepath.Dir(path), 0755); err != nil {
return fmt.Errorf("failed to create config directory: %w", err)
}
return os.WriteFile(path, data, 0644)
}
func (cm *ClusterConfigManager) updateNestedMap(m map[string]interface{}, section, key string, val interface{}) {
if _, ok := m[section]; !ok {
m[section] = make(map[string]interface{})
}
s := m[section].(map[string]interface{})
s[key] = val
}
func (cm *ClusterConfigManager) ensureRequiredSection(m map[string]interface{}, key string) map[string]interface{} {
if _, ok := m[key]; !ok {
m[key] = make(map[string]interface{})
}
return m[key].(map[string]interface{})
}

156
pkg/ipfs/cluster_peer.go Normal file
View File

@ -0,0 +1,156 @@
package ipfs
import (
"fmt"
"os"
"os/exec"
"path/filepath"
"strings"
"time"
"github.com/libp2p/go-libp2p/core/host"
"github.com/multiformats/go-multiaddr"
"go.uber.org/zap"
)
// UpdatePeerAddresses updates the peer_addresses in service.json with given multiaddresses
func (cm *ClusterConfigManager) UpdatePeerAddresses(addrs []string) error {
serviceJSONPath := filepath.Join(cm.clusterPath, "service.json")
cfg, err := cm.loadOrCreateConfig(serviceJSONPath)
if err != nil {
return err
}
seen := make(map[string]bool)
uniqueAddrs := []string{}
for _, addr := range addrs {
if !seen[addr] {
uniqueAddrs = append(uniqueAddrs, addr)
seen[addr] = true
}
}
cfg.Cluster.PeerAddresses = uniqueAddrs
return cm.saveConfig(serviceJSONPath, cfg)
}
// UpdateAllClusterPeers discovers all cluster peers from the gateway and updates local config
func (cm *ClusterConfigManager) UpdateAllClusterPeers() error {
peers, err := cm.DiscoverClusterPeersFromGateway()
if err != nil {
return fmt.Errorf("failed to discover cluster peers: %w", err)
}
if len(peers) == 0 {
return nil
}
peerAddrs := []string{}
for _, p := range peers {
peerAddrs = append(peerAddrs, p.Multiaddress)
}
return cm.UpdatePeerAddresses(peerAddrs)
}
// RepairPeerConfiguration attempts to fix configuration issues and re-synchronize peers
func (cm *ClusterConfigManager) RepairPeerConfiguration() error {
cm.logger.Info("Attempting to repair IPFS Cluster peer configuration")
_ = cm.FixIPFSConfigAddresses()
peers, err := cm.DiscoverClusterPeersFromGateway()
if err != nil {
cm.logger.Warn("Could not discover peers from gateway during repair", zap.Error(err))
} else {
peerAddrs := []string{}
for _, p := range peers {
peerAddrs = append(peerAddrs, p.Multiaddress)
}
if len(peerAddrs) > 0 {
_ = cm.UpdatePeerAddresses(peerAddrs)
}
}
return nil
}
// DiscoverClusterPeersFromGateway queries the central gateway for registered IPFS Cluster peers
func (cm *ClusterConfigManager) DiscoverClusterPeersFromGateway() ([]ClusterPeerInfo, error) {
// Not implemented - would require a central gateway URL in config
return nil, nil
}
// DiscoverClusterPeersFromLibP2P uses libp2p host to find other cluster peers
func (cm *ClusterConfigManager) DiscoverClusterPeersFromLibP2P(h host.Host) error {
if h == nil {
return nil
}
var clusterPeers []string
for _, p := range h.Peerstore().Peers() {
if p == h.ID() {
continue
}
info := h.Peerstore().PeerInfo(p)
for _, addr := range info.Addrs {
if strings.Contains(addr.String(), "/tcp/9096") || strings.Contains(addr.String(), "/tcp/9094") {
ma := addr.Encapsulate(multiaddr.StringCast(fmt.Sprintf("/p2p/%s", p.String())))
clusterPeers = append(clusterPeers, ma.String())
}
}
}
if len(clusterPeers) > 0 {
return cm.UpdatePeerAddresses(clusterPeers)
}
return nil
}
func (cm *ClusterConfigManager) getPeerID() (string, error) {
dataDir := cm.cfg.Node.DataDir
if strings.HasPrefix(dataDir, "~") {
home, _ := os.UserHomeDir()
dataDir = filepath.Join(home, dataDir[1:])
}
possiblePaths := []string{
filepath.Join(dataDir, "ipfs", "repo"),
filepath.Join(dataDir, "node-1", "ipfs", "repo"),
filepath.Join(dataDir, "node-2", "ipfs", "repo"),
filepath.Join(filepath.Dir(dataDir), "node-1", "ipfs", "repo"),
filepath.Join(filepath.Dir(dataDir), "node-2", "ipfs", "repo"),
}
var ipfsRepoPath string
for _, path := range possiblePaths {
if _, err := os.Stat(filepath.Join(path, "config")); err == nil {
ipfsRepoPath = path
break
}
}
if ipfsRepoPath == "" {
return "", fmt.Errorf("could not find IPFS repo path")
}
idCmd := exec.Command("ipfs", "id", "-f", "<id>")
idCmd.Env = append(os.Environ(), "IPFS_PATH="+ipfsRepoPath)
out, err := idCmd.Output()
if err != nil {
return "", err
}
return strings.TrimSpace(string(out)), nil
}
// ClusterPeerInfo represents information about an IPFS Cluster peer
type ClusterPeerInfo struct {
ID string `json:"id"`
Multiaddress string `json:"multiaddress"`
NodeName string `json:"node_name"`
LastSeen time.Time `json:"last_seen"`
}

119
pkg/ipfs/cluster_util.go Normal file
View File

@ -0,0 +1,119 @@
package ipfs
import (
"crypto/rand"
"encoding/hex"
"fmt"
"net"
"net/http"
"net/url"
"os"
"strings"
"time"
)
func loadOrGenerateClusterSecret(path string) (string, error) {
if data, err := os.ReadFile(path); err == nil {
secret := strings.TrimSpace(string(data))
if len(secret) == 64 {
return secret, nil
}
}
secret, err := generateRandomSecret()
if err != nil {
return "", err
}
_ = os.WriteFile(path, []byte(secret), 0600)
return secret, nil
}
func generateRandomSecret() (string, error) {
bytes := make([]byte, 32)
if _, err := rand.Read(bytes); err != nil {
return "", err
}
return hex.EncodeToString(bytes), nil
}
func parseClusterPorts(rawURL string) (int, int, error) {
if !strings.HasPrefix(rawURL, "http") {
rawURL = "http://" + rawURL
}
u, err := url.Parse(rawURL)
if err != nil {
return 9096, 9094, nil
}
_, portStr, err := net.SplitHostPort(u.Host)
if err != nil {
return 9096, 9094, nil
}
var port int
fmt.Sscanf(portStr, "%d", &port)
if port == 0 {
return 9096, 9094, nil
}
return port + 2, port, nil
}
func parseIPFSPort(rawURL string) (int, error) {
if !strings.HasPrefix(rawURL, "http") {
rawURL = "http://" + rawURL
}
u, err := url.Parse(rawURL)
if err != nil {
return 5001, nil
}
_, portStr, err := net.SplitHostPort(u.Host)
if err != nil {
return 5001, nil
}
var port int
fmt.Sscanf(portStr, "%d", &port)
if port == 0 {
return 5001, nil
}
return port, nil
}
func parsePeerHostAndPort(multiaddr string) (string, int) {
parts := strings.Split(multiaddr, "/")
var hostStr string
var port int
for i, part := range parts {
if part == "ip4" || part == "dns" || part == "dns4" {
hostStr = parts[i+1]
} else if part == "tcp" {
fmt.Sscanf(parts[i+1], "%d", &port)
}
}
return hostStr, port
}
func extractIPFromMultiaddrForCluster(maddr string) string {
parts := strings.Split(maddr, "/")
for i, part := range parts {
if (part == "ip4" || part == "dns" || part == "dns4") && i+1 < len(parts) {
return parts[i+1]
}
}
return ""
}
func extractDomainFromMultiaddr(maddr string) string {
parts := strings.Split(maddr, "/")
for i, part := range parts {
if (part == "dns" || part == "dns4" || part == "dns6") && i+1 < len(parts) {
return parts[i+1]
}
}
return ""
}
func newStandardHTTPClient() *http.Client {
return &http.Client{
Timeout: 10 * time.Second,
}
}

204
pkg/node/gateway.go Normal file
View File

@ -0,0 +1,204 @@
package node
import (
"context"
"crypto/tls"
"fmt"
"net"
"net/http"
"os"
"path/filepath"
"github.com/DeBrosOfficial/network/pkg/gateway"
"github.com/DeBrosOfficial/network/pkg/ipfs"
"github.com/DeBrosOfficial/network/pkg/logging"
"golang.org/x/crypto/acme"
"golang.org/x/crypto/acme/autocert"
)
// startHTTPGateway initializes and starts the full API gateway
func (n *Node) startHTTPGateway(ctx context.Context) error {
if !n.config.HTTPGateway.Enabled {
n.logger.ComponentInfo(logging.ComponentNode, "HTTP Gateway disabled in config")
return nil
}
logFile := filepath.Join(os.ExpandEnv(n.config.Node.DataDir), "..", "logs", "gateway.log")
logsDir := filepath.Dir(logFile)
_ = os.MkdirAll(logsDir, 0755)
gatewayLogger, err := logging.NewFileLogger(logging.ComponentGeneral, logFile, false)
if err != nil {
return err
}
gwCfg := &gateway.Config{
ListenAddr: n.config.HTTPGateway.ListenAddr,
ClientNamespace: n.config.HTTPGateway.ClientNamespace,
BootstrapPeers: n.config.Discovery.BootstrapPeers,
NodePeerID: loadNodePeerIDFromIdentity(n.config.Node.DataDir),
RQLiteDSN: n.config.HTTPGateway.RQLiteDSN,
OlricServers: n.config.HTTPGateway.OlricServers,
OlricTimeout: n.config.HTTPGateway.OlricTimeout,
IPFSClusterAPIURL: n.config.HTTPGateway.IPFSClusterAPIURL,
IPFSAPIURL: n.config.HTTPGateway.IPFSAPIURL,
IPFSTimeout: n.config.HTTPGateway.IPFSTimeout,
EnableHTTPS: n.config.HTTPGateway.HTTPS.Enabled,
DomainName: n.config.HTTPGateway.HTTPS.Domain,
TLSCacheDir: n.config.HTTPGateway.HTTPS.CacheDir,
}
apiGateway, err := gateway.New(gatewayLogger, gwCfg)
if err != nil {
return err
}
n.apiGateway = apiGateway
var certManager *autocert.Manager
if gwCfg.EnableHTTPS && gwCfg.DomainName != "" {
tlsCacheDir := gwCfg.TLSCacheDir
if tlsCacheDir == "" {
tlsCacheDir = "/home/debros/.orama/tls-cache"
}
_ = os.MkdirAll(tlsCacheDir, 0700)
certManager = &autocert.Manager{
Prompt: autocert.AcceptTOS,
HostPolicy: autocert.HostWhitelist(gwCfg.DomainName),
Cache: autocert.DirCache(tlsCacheDir),
Email: fmt.Sprintf("admin@%s", gwCfg.DomainName),
Client: &acme.Client{
DirectoryURL: "https://acme-staging-v02.api.letsencrypt.org/directory",
},
}
n.certManager = certManager
n.certReady = make(chan struct{})
}
httpReady := make(chan struct{})
go func() {
if gwCfg.EnableHTTPS && gwCfg.DomainName != "" && certManager != nil {
httpsPort := 443
httpPort := 80
httpServer := &http.Server{
Addr: fmt.Sprintf(":%d", httpPort),
Handler: certManager.HTTPHandler(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
target := fmt.Sprintf("https://%s%s", r.Host, r.URL.RequestURI())
http.Redirect(w, r, target, http.StatusMovedPermanently)
})),
}
httpListener, err := net.Listen("tcp", fmt.Sprintf(":%d", httpPort))
if err != nil {
close(httpReady)
return
}
go httpServer.Serve(httpListener)
// Pre-provision cert
certReq := &tls.ClientHelloInfo{ServerName: gwCfg.DomainName}
_, certErr := certManager.GetCertificate(certReq)
if certErr != nil {
close(httpReady)
httpServer.Handler = apiGateway.Routes()
return
}
close(httpReady)
tlsConfig := &tls.Config{
MinVersion: tls.VersionTLS12,
GetCertificate: certManager.GetCertificate,
}
httpsServer := &http.Server{
Addr: fmt.Sprintf(":%d", httpsPort),
TLSConfig: tlsConfig,
Handler: apiGateway.Routes(),
}
n.apiGatewayServer = httpsServer
ln, err := tls.Listen("tcp", fmt.Sprintf(":%d", httpsPort), tlsConfig)
if err == nil {
httpsServer.Serve(ln)
}
} else {
close(httpReady)
server := &http.Server{
Addr: gwCfg.ListenAddr,
Handler: apiGateway.Routes(),
}
n.apiGatewayServer = server
ln, err := net.Listen("tcp", gwCfg.ListenAddr)
if err == nil {
server.Serve(ln)
}
}
}()
// SNI Gateway
if n.config.HTTPGateway.SNI.Enabled && n.certManager != nil {
go n.startSNIGateway(ctx, httpReady)
}
return nil
}
func (n *Node) startSNIGateway(ctx context.Context, httpReady <-chan struct{}) {
<-httpReady
domain := n.config.HTTPGateway.HTTPS.Domain
if domain == "" {
return
}
certReq := &tls.ClientHelloInfo{ServerName: domain}
tlsCert, err := n.certManager.GetCertificate(certReq)
if err != nil {
return
}
tlsCacheDir := n.config.HTTPGateway.HTTPS.CacheDir
if tlsCacheDir == "" {
tlsCacheDir = "/home/debros/.orama/tls-cache"
}
certPath := filepath.Join(tlsCacheDir, domain+".crt")
keyPath := filepath.Join(tlsCacheDir, domain+".key")
if err := extractPEMFromTLSCert(tlsCert, certPath, keyPath); err == nil {
if n.certReady != nil {
close(n.certReady)
}
}
sniCfg := n.config.HTTPGateway.SNI
sniGateway, err := gateway.NewTCPSNIGateway(n.logger, &sniCfg)
if err == nil {
n.sniGateway = sniGateway
sniGateway.Start(ctx)
}
}
// startIPFSClusterConfig initializes and ensures IPFS Cluster configuration
func (n *Node) startIPFSClusterConfig() error {
n.logger.ComponentInfo(logging.ComponentNode, "Initializing IPFS Cluster configuration")
cm, err := ipfs.NewClusterConfigManager(n.config, n.logger.Logger)
if err != nil {
return err
}
n.clusterConfigManager = cm
_ = cm.FixIPFSConfigAddresses()
if err := cm.EnsureConfig(); err != nil {
return err
}
_ = cm.RepairPeerConfiguration()
return nil
}

302
pkg/node/libp2p.go Normal file
View File

@ -0,0 +1,302 @@
package node
import (
"context"
"fmt"
"os"
"path/filepath"
"strings"
"time"
"github.com/DeBrosOfficial/network/pkg/discovery"
"github.com/DeBrosOfficial/network/pkg/encryption"
"github.com/DeBrosOfficial/network/pkg/logging"
"github.com/DeBrosOfficial/network/pkg/pubsub"
"github.com/libp2p/go-libp2p"
libp2ppubsub "github.com/libp2p/go-libp2p-pubsub"
"github.com/libp2p/go-libp2p/core/crypto"
"github.com/libp2p/go-libp2p/core/peer"
noise "github.com/libp2p/go-libp2p/p2p/security/noise"
"github.com/multiformats/go-multiaddr"
"go.uber.org/zap"
)
// startLibP2P initializes the LibP2P host
func (n *Node) startLibP2P() error {
n.logger.ComponentInfo(logging.ComponentLibP2P, "Starting LibP2P host")
// Load or create persistent identity
identity, err := n.loadOrCreateIdentity()
if err != nil {
return fmt.Errorf("failed to load identity: %w", err)
}
// Create LibP2P host with explicit listen addresses
var opts []libp2p.Option
opts = append(opts,
libp2p.Identity(identity),
libp2p.Security(noise.ID, noise.New),
libp2p.DefaultMuxers,
)
// Add explicit listen addresses from config
if len(n.config.Node.ListenAddresses) > 0 {
listenAddrs := make([]multiaddr.Multiaddr, 0, len(n.config.Node.ListenAddresses))
for _, addr := range n.config.Node.ListenAddresses {
ma, err := multiaddr.NewMultiaddr(addr)
if err != nil {
return fmt.Errorf("invalid listen address %s: %w", addr, err)
}
listenAddrs = append(listenAddrs, ma)
}
opts = append(opts, libp2p.ListenAddrs(listenAddrs...))
n.logger.ComponentInfo(logging.ComponentLibP2P, "Configured listen addresses",
zap.Strings("addrs", n.config.Node.ListenAddresses))
}
// For localhost/development, disable NAT services
isLocalhost := len(n.config.Node.ListenAddresses) > 0 &&
(strings.Contains(n.config.Node.ListenAddresses[0], "localhost") ||
strings.Contains(n.config.Node.ListenAddresses[0], "127.0.0.1"))
if isLocalhost {
n.logger.ComponentInfo(logging.ComponentLibP2P, "Localhost detected - disabling NAT services for local development")
} else {
n.logger.ComponentInfo(logging.ComponentLibP2P, "Production mode - enabling NAT services")
opts = append(opts,
libp2p.EnableNATService(),
libp2p.EnableAutoNATv2(),
libp2p.EnableRelay(),
libp2p.NATPortMap(),
libp2p.EnableAutoRelayWithPeerSource(
peerSource(n.config.Discovery.BootstrapPeers, n.logger.Logger),
),
)
}
h, err := libp2p.New(opts...)
if err != nil {
return err
}
n.host = h
// Initialize pubsub
ps, err := libp2ppubsub.NewGossipSub(context.Background(), h,
libp2ppubsub.WithPeerExchange(true),
libp2ppubsub.WithFloodPublish(true),
libp2ppubsub.WithDirectPeers(nil),
)
if err != nil {
return fmt.Errorf("failed to create pubsub: %w", err)
}
// Create pubsub adapter
n.pubsub = pubsub.NewClientAdapter(ps, n.config.Discovery.NodeNamespace)
n.logger.Info("Initialized pubsub adapter on namespace", zap.String("namespace", n.config.Discovery.NodeNamespace))
// Connect to peers
if err := n.connectToPeers(context.Background()); err != nil {
n.logger.ComponentWarn(logging.ComponentNode, "Failed to connect to peers", zap.Error(err))
}
// Start reconnection loop
if len(n.config.Discovery.BootstrapPeers) > 0 {
peerCtx, cancel := context.WithCancel(context.Background())
n.peerDiscoveryCancel = cancel
go n.peerReconnectionLoop(peerCtx)
}
// Add peers to peerstore
for _, peerAddr := range n.config.Discovery.BootstrapPeers {
if ma, err := multiaddr.NewMultiaddr(peerAddr); err == nil {
if peerInfo, err := peer.AddrInfoFromP2pAddr(ma); err == nil {
n.host.Peerstore().AddAddrs(peerInfo.ID, peerInfo.Addrs, time.Hour*24)
}
}
}
// Initialize discovery manager
n.discoveryManager = discovery.NewManager(h, nil, n.logger.Logger)
n.discoveryManager.StartProtocolHandler()
n.logger.ComponentInfo(logging.ComponentNode, "LibP2P host started successfully")
// Start peer discovery
n.startPeerDiscovery()
return nil
}
func (n *Node) peerReconnectionLoop(ctx context.Context) {
interval := 5 * time.Second
consecutiveFailures := 0
for {
select {
case <-ctx.Done():
return
default:
}
if !n.hasPeerConnections() {
if err := n.connectToPeers(context.Background()); err != nil {
consecutiveFailures++
jitteredInterval := addJitter(interval)
select {
case <-ctx.Done():
return
case <-time.After(jitteredInterval):
}
interval = calculateNextBackoff(interval)
} else {
interval = 5 * time.Second
consecutiveFailures = 0
select {
case <-ctx.Done():
return
case <-time.After(30 * time.Second):
}
}
} else {
select {
case <-ctx.Done():
return
case <-time.After(30 * time.Second):
}
}
}
}
func (n *Node) connectToPeers(ctx context.Context) error {
for _, peerAddr := range n.config.Discovery.BootstrapPeers {
if err := n.connectToPeerAddr(ctx, peerAddr); err != nil {
continue
}
}
return nil
}
func (n *Node) connectToPeerAddr(ctx context.Context, addr string) error {
ma, err := multiaddr.NewMultiaddr(addr)
if err != nil {
return err
}
peerInfo, err := peer.AddrInfoFromP2pAddr(ma)
if err != nil {
return err
}
if n.host != nil && peerInfo.ID == n.host.ID() {
return nil
}
return n.host.Connect(ctx, *peerInfo)
}
func (n *Node) hasPeerConnections() bool {
if n.host == nil || len(n.config.Discovery.BootstrapPeers) == 0 {
return false
}
connectedPeers := n.host.Network().Peers()
if len(connectedPeers) == 0 {
return false
}
bootstrapIDs := make(map[peer.ID]bool)
for _, addr := range n.config.Discovery.BootstrapPeers {
if ma, err := multiaddr.NewMultiaddr(addr); err == nil {
if info, err := peer.AddrInfoFromP2pAddr(ma); err == nil {
bootstrapIDs[info.ID] = true
}
}
}
for _, p := range connectedPeers {
if bootstrapIDs[p] {
return true
}
}
return false
}
func (n *Node) loadOrCreateIdentity() (crypto.PrivKey, error) {
identityFile := filepath.Join(os.ExpandEnv(n.config.Node.DataDir), "identity.key")
if strings.HasPrefix(identityFile, "~") {
home, _ := os.UserHomeDir()
identityFile = filepath.Join(home, identityFile[1:])
}
if _, err := os.Stat(identityFile); err == nil {
info, err := encryption.LoadIdentity(identityFile)
if err == nil {
return info.PrivateKey, nil
}
}
info, err := encryption.GenerateIdentity()
if err != nil {
return nil, err
}
if err := encryption.SaveIdentity(info, identityFile); err != nil {
return nil, err
}
return info.PrivateKey, nil
}
func (n *Node) startPeerDiscovery() {
if n.discoveryManager == nil {
return
}
discoveryConfig := discovery.Config{
DiscoveryInterval: n.config.Discovery.DiscoveryInterval,
MaxConnections: n.config.Node.MaxConnections,
}
n.discoveryManager.Start(discoveryConfig)
}
func (n *Node) stopPeerDiscovery() {
if n.discoveryManager != nil {
n.discoveryManager.Stop()
}
}
func (n *Node) GetPeerID() string {
if n.host == nil {
return ""
}
return n.host.ID().String()
}
func peerSource(peerAddrs []string, logger *zap.Logger) func(context.Context, int) <-chan peer.AddrInfo {
return func(ctx context.Context, num int) <-chan peer.AddrInfo {
out := make(chan peer.AddrInfo, num)
go func() {
defer close(out)
count := 0
for _, s := range peerAddrs {
if count >= num {
return
}
ma, err := multiaddr.NewMultiaddr(s)
if err != nil {
continue
}
ai, err := peer.AddrInfoFromP2pAddr(ma)
if err != nil {
continue
}
select {
case out <- *ai:
count++
case <-ctx.Done():
return
}
}
}()
return out
}
}

12
pkg/node/monitoring.go
View File

@ -220,9 +220,9 @@ func (n *Node) startConnectionMonitoring() {
// First try to discover from LibP2P connections (works even if cluster peers aren't connected yet)
// This runs every minute to discover peers automatically via LibP2P discovery
if time.Now().Unix()%60 == 0 {
if success, err := n.clusterConfigManager.DiscoverClusterPeersFromLibP2P(n.host); err != nil {
if err := n.clusterConfigManager.DiscoverClusterPeersFromLibP2P(n.host); err != nil {
n.logger.ComponentWarn(logging.ComponentNode, "Failed to discover cluster peers from LibP2P", zap.Error(err))
} else if success {
} else {
n.logger.ComponentInfo(logging.ComponentNode, "Cluster peer addresses discovered from LibP2P")
}
}
@ -230,16 +230,16 @@ func (n *Node) startConnectionMonitoring() {
// Also try to update from cluster API (works once peers are connected)
// Update all cluster peers every 2 minutes to discover new peers
if time.Now().Unix()%120 == 0 {
if success, err := n.clusterConfigManager.UpdateAllClusterPeers(); err != nil {
if err := n.clusterConfigManager.UpdateAllClusterPeers(); err != nil {
n.logger.ComponentWarn(logging.ComponentNode, "Failed to update cluster peers during monitoring", zap.Error(err))
} else if success {
} else {
n.logger.ComponentInfo(logging.ComponentNode, "Cluster peer addresses updated during monitoring")
}
// Try to repair peer configuration
if success, err := n.clusterConfigManager.RepairPeerConfiguration(); err != nil {
if err := n.clusterConfigManager.RepairPeerConfiguration(); err != nil {
n.logger.ComponentWarn(logging.ComponentNode, "Failed to repair peer addresses during monitoring", zap.Error(err))
} else if success {
} else {
n.logger.ComponentInfo(logging.ComponentNode, "Peer configuration repaired during monitoring")
}
}

1172
pkg/node/node.go
View File

File diff suppressed because it is too large Load Diff

98
pkg/node/rqlite.go Normal file
View File

@ -0,0 +1,98 @@
package node
import (
"context"
"fmt"
database "github.com/DeBrosOfficial/network/pkg/rqlite"
"go.uber.org/zap"
"time"
)
// startRQLite initializes and starts the RQLite database
func (n *Node) startRQLite(ctx context.Context) error {
n.logger.Info("Starting RQLite database")
// Determine node identifier for log filename - use node ID for unique filenames
nodeID := n.config.Node.ID
if nodeID == "" {
// Default to "node" if ID is not set
nodeID = "node"
}
// Create RQLite manager
n.rqliteManager = database.NewRQLiteManager(&n.config.Database, &n.config.Discovery, n.config.Node.DataDir, n.logger.Logger)
n.rqliteManager.SetNodeType(nodeID)
// Initialize cluster discovery service if LibP2P host is available
if n.host != nil && n.discoveryManager != nil {
// Create cluster discovery service (all nodes are unified)
n.clusterDiscovery = database.NewClusterDiscoveryService(
n.host,
n.discoveryManager,
n.rqliteManager,
n.config.Node.ID,
"node", // Unified node type
n.config.Discovery.RaftAdvAddress,
n.config.Discovery.HttpAdvAddress,
n.config.Node.DataDir,
n.logger.Logger,
)
// Set discovery service on RQLite manager BEFORE starting RQLite
// This is critical for pre-start cluster discovery during recovery
n.rqliteManager.SetDiscoveryService(n.clusterDiscovery)
// Start cluster discovery (but don't trigger initial sync yet)
if err := n.clusterDiscovery.Start(ctx); err != nil {
return fmt.Errorf("failed to start cluster discovery: %w", err)
}
// Publish initial metadata (with log_index=0) so peers can discover us during recovery
// The metadata will be updated with actual log index after RQLite starts
n.clusterDiscovery.UpdateOwnMetadata()
n.logger.Info("Cluster discovery service started (waiting for RQLite)")
}
// If node-to-node TLS is configured, wait for certificates to be provisioned
// This ensures RQLite can start with TLS when joining through the SNI gateway
if n.config.Database.NodeCert != "" && n.config.Database.NodeKey != "" && n.certReady != nil {
n.logger.Info("RQLite node TLS configured, waiting for certificates to be provisioned...",
zap.String("node_cert", n.config.Database.NodeCert),
zap.String("node_key", n.config.Database.NodeKey))
// Wait for certificate ready signal with timeout
certTimeout := 5 * time.Minute
select {
case <-n.certReady:
n.logger.Info("Certificates ready, proceeding with RQLite startup")
case <-time.After(certTimeout):
return fmt.Errorf("timeout waiting for TLS certificates after %v - ensure HTTPS is configured and ports 80/443 are accessible for ACME challenges", certTimeout)
case <-ctx.Done():
return fmt.Errorf("context cancelled while waiting for certificates: %w", ctx.Err())
}
}
// Start RQLite FIRST before updating metadata
if err := n.rqliteManager.Start(ctx); err != nil {
return err
}
// NOW update metadata after RQLite is running
if n.clusterDiscovery != nil {
n.clusterDiscovery.UpdateOwnMetadata()
n.clusterDiscovery.TriggerSync() // Do initial cluster sync now that RQLite is ready
n.logger.Info("RQLite metadata published and cluster synced")
}
// Create adapter for sql.DB compatibility
adapter, err := database.NewRQLiteAdapter(n.rqliteManager)
if err != nil {
return fmt.Errorf("failed to create RQLite adapter: %w", err)
}
n.rqliteAdapter = adapter
return nil
}

127
pkg/node/utils.go Normal file
View File

@ -0,0 +1,127 @@
package node
import (
"crypto/tls"
"crypto/x509"
"encoding/pem"
"fmt"
mathrand "math/rand"
"net"
"os"
"path/filepath"
"strings"
"time"
"github.com/DeBrosOfficial/network/pkg/encryption"
"github.com/multiformats/go-multiaddr"
)
func extractIPFromMultiaddr(multiaddrStr string) string {
ma, err := multiaddr.NewMultiaddr(multiaddrStr)
if err != nil {
return ""
}
var ip string
var dnsName string
multiaddr.ForEach(ma, func(c multiaddr.Component) bool {
switch c.Protocol().Code {
case multiaddr.P_IP4, multiaddr.P_IP6:
ip = c.Value()
return false
case multiaddr.P_DNS4, multiaddr.P_DNS6, multiaddr.P_DNSADDR:
dnsName = c.Value()
}
return true
})
if ip != "" {
return ip
}
if dnsName != "" {
if resolvedIPs, err := net.LookupIP(dnsName); err == nil && len(resolvedIPs) > 0 {
for _, resolvedIP := range resolvedIPs {
if resolvedIP.To4() != nil {
return resolvedIP.String()
}
}
return resolvedIPs[0].String()
}
}
return ""
}
func calculateNextBackoff(current time.Duration) time.Duration {
next := time.Duration(float64(current) * 1.5)
maxInterval := 10 * time.Minute
if next > maxInterval {
next = maxInterval
}
return next
}
func addJitter(interval time.Duration) time.Duration {
jitterPercent := 0.2
jitterRange := float64(interval) * jitterPercent
jitter := (mathrand.Float64() - 0.5) * 2 * jitterRange
result := time.Duration(float64(interval) + jitter)
if result < time.Second {
result = time.Second
}
return result
}
func loadNodePeerIDFromIdentity(dataDir string) string {
identityFile := filepath.Join(os.ExpandEnv(dataDir), "identity.key")
if strings.HasPrefix(identityFile, "~") {
home, _ := os.UserHomeDir()
identityFile = filepath.Join(home, identityFile[1:])
}
if info, err := encryption.LoadIdentity(identityFile); err == nil {
return info.PeerID.String()
}
return ""
}
func extractPEMFromTLSCert(tlsCert *tls.Certificate, certPath, keyPath string) error {
if tlsCert == nil || len(tlsCert.Certificate) == 0 {
return fmt.Errorf("invalid tls certificate")
}
certFile, err := os.Create(certPath)
if err != nil {
return err
}
defer certFile.Close()
for _, certBytes := range tlsCert.Certificate {
pem.Encode(certFile, &pem.Block{Type: "CERTIFICATE", Bytes: certBytes})
}
if tlsCert.PrivateKey == nil {
return fmt.Errorf("private key is nil")
}
keyFile, err := os.Create(keyPath)
if err != nil {
return err
}
defer keyFile.Close()
var keyBytes []byte
switch key := tlsCert.PrivateKey.(type) {
case *x509.Certificate:
keyBytes, _ = x509.MarshalPKCS8PrivateKey(key)
default:
keyBytes, _ = x509.MarshalPKCS8PrivateKey(tlsCert.PrivateKey)
}
pem.Encode(keyFile, &pem.Block{Type: "PRIVATE KEY", Bytes: keyBytes})
os.Chmod(certPath, 0644)
os.Chmod(keyPath, 0600)
return nil
}

301
pkg/rqlite/cluster.go Normal file
View File

@ -0,0 +1,301 @@
package rqlite
import (
"context"
"encoding/json"
"fmt"
"net/http"
"os"
"path/filepath"
"strings"
"time"
)
// establishLeadershipOrJoin handles post-startup cluster establishment
func (r *RQLiteManager) establishLeadershipOrJoin(ctx context.Context, rqliteDataDir string) error {
timeout := 5 * time.Minute
if r.config.RQLiteJoinAddress == "" {
timeout = 2 * time.Minute
}
sqlCtx, cancel := context.WithTimeout(ctx, timeout)
defer cancel()
if err := r.waitForSQLAvailable(sqlCtx); err != nil {
if r.cmd != nil && r.cmd.Process != nil {
_ = r.cmd.Process.Kill()
}
return err
}
return nil
}
// waitForMinClusterSizeBeforeStart waits for minimum cluster size to be discovered
func (r *RQLiteManager) waitForMinClusterSizeBeforeStart(ctx context.Context, rqliteDataDir string) error {
if r.discoveryService == nil {
return fmt.Errorf("discovery service not available")
}
requiredRemotePeers := r.config.MinClusterSize - 1
_ = r.discoveryService.TriggerPeerExchange(ctx)
checkInterval := 2 * time.Second
for {
select {
case <-ctx.Done():
return ctx.Err()
default:
}
r.discoveryService.TriggerSync()
time.Sleep(checkInterval)
allPeers := r.discoveryService.GetAllPeers()
remotePeerCount := 0
for _, peer := range allPeers {
if peer.NodeID != r.discoverConfig.RaftAdvAddress {
remotePeerCount++
}
}
if remotePeerCount >= requiredRemotePeers {
peersPath := filepath.Join(rqliteDataDir, "raft", "peers.json")
r.discoveryService.TriggerSync()
time.Sleep(2 * time.Second)
if info, err := os.Stat(peersPath); err == nil && info.Size() > 10 {
data, err := os.ReadFile(peersPath)
if err == nil {
var peers []map[string]interface{}
if err := json.Unmarshal(data, &peers); err == nil && len(peers) >= requiredRemotePeers {
return nil
}
}
}
}
}
}
// performPreStartClusterDiscovery builds peers.json before starting RQLite
func (r *RQLiteManager) performPreStartClusterDiscovery(ctx context.Context, rqliteDataDir string) error {
if r.discoveryService == nil {
return fmt.Errorf("discovery service not available")
}
_ = r.discoveryService.TriggerPeerExchange(ctx)
time.Sleep(1 * time.Second)
r.discoveryService.TriggerSync()
time.Sleep(2 * time.Second)
discoveryDeadline := time.Now().Add(30 * time.Second)
var discoveredPeers int
for time.Now().Before(discoveryDeadline) {
allPeers := r.discoveryService.GetAllPeers()
discoveredPeers = len(allPeers)
if discoveredPeers >= r.config.MinClusterSize {
break
}
time.Sleep(2 * time.Second)
}
if discoveredPeers <= 1 {
return nil
}
if r.hasExistingRaftState(rqliteDataDir) {
ourLogIndex := r.getRaftLogIndex()
maxPeerIndex := uint64(0)
for _, peer := range r.discoveryService.GetAllPeers() {
if peer.NodeID != r.discoverConfig.RaftAdvAddress && peer.RaftLogIndex > maxPeerIndex {
maxPeerIndex = peer.RaftLogIndex
}
}
if ourLogIndex == 0 && maxPeerIndex > 0 {
_ = r.clearRaftState(rqliteDataDir)
_ = r.discoveryService.ForceWritePeersJSON()
}
}
r.discoveryService.TriggerSync()
time.Sleep(2 * time.Second)
return nil
}
// recoverCluster restarts RQLite using peers.json
func (r *RQLiteManager) recoverCluster(ctx context.Context, peersJSONPath string) error {
_ = r.Stop()
time.Sleep(2 * time.Second)
rqliteDataDir, err := r.rqliteDataDirPath()
if err != nil {
return err
}
if err := r.launchProcess(ctx, rqliteDataDir); err != nil {
return err
}
return r.waitForReadyAndConnect(ctx)
}
// recoverFromSplitBrain automatically recovers from split-brain state
func (r *RQLiteManager) recoverFromSplitBrain(ctx context.Context) error {
if r.discoveryService == nil {
return fmt.Errorf("discovery service not available")
}
r.discoveryService.TriggerPeerExchange(ctx)
time.Sleep(2 * time.Second)
r.discoveryService.TriggerSync()
time.Sleep(2 * time.Second)
rqliteDataDir, _ := r.rqliteDataDirPath()
ourIndex := r.getRaftLogIndex()
maxPeerIndex := uint64(0)
for _, peer := range r.discoveryService.GetAllPeers() {
if peer.NodeID != r.discoverConfig.RaftAdvAddress && peer.RaftLogIndex > maxPeerIndex {
maxPeerIndex = peer.RaftLogIndex
}
}
if ourIndex == 0 && maxPeerIndex > 0 {
_ = r.clearRaftState(rqliteDataDir)
r.discoveryService.TriggerPeerExchange(ctx)
time.Sleep(1 * time.Second)
_ = r.discoveryService.ForceWritePeersJSON()
return r.recoverCluster(ctx, filepath.Join(rqliteDataDir, "raft", "peers.json"))
}
return nil
}
// isInSplitBrainState detects if we're in a split-brain scenario
func (r *RQLiteManager) isInSplitBrainState() bool {
status, err := r.getRQLiteStatus()
if err != nil || r.discoveryService == nil {
return false
}
raft := status.Store.Raft
if raft.State == "Follower" && raft.Term == 0 && raft.NumPeers == 0 && !raft.Voter {
peers := r.discoveryService.GetActivePeers()
if len(peers) == 0 {
return false
}
reachableCount := 0
splitBrainCount := 0
for _, peer := range peers {
if r.isPeerReachable(peer.HTTPAddress) {
reachableCount++
peerStatus, err := r.getPeerRQLiteStatus(peer.HTTPAddress)
if err == nil {
praft := peerStatus.Store.Raft
if praft.State == "Follower" && praft.Term == 0 && praft.NumPeers == 0 && !praft.Voter {
splitBrainCount++
}
}
}
}
return reachableCount > 0 && splitBrainCount == reachableCount
}
return false
}
func (r *RQLiteManager) isPeerReachable(httpAddr string) bool {
client := &http.Client{Timeout: 3 * time.Second}
resp, err := client.Get(fmt.Sprintf("http://%s/status", httpAddr))
if err == nil {
resp.Body.Close()
return resp.StatusCode == http.StatusOK
}
return false
}
func (r *RQLiteManager) getPeerRQLiteStatus(httpAddr string) (*RQLiteStatus, error) {
client := &http.Client{Timeout: 3 * time.Second}
resp, err := client.Get(fmt.Sprintf("http://%s/status", httpAddr))
if err != nil {
return nil, err
}
defer resp.Body.Close()
var status RQLiteStatus
if err := json.NewDecoder(resp.Body).Decode(&status); err != nil {
return nil, err
}
return &status, nil
}
func (r *RQLiteManager) startHealthMonitoring(ctx context.Context) {
time.Sleep(30 * time.Second)
ticker := time.NewTicker(60 * time.Second)
defer ticker.Stop()
for {
select {
case <-ctx.Done():
return
case <-ticker.C:
if r.isInSplitBrainState() {
_ = r.recoverFromSplitBrain(ctx)
}
}
}
}
// checkNeedsClusterRecovery checks if the node has old cluster state that requires coordinated recovery
func (r *RQLiteManager) checkNeedsClusterRecovery(rqliteDataDir string) (bool, error) {
snapshotsDir := filepath.Join(rqliteDataDir, "rsnapshots")
if _, err := os.Stat(snapshotsDir); os.IsNotExist(err) {
return false, nil
}
entries, err := os.ReadDir(snapshotsDir)
if err != nil {
return false, err
}
hasSnapshots := false
for _, entry := range entries {
if entry.IsDir() || strings.HasSuffix(entry.Name(), ".db") {
hasSnapshots = true
break
}
}
if !hasSnapshots {
return false, nil
}
raftLogPath := filepath.Join(rqliteDataDir, "raft.db")
if info, err := os.Stat(raftLogPath); err == nil {
if info.Size() <= 8*1024*1024 {
return true, nil
}
}
return false, nil
}
func (r *RQLiteManager) hasExistingRaftState(rqliteDataDir string) bool {
raftLogPath := filepath.Join(rqliteDataDir, "raft.db")
if info, err := os.Stat(raftLogPath); err == nil && info.Size() > 1024 {
return true
}
peersPath := filepath.Join(rqliteDataDir, "raft", "peers.json")
_, err := os.Stat(peersPath)
return err == nil
}
func (r *RQLiteManager) clearRaftState(rqliteDataDir string) error {
_ = os.Remove(filepath.Join(rqliteDataDir, "raft.db"))
_ = os.Remove(filepath.Join(rqliteDataDir, "raft", "peers.json"))
return nil
}

860
pkg/rqlite/cluster_discovery.go
View File

@ -2,20 +2,12 @@ package rqlite
import (
"context"
"encoding/json"
"fmt"
"net"
"net/netip"
"os"
"path/filepath"
"strings"
"sync"
"time"
"github.com/DeBrosOfficial/network/pkg/discovery"
"github.com/libp2p/go-libp2p/core/host"
"github.com/libp2p/go-libp2p/core/peer"
"github.com/multiformats/go-multiaddr"
"go.uber.org/zap"
)
@ -160,855 +152,3 @@ func (c *ClusterDiscoveryService) periodicCleanup(ctx context.Context) {
}
}
}
// collectPeerMetadata collects RQLite metadata from LibP2P peers
func (c *ClusterDiscoveryService) collectPeerMetadata() []*discovery.RQLiteNodeMetadata {
connectedPeers := c.host.Network().Peers()
var metadata []*discovery.RQLiteNodeMetadata
// Metadata collection is routine - no need to log every occurrence
c.mu.RLock()
currentRaftAddr := c.raftAddress
currentHTTPAddr := c.httpAddress
c.mu.RUnlock()
// Add ourselves
ourMetadata := &discovery.RQLiteNodeMetadata{
NodeID: currentRaftAddr, // RQLite uses raft address as node ID
RaftAddress: currentRaftAddr,
HTTPAddress: currentHTTPAddr,
NodeType: c.nodeType,
RaftLogIndex: c.rqliteManager.getRaftLogIndex(),
LastSeen: time.Now(),
ClusterVersion: "1.0",
}
if c.adjustSelfAdvertisedAddresses(ourMetadata) {
c.logger.Debug("Adjusted self-advertised RQLite addresses",
zap.String("raft_address", ourMetadata.RaftAddress),
zap.String("http_address", ourMetadata.HTTPAddress))
}
metadata = append(metadata, ourMetadata)
staleNodeIDs := make([]string, 0)
// Query connected peers for their RQLite metadata
// For now, we'll use a simple approach - store metadata in peer metadata store
// In a full implementation, this would use a custom protocol to exchange RQLite metadata
for _, peerID := range connectedPeers {
// Try to get stored metadata from peerstore
// This would be populated by a peer exchange protocol
if val, err := c.host.Peerstore().Get(peerID, "rqlite_metadata"); err == nil {
if jsonData, ok := val.([]byte); ok {
var peerMeta discovery.RQLiteNodeMetadata
if err := json.Unmarshal(jsonData, &peerMeta); err == nil {
if updated, stale := c.adjustPeerAdvertisedAddresses(peerID, &peerMeta); updated && stale != "" {
staleNodeIDs = append(staleNodeIDs, stale)
}
peerMeta.LastSeen = time.Now()
metadata = append(metadata, &peerMeta)
}
}
}
}
// Clean up stale entries if NodeID changed
if len(staleNodeIDs) > 0 {
c.mu.Lock()
for _, id := range staleNodeIDs {
delete(c.knownPeers, id)
delete(c.peerHealth, id)
}
c.mu.Unlock()
}
return metadata
}
// membershipUpdateResult contains the result of a membership update operation
type membershipUpdateResult struct {
peersJSON []map[string]interface{}
added []string
updated []string
changed bool
}
// updateClusterMembership updates the cluster membership based on discovered peers
func (c *ClusterDiscoveryService) updateClusterMembership() {
metadata := c.collectPeerMetadata()
// Compute membership changes while holding lock
c.mu.Lock()
result := c.computeMembershipChangesLocked(metadata)
c.mu.Unlock()
// Perform file I/O outside the lock
if result.changed {
// Log state changes (peer added/removed) at Info level
if len(result.added) > 0 || len(result.updated) > 0 {
c.logger.Info("Membership changed",
zap.Int("added", len(result.added)),
zap.Int("updated", len(result.updated)),
zap.Strings("added", result.added),
zap.Strings("updated", result.updated))
}
// Write peers.json without holding lock
if err := c.writePeersJSONWithData(result.peersJSON); err != nil {
c.logger.Error("Failed to write peers.json",
zap.Error(err),
zap.String("data_dir", c.dataDir),
zap.Int("peers", len(result.peersJSON)))
} else {
c.logger.Debug("peers.json updated",
zap.Int("peers", len(result.peersJSON)))
}
// Update lastUpdate timestamp
c.mu.Lock()
c.lastUpdate = time.Now()
c.mu.Unlock()
}
// No changes - don't log (reduces noise)
}
// computeMembershipChangesLocked computes membership changes and returns snapshot data
// Must be called with lock held
func (c *ClusterDiscoveryService) computeMembershipChangesLocked(metadata []*discovery.RQLiteNodeMetadata) membershipUpdateResult {
// Track changes
added := []string{}
updated := []string{}
// Update known peers, but skip self for health tracking
for _, meta := range metadata {
// Skip self-metadata for health tracking (we only track remote peers)
isSelf := meta.NodeID == c.raftAddress
if existing, ok := c.knownPeers[meta.NodeID]; ok {
// Update existing peer
if existing.RaftLogIndex != meta.RaftLogIndex ||
existing.HTTPAddress != meta.HTTPAddress ||
existing.RaftAddress != meta.RaftAddress {
updated = append(updated, meta.NodeID)
}
} else {
// New peer discovered
added = append(added, meta.NodeID)
c.logger.Info("Node added",
zap.String("node", meta.NodeID),
zap.String("raft", meta.RaftAddress),
zap.String("type", meta.NodeType),
zap.Uint64("log_index", meta.RaftLogIndex))
}
c.knownPeers[meta.NodeID] = meta
// Update health tracking only for remote peers
if !isSelf {
if _, ok := c.peerHealth[meta.NodeID]; !ok {
c.peerHealth[meta.NodeID] = &PeerHealth{
LastSeen: time.Now(),
LastSuccessful: time.Now(),
Status: "active",
}
} else {
c.peerHealth[meta.NodeID].LastSeen = time.Now()
c.peerHealth[meta.NodeID].Status = "active"
c.peerHealth[meta.NodeID].FailureCount = 0
}
}
}
// CRITICAL FIX: Count remote peers (excluding self)
remotePeerCount := 0
for _, peer := range c.knownPeers {
if peer.NodeID != c.raftAddress {
remotePeerCount++
}
}
// Get peers JSON snapshot (for checking if it would be empty)
peers := c.getPeersJSONUnlocked()
// Determine if we should write peers.json
shouldWrite := len(added) > 0 || len(updated) > 0 || c.lastUpdate.IsZero()
// CRITICAL FIX: Don't write peers.json until we have minimum cluster size
// This prevents RQLite from starting as a single-node cluster
// For min_cluster_size=3, we need at least 2 remote peers (plus self = 3 total)
if shouldWrite {
// For initial sync, wait until we have at least (MinClusterSize - 1) remote peers
// This ensures peers.json contains enough peers for proper cluster formation
if c.lastUpdate.IsZero() {
requiredRemotePeers := c.minClusterSize - 1
if remotePeerCount < requiredRemotePeers {
c.logger.Info("Waiting for peers",
zap.Int("have", remotePeerCount),
zap.Int("need", requiredRemotePeers),
zap.Int("min_size", c.minClusterSize))
return membershipUpdateResult{
changed: false,
}
}
}
// Additional safety check: don't write empty peers.json (would cause single-node cluster)
if len(peers) == 0 && c.lastUpdate.IsZero() {
c.logger.Info("No remote peers - waiting")
return membershipUpdateResult{
changed: false,
}
}
// Log initial sync if this is the first time
if c.lastUpdate.IsZero() {
c.logger.Info("Initial sync",
zap.Int("total", len(c.knownPeers)),
zap.Int("remote", remotePeerCount),
zap.Int("in_json", len(peers)))
}
return membershipUpdateResult{
peersJSON: peers,
added: added,
updated: updated,
changed: true,
}
}
return membershipUpdateResult{
changed: false,
}
}
// removeInactivePeers removes peers that haven't been seen for longer than the inactivity limit
func (c *ClusterDiscoveryService) removeInactivePeers() {
c.mu.Lock()
defer c.mu.Unlock()
now := time.Now()
removed := []string{}
for nodeID, health := range c.peerHealth {
inactiveDuration := now.Sub(health.LastSeen)
if inactiveDuration > c.inactivityLimit {
// Mark as inactive and remove
c.logger.Warn("Node removed",
zap.String("node", nodeID),
zap.String("reason", "inactive"),
zap.Duration("inactive_duration", inactiveDuration))
delete(c.knownPeers, nodeID)
delete(c.peerHealth, nodeID)
removed = append(removed, nodeID)
}
}
// Regenerate peers.json if any peers were removed
if len(removed) > 0 {
c.logger.Info("Removed inactive",
zap.Int("count", len(removed)),
zap.Strings("nodes", removed))
if err := c.writePeersJSON(); err != nil {
c.logger.Error("Failed to write peers.json after cleanup", zap.Error(err))
}
}
}
// getPeersJSON generates the peers.json structure from active peers (acquires lock)
func (c *ClusterDiscoveryService) getPeersJSON() []map[string]interface{} {
c.mu.RLock()
defer c.mu.RUnlock()
return c.getPeersJSONUnlocked()
}
// getPeersJSONUnlocked generates the peers.json structure (must be called with lock held)
func (c *ClusterDiscoveryService) getPeersJSONUnlocked() []map[string]interface{} {
peers := make([]map[string]interface{}, 0, len(c.knownPeers))
for _, peer := range c.knownPeers {
// CRITICAL FIX: Include ALL peers (including self) in peers.json
// When using expect configuration with recovery, RQLite needs the complete
// expected cluster configuration to properly form consensus.
// The peers.json file is used by RQLite's recovery mechanism to know
// what the full cluster membership should be, including the local node.
peerEntry := map[string]interface{}{
"id": peer.RaftAddress, // RQLite uses raft address as node ID
"address": peer.RaftAddress,
"non_voter": false,
}
peers = append(peers, peerEntry)
}
return peers
}
// writePeersJSON atomically writes the peers.json file (acquires lock)
func (c *ClusterDiscoveryService) writePeersJSON() error {
c.mu.RLock()
peers := c.getPeersJSONUnlocked()
c.mu.RUnlock()
return c.writePeersJSONWithData(peers)
}
// writePeersJSONWithData writes the peers.json file with provided data (no lock needed)
func (c *ClusterDiscoveryService) writePeersJSONWithData(peers []map[string]interface{}) error {
// Expand ~ in data directory path
dataDir := os.ExpandEnv(c.dataDir)
if strings.HasPrefix(dataDir, "~") {
home, err := os.UserHomeDir()
if err != nil {
return fmt.Errorf("failed to determine home directory: %w", err)
}
dataDir = filepath.Join(home, dataDir[1:])
}
// Get the RQLite raft directory
rqliteDir := filepath.Join(dataDir, "rqlite", "raft")
// Writing peers.json - routine operation, no need to log details
if err := os.MkdirAll(rqliteDir, 0755); err != nil {
return fmt.Errorf("failed to create raft directory %s: %w", rqliteDir, err)
}
peersFile := filepath.Join(rqliteDir, "peers.json")
backupFile := filepath.Join(rqliteDir, "peers.json.backup")
// Backup existing peers.json if it exists
if _, err := os.Stat(peersFile); err == nil {
// Backup existing peers.json if it exists - routine operation
data, err := os.ReadFile(peersFile)
if err == nil {
_ = os.WriteFile(backupFile, data, 0644)
}
}
// Marshal to JSON
data, err := json.MarshalIndent(peers, "", " ")
if err != nil {
return fmt.Errorf("failed to marshal peers.json: %w", err)
}
// Marshaled peers.json - routine operation
// Write atomically using temp file + rename
tempFile := peersFile + ".tmp"
if err := os.WriteFile(tempFile, data, 0644); err != nil {
return fmt.Errorf("failed to write temp peers.json %s: %w", tempFile, err)
}
if err := os.Rename(tempFile, peersFile); err != nil {
return fmt.Errorf("failed to rename %s to %s: %w", tempFile, peersFile, err)
}
nodeIDs := make([]string, 0, len(peers))
for _, p := range peers {
if id, ok := p["id"].(string); ok {
nodeIDs = append(nodeIDs, id)
}
}
c.logger.Info("peers.json written",
zap.Int("peers", len(peers)),
zap.Strings("nodes", nodeIDs))
return nil
}
// GetActivePeers returns a list of active peers (not including self)
func (c *ClusterDiscoveryService) GetActivePeers() []*discovery.RQLiteNodeMetadata {
c.mu.RLock()
defer c.mu.RUnlock()
peers := make([]*discovery.RQLiteNodeMetadata, 0, len(c.knownPeers))
for _, peer := range c.knownPeers {
// Skip self (compare by raft address since that's the NodeID now)
if peer.NodeID == c.raftAddress {
continue
}
peers = append(peers, peer)
}
return peers
}
// GetAllPeers returns a list of all known peers (including self)
func (c *ClusterDiscoveryService) GetAllPeers() []*discovery.RQLiteNodeMetadata {
c.mu.RLock()
defer c.mu.RUnlock()
peers := make([]*discovery.RQLiteNodeMetadata, 0, len(c.knownPeers))
for _, peer := range c.knownPeers {
peers = append(peers, peer)
}
return peers
}
// GetNodeWithHighestLogIndex returns the node with the highest Raft log index
func (c *ClusterDiscoveryService) GetNodeWithHighestLogIndex() *discovery.RQLiteNodeMetadata {
c.mu.RLock()
defer c.mu.RUnlock()
var highest *discovery.RQLiteNodeMetadata
var maxIndex uint64 = 0
for _, peer := range c.knownPeers {
// Skip self (compare by raft address since that's the NodeID now)
if peer.NodeID == c.raftAddress {
continue
}
if peer.RaftLogIndex > maxIndex {
maxIndex = peer.RaftLogIndex
highest = peer
}
}
return highest
}
// HasRecentPeersJSON checks if peers.json was recently updated
func (c *ClusterDiscoveryService) HasRecentPeersJSON() bool {
c.mu.RLock()
defer c.mu.RUnlock()
// Consider recent if updated in last 5 minutes
return time.Since(c.lastUpdate) < 5*time.Minute
}
// FindJoinTargets discovers join targets via LibP2P
func (c *ClusterDiscoveryService) FindJoinTargets() []string {
c.mu.RLock()
defer c.mu.RUnlock()
targets := []string{}
// All nodes are equal - prioritize by Raft log index (more advanced = better)
type nodeWithIndex struct {
address string
logIndex uint64
}
var nodes []nodeWithIndex
for _, peer := range c.knownPeers {
nodes = append(nodes, nodeWithIndex{peer.RaftAddress, peer.RaftLogIndex})
}
// Sort by log index descending (higher log index = more up-to-date)
for i := 0; i < len(nodes)-1; i++ {
for j := i + 1; j < len(nodes); j++ {
if nodes[j].logIndex > nodes[i].logIndex {
nodes[i], nodes[j] = nodes[j], nodes[i]
}
}
}
for _, n := range nodes {
targets = append(targets, n.address)
}
return targets
}
// WaitForDiscoverySettling waits for LibP2P discovery to settle (used on concurrent startup)
func (c *ClusterDiscoveryService) WaitForDiscoverySettling(ctx context.Context) {
settleDuration := 60 * time.Second
c.logger.Info("Waiting for discovery to settle",
zap.Duration("duration", settleDuration))
select {
case <-ctx.Done():
return
case <-time.After(settleDuration):
}
// Collect final peer list
c.updateClusterMembership()
c.mu.RLock()
peerCount := len(c.knownPeers)
c.mu.RUnlock()
c.logger.Info("Discovery settled",
zap.Int("peer_count", peerCount))
}
// TriggerSync manually triggers a cluster membership sync
func (c *ClusterDiscoveryService) TriggerSync() {
// All nodes use the same discovery timing for consistency
c.updateClusterMembership()
}
// ForceWritePeersJSON forces writing peers.json regardless of membership changes
// This is useful after clearing raft state when we need to recreate peers.json
func (c *ClusterDiscoveryService) ForceWritePeersJSON() error {
c.logger.Info("Force writing peers.json")
// First, collect latest peer metadata to ensure we have current information
metadata := c.collectPeerMetadata()
// Update known peers with latest metadata (without writing file yet)
c.mu.Lock()
for _, meta := range metadata {
c.knownPeers[meta.NodeID] = meta
// Update health tracking for remote peers
if meta.NodeID != c.raftAddress {
if _, ok := c.peerHealth[meta.NodeID]; !ok {
c.peerHealth[meta.NodeID] = &PeerHealth{
LastSeen: time.Now(),
LastSuccessful: time.Now(),
Status: "active",
}
} else {
c.peerHealth[meta.NodeID].LastSeen = time.Now()
c.peerHealth[meta.NodeID].Status = "active"
}
}
}
peers := c.getPeersJSONUnlocked()
c.mu.Unlock()
// Now force write the file
if err := c.writePeersJSONWithData(peers); err != nil {
c.logger.Error("Failed to force write peers.json",
zap.Error(err),
zap.String("data_dir", c.dataDir),
zap.Int("peers", len(peers)))
return err
}
c.logger.Info("peers.json written",
zap.Int("peers", len(peers)))
return nil
}
// TriggerPeerExchange actively exchanges peer information with connected peers
// This populates the peerstore with RQLite metadata from other nodes
func (c *ClusterDiscoveryService) TriggerPeerExchange(ctx context.Context) error {
if c.discoveryMgr == nil {
return fmt.Errorf("discovery manager not available")
}
collected := c.discoveryMgr.TriggerPeerExchange(ctx)
c.logger.Debug("Exchange completed", zap.Int("with_metadata", collected))
return nil
}
// UpdateOwnMetadata updates our own RQLite metadata in the peerstore
func (c *ClusterDiscoveryService) UpdateOwnMetadata() {
c.mu.RLock()
currentRaftAddr := c.raftAddress
currentHTTPAddr := c.httpAddress
c.mu.RUnlock()
metadata := &discovery.RQLiteNodeMetadata{
NodeID: currentRaftAddr, // RQLite uses raft address as node ID
RaftAddress: currentRaftAddr,
HTTPAddress: currentHTTPAddr,
NodeType: c.nodeType,
RaftLogIndex: c.rqliteManager.getRaftLogIndex(),
LastSeen: time.Now(),
ClusterVersion: "1.0",
}
// Adjust addresses if needed
if c.adjustSelfAdvertisedAddresses(metadata) {
c.logger.Debug("Adjusted self-advertised RQLite addresses in UpdateOwnMetadata",
zap.String("raft_address", metadata.RaftAddress),
zap.String("http_address", metadata.HTTPAddress))
}
// Store in our own peerstore for peer exchange
data, err := json.Marshal(metadata)
if err != nil {
c.logger.Error("Failed to marshal own metadata", zap.Error(err))
return
}
if err := c.host.Peerstore().Put(c.host.ID(), "rqlite_metadata", data); err != nil {
c.logger.Error("Failed to store own metadata", zap.Error(err))
return
}
c.logger.Debug("Metadata updated",
zap.String("node", metadata.NodeID),
zap.Uint64("log_index", metadata.RaftLogIndex))
}
// StoreRemotePeerMetadata stores metadata received from a remote peer
func (c *ClusterDiscoveryService) StoreRemotePeerMetadata(peerID peer.ID, metadata *discovery.RQLiteNodeMetadata) error {
if metadata == nil {
return fmt.Errorf("metadata is nil")
}
// Adjust addresses if needed (replace localhost with actual IP)
if updated, stale := c.adjustPeerAdvertisedAddresses(peerID, metadata); updated && stale != "" {
// Clean up stale entry if NodeID changed
c.mu.Lock()
delete(c.knownPeers, stale)
delete(c.peerHealth, stale)
c.mu.Unlock()
}
metadata.LastSeen = time.Now()
data, err := json.Marshal(metadata)
if err != nil {
return fmt.Errorf("failed to marshal metadata: %w", err)
}
if err := c.host.Peerstore().Put(peerID, "rqlite_metadata", data); err != nil {
return fmt.Errorf("failed to store metadata: %w", err)
}
c.logger.Debug("Metadata stored",
zap.String("peer", shortPeerID(peerID)),
zap.String("node", metadata.NodeID))
return nil
}
// adjustPeerAdvertisedAddresses adjusts peer metadata addresses by replacing localhost/loopback
// with the actual IP address from LibP2P connection. Returns (updated, staleNodeID).
// staleNodeID is non-empty if NodeID changed (indicating old entry should be cleaned up).
func (c *ClusterDiscoveryService) adjustPeerAdvertisedAddresses(peerID peer.ID, meta *discovery.RQLiteNodeMetadata) (bool, string) {
ip := c.selectPeerIP(peerID)
if ip == "" {
return false, ""
}
changed, stale := rewriteAdvertisedAddresses(meta, ip, true)
if changed {
c.logger.Debug("Addresses normalized",
zap.String("peer", shortPeerID(peerID)),
zap.String("raft", meta.RaftAddress),
zap.String("http_address", meta.HTTPAddress))
}
return changed, stale
}
// adjustSelfAdvertisedAddresses adjusts our own metadata addresses by replacing localhost/loopback
// with the actual IP address from LibP2P host. Updates internal state if changed.
func (c *ClusterDiscoveryService) adjustSelfAdvertisedAddresses(meta *discovery.RQLiteNodeMetadata) bool {
ip := c.selectSelfIP()
if ip == "" {
return false
}
changed, _ := rewriteAdvertisedAddresses(meta, ip, true)
if !changed {
return false
}
// Update internal state with corrected addresses
c.mu.Lock()
c.raftAddress = meta.RaftAddress
c.httpAddress = meta.HTTPAddress
c.mu.Unlock()
if c.rqliteManager != nil {
c.rqliteManager.UpdateAdvertisedAddresses(meta.RaftAddress, meta.HTTPAddress)
}
return true
}
// selectPeerIP selects the best IP address for a peer from LibP2P connections.
// Prefers public IPs, falls back to private IPs if no public IP is available.
func (c *ClusterDiscoveryService) selectPeerIP(peerID peer.ID) string {
var fallback string
// First, try to get IP from active connections
for _, conn := range c.host.Network().ConnsToPeer(peerID) {
if ip, public := ipFromMultiaddr(conn.RemoteMultiaddr()); ip != "" {
if shouldReplaceHost(ip) {
continue
}
if public {
return ip
}
if fallback == "" {
fallback = ip
}
}
}
// Fallback to peerstore addresses
for _, addr := range c.host.Peerstore().Addrs(peerID) {
if ip, public := ipFromMultiaddr(addr); ip != "" {
if shouldReplaceHost(ip) {
continue
}
if public {
return ip
}
if fallback == "" {
fallback = ip
}
}
}
return fallback
}
// selectSelfIP selects the best IP address for ourselves from LibP2P host addresses.
// Prefers public IPs, falls back to private IPs if no public IP is available.
func (c *ClusterDiscoveryService) selectSelfIP() string {
var fallback string
for _, addr := range c.host.Addrs() {
if ip, public := ipFromMultiaddr(addr); ip != "" {
if shouldReplaceHost(ip) {
continue
}
if public {
return ip
}
if fallback == "" {
fallback = ip
}
}
}
return fallback
}
// rewriteAdvertisedAddresses rewrites RaftAddress and HTTPAddress in metadata,
// replacing localhost/loopback addresses with the provided IP.
// Returns (changed, staleNodeID). staleNodeID is non-empty if NodeID changed.
func rewriteAdvertisedAddresses(meta *discovery.RQLiteNodeMetadata, newHost string, allowNodeIDRewrite bool) (bool, string) {
if meta == nil || newHost == "" {
return false, ""
}
originalNodeID := meta.NodeID
changed := false
nodeIDChanged := false
// Replace host in RaftAddress if it's localhost/loopback
if newAddr, replaced := replaceAddressHost(meta.RaftAddress, newHost); replaced {
if meta.RaftAddress != newAddr {
meta.RaftAddress = newAddr
changed = true
}
}
// Replace host in HTTPAddress if it's localhost/loopback
if newAddr, replaced := replaceAddressHost(meta.HTTPAddress, newHost); replaced {
if meta.HTTPAddress != newAddr {
meta.HTTPAddress = newAddr
changed = true
}
}
// Update NodeID to match RaftAddress if it changed
if allowNodeIDRewrite {
if meta.RaftAddress != "" && (meta.NodeID == "" || meta.NodeID == originalNodeID || shouldReplaceHost(hostFromAddress(meta.NodeID))) {
if meta.NodeID != meta.RaftAddress {
meta.NodeID = meta.RaftAddress
nodeIDChanged = meta.NodeID != originalNodeID
if nodeIDChanged {
changed = true
}
}
}
}
if nodeIDChanged {
return changed, originalNodeID
}
return changed, ""
}
// replaceAddressHost replaces the host part of an address if it's localhost/loopback.
// Returns (newAddress, replaced). replaced is true if host was replaced.
func replaceAddressHost(address, newHost string) (string, bool) {
if address == "" || newHost == "" {
return address, false
}
host, port, err := net.SplitHostPort(address)
if err != nil {
return address, false
}
if !shouldReplaceHost(host) {
return address, false
}
return net.JoinHostPort(newHost, port), true
}
// shouldReplaceHost returns true if the host should be replaced (localhost, loopback, etc.)
func shouldReplaceHost(host string) bool {
if host == "" {
return true
}
if strings.EqualFold(host, "localhost") {
return true
}
// Check if it's a loopback or unspecified address
if addr, err := netip.ParseAddr(host); err == nil {
if addr.IsLoopback() || addr.IsUnspecified() {
return true
}
}
return false
}
// hostFromAddress extracts the host part from a host:port address
func hostFromAddress(address string) string {
host, _, err := net.SplitHostPort(address)
if err != nil {
return ""
}
return host
}
// ipFromMultiaddr extracts an IP address from a multiaddr and returns (ip, isPublic)
func ipFromMultiaddr(addr multiaddr.Multiaddr) (string, bool) {
if addr == nil {
return "", false
}
if v4, err := addr.ValueForProtocol(multiaddr.P_IP4); err == nil {
return v4, isPublicIP(v4)
}
if v6, err := addr.ValueForProtocol(multiaddr.P_IP6); err == nil {
return v6, isPublicIP(v6)
}
return "", false
}
// isPublicIP returns true if the IP is a public (non-private, non-loopback) address
func isPublicIP(ip string) bool {
addr, err := netip.ParseAddr(ip)
if err != nil {
return false
}
// Exclude loopback, unspecified, link-local, multicast, and private addresses
if addr.IsLoopback() || addr.IsUnspecified() || addr.IsLinkLocalUnicast() || addr.IsLinkLocalMulticast() || addr.IsPrivate() {
return false
}
return true
}
// shortPeerID returns a shortened version of a peer ID for logging
func shortPeerID(id peer.ID) string {
s := id.String()
if len(s) <= 8 {
return s
}
return s[:8] + "..."
}

318
pkg/rqlite/cluster_discovery_membership.go Normal file
View File

@ -0,0 +1,318 @@
package rqlite
import (
"encoding/json"
"fmt"
"os"
"path/filepath"
"strings"
"time"
"github.com/DeBrosOfficial/network/pkg/discovery"
"go.uber.org/zap"
)
// collectPeerMetadata collects RQLite metadata from LibP2P peers
func (c *ClusterDiscoveryService) collectPeerMetadata() []*discovery.RQLiteNodeMetadata {
connectedPeers := c.host.Network().Peers()
var metadata []*discovery.RQLiteNodeMetadata
c.mu.RLock()
currentRaftAddr := c.raftAddress
currentHTTPAddr := c.httpAddress
c.mu.RUnlock()
// Add ourselves
ourMetadata := &discovery.RQLiteNodeMetadata{
NodeID: currentRaftAddr, // RQLite uses raft address as node ID
RaftAddress: currentRaftAddr,
HTTPAddress: currentHTTPAddr,
NodeType: c.nodeType,
RaftLogIndex: c.rqliteManager.getRaftLogIndex(),
LastSeen: time.Now(),
ClusterVersion: "1.0",
}
if c.adjustSelfAdvertisedAddresses(ourMetadata) {
c.logger.Debug("Adjusted self-advertised RQLite addresses",
zap.String("raft_address", ourMetadata.RaftAddress),
zap.String("http_address", ourMetadata.HTTPAddress))
}
metadata = append(metadata, ourMetadata)
staleNodeIDs := make([]string, 0)
for _, peerID := range connectedPeers {
if val, err := c.host.Peerstore().Get(peerID, "rqlite_metadata"); err == nil {
if jsonData, ok := val.([]byte); ok {
var peerMeta discovery.RQLiteNodeMetadata
if err := json.Unmarshal(jsonData, &peerMeta); err == nil {
if updated, stale := c.adjustPeerAdvertisedAddresses(peerID, &peerMeta); updated && stale != "" {
staleNodeIDs = append(staleNodeIDs, stale)
}
peerMeta.LastSeen = time.Now()
metadata = append(metadata, &peerMeta)
}
}
}
}
if len(staleNodeIDs) > 0 {
c.mu.Lock()
for _, id := range staleNodeIDs {
delete(c.knownPeers, id)
delete(c.peerHealth, id)
}
c.mu.Unlock()
}
return metadata
}
type membershipUpdateResult struct {
peersJSON []map[string]interface{}
added []string
updated []string
changed bool
}
func (c *ClusterDiscoveryService) updateClusterMembership() {
metadata := c.collectPeerMetadata()
c.mu.Lock()
result := c.computeMembershipChangesLocked(metadata)
c.mu.Unlock()
if result.changed {
if len(result.added) > 0 || len(result.updated) > 0 {
c.logger.Info("Membership changed",
zap.Int("added", len(result.added)),
zap.Int("updated", len(result.updated)),
zap.Strings("added", result.added),
zap.Strings("updated", result.updated))
}
if err := c.writePeersJSONWithData(result.peersJSON); err != nil {
c.logger.Error("Failed to write peers.json",
zap.Error(err),
zap.String("data_dir", c.dataDir),
zap.Int("peers", len(result.peersJSON)))
} else {
c.logger.Debug("peers.json updated",
zap.Int("peers", len(result.peersJSON)))
}
c.mu.Lock()
c.lastUpdate = time.Now()
c.mu.Unlock()
}
}
func (c *ClusterDiscoveryService) computeMembershipChangesLocked(metadata []*discovery.RQLiteNodeMetadata) membershipUpdateResult {
added := []string{}
updated := []string{}
for _, meta := range metadata {
isSelf := meta.NodeID == c.raftAddress
if existing, ok := c.knownPeers[meta.NodeID]; ok {
if existing.RaftLogIndex != meta.RaftLogIndex ||
existing.HTTPAddress != meta.HTTPAddress ||
existing.RaftAddress != meta.RaftAddress {
updated = append(updated, meta.NodeID)
}
} else {
added = append(added, meta.NodeID)
c.logger.Info("Node added",
zap.String("node", meta.NodeID),
zap.String("raft", meta.RaftAddress),
zap.String("type", meta.NodeType),
zap.Uint64("log_index", meta.RaftLogIndex))
}
c.knownPeers[meta.NodeID] = meta
if !isSelf {
if _, ok := c.peerHealth[meta.NodeID]; !ok {
c.peerHealth[meta.NodeID] = &PeerHealth{
LastSeen: time.Now(),
LastSuccessful: time.Now(),
Status: "active",
}
} else {
c.peerHealth[meta.NodeID].LastSeen = time.Now()
c.peerHealth[meta.NodeID].Status = "active"
c.peerHealth[meta.NodeID].FailureCount = 0
}
}
}
remotePeerCount := 0
for _, peer := range c.knownPeers {
if peer.NodeID != c.raftAddress {
remotePeerCount++
}
}
peers := c.getPeersJSONUnlocked()
shouldWrite := len(added) > 0 || len(updated) > 0 || c.lastUpdate.IsZero()
if shouldWrite {
if c.lastUpdate.IsZero() {
requiredRemotePeers := c.minClusterSize - 1
if remotePeerCount < requiredRemotePeers {
c.logger.Info("Waiting for peers",
zap.Int("have", remotePeerCount),
zap.Int("need", requiredRemotePeers),
zap.Int("min_size", c.minClusterSize))
return membershipUpdateResult{
changed: false,
}
}
}
if len(peers) == 0 && c.lastUpdate.IsZero() {
c.logger.Info("No remote peers - waiting")
return membershipUpdateResult{
changed: false,
}
}
if c.lastUpdate.IsZero() {
c.logger.Info("Initial sync",
zap.Int("total", len(c.knownPeers)),
zap.Int("remote", remotePeerCount),
zap.Int("in_json", len(peers)))
}
return membershipUpdateResult{
peersJSON: peers,
added: added,
updated: updated,
changed: true,
}
}
return membershipUpdateResult{
changed: false,
}
}
func (c *ClusterDiscoveryService) removeInactivePeers() {
c.mu.Lock()
defer c.mu.Unlock()
now := time.Now()
removed := []string{}
for nodeID, health := range c.peerHealth {
inactiveDuration := now.Sub(health.LastSeen)
if inactiveDuration > c.inactivityLimit {
c.logger.Warn("Node removed",
zap.String("node", nodeID),
zap.String("reason", "inactive"),
zap.Duration("inactive_duration", inactiveDuration))
delete(c.knownPeers, nodeID)
delete(c.peerHealth, nodeID)
removed = append(removed, nodeID)
}
}
if len(removed) > 0 {
c.logger.Info("Removed inactive",
zap.Int("count", len(removed)),
zap.Strings("nodes", removed))
if err := c.writePeersJSON(); err != nil {
c.logger.Error("Failed to write peers.json after cleanup", zap.Error(err))
}
}
}
func (c *ClusterDiscoveryService) getPeersJSON() []map[string]interface{} {
c.mu.RLock()
defer c.mu.RUnlock()
return c.getPeersJSONUnlocked()
}
func (c *ClusterDiscoveryService) getPeersJSONUnlocked() []map[string]interface{} {
peers := make([]map[string]interface{}, 0, len(c.knownPeers))
for _, peer := range c.knownPeers {
peerEntry := map[string]interface{}{
"id": peer.RaftAddress,
"address": peer.RaftAddress,
"non_voter": false,
}
peers = append(peers, peerEntry)
}
return peers
}
func (c *ClusterDiscoveryService) writePeersJSON() error {
c.mu.RLock()
peers := c.getPeersJSONUnlocked()
c.mu.RUnlock()
return c.writePeersJSONWithData(peers)
}
func (c *ClusterDiscoveryService) writePeersJSONWithData(peers []map[string]interface{}) error {
dataDir := os.ExpandEnv(c.dataDir)
if strings.HasPrefix(dataDir, "~") {
home, err := os.UserHomeDir()
if err != nil {
return fmt.Errorf("failed to determine home directory: %w", err)
}
dataDir = filepath.Join(home, dataDir[1:])
}
rqliteDir := filepath.Join(dataDir, "rqlite", "raft")
if err := os.MkdirAll(rqliteDir, 0755); err != nil {
return fmt.Errorf("failed to create raft directory %s: %w", rqliteDir, err)
}
peersFile := filepath.Join(rqliteDir, "peers.json")
backupFile := filepath.Join(rqliteDir, "peers.json.backup")
if _, err := os.Stat(peersFile); err == nil {
data, err := os.ReadFile(peersFile)
if err == nil {
_ = os.WriteFile(backupFile, data, 0644)
}
}
data, err := json.MarshalIndent(peers, "", " ")
if err != nil {
return fmt.Errorf("failed to marshal peers.json: %w", err)
}
tempFile := peersFile + ".tmp"
if err := os.WriteFile(tempFile, data, 0644); err != nil {
return fmt.Errorf("failed to write temp peers.json %s: %w", tempFile, err)
}
if err := os.Rename(tempFile, peersFile); err != nil {
return fmt.Errorf("failed to rename %s to %s: %w", tempFile, peersFile, err)
}
nodeIDs := make([]string, 0, len(peers))
for _, p := range peers {
if id, ok := p["id"].(string); ok {
nodeIDs = append(nodeIDs, id)
}
}
c.logger.Info("peers.json written",
zap.Int("peers", len(peers)),
zap.Strings("nodes", nodeIDs))
return nil
}

251
pkg/rqlite/cluster_discovery_queries.go Normal file
View File

@ -0,0 +1,251 @@
package rqlite
import (
"context"
"encoding/json"
"fmt"
"time"
"github.com/DeBrosOfficial/network/pkg/discovery"
"github.com/libp2p/go-libp2p/core/peer"
"go.uber.org/zap"
)
// GetActivePeers returns a list of active peers (not including self)
func (c *ClusterDiscoveryService) GetActivePeers() []*discovery.RQLiteNodeMetadata {
c.mu.RLock()
defer c.mu.RUnlock()
peers := make([]*discovery.RQLiteNodeMetadata, 0, len(c.knownPeers))
for _, peer := range c.knownPeers {
if peer.NodeID == c.raftAddress {
continue
}
peers = append(peers, peer)
}
return peers
}
// GetAllPeers returns a list of all known peers (including self)
func (c *ClusterDiscoveryService) GetAllPeers() []*discovery.RQLiteNodeMetadata {
c.mu.RLock()
defer c.mu.RUnlock()
peers := make([]*discovery.RQLiteNodeMetadata, 0, len(c.knownPeers))
for _, peer := range c.knownPeers {
peers = append(peers, peer)
}
return peers
}
// GetNodeWithHighestLogIndex returns the node with the highest Raft log index
func (c *ClusterDiscoveryService) GetNodeWithHighestLogIndex() *discovery.RQLiteNodeMetadata {
c.mu.RLock()
defer c.mu.RUnlock()
var highest *discovery.RQLiteNodeMetadata
var maxIndex uint64 = 0
for _, peer := range c.knownPeers {
if peer.NodeID == c.raftAddress {
continue
}
if peer.RaftLogIndex > maxIndex {
maxIndex = peer.RaftLogIndex
highest = peer
}
}
return highest
}
// HasRecentPeersJSON checks if peers.json was recently updated
func (c *ClusterDiscoveryService) HasRecentPeersJSON() bool {
c.mu.RLock()
defer c.mu.RUnlock()
return time.Since(c.lastUpdate) < 5*time.Minute
}
// FindJoinTargets discovers join targets via LibP2P
func (c *ClusterDiscoveryService) FindJoinTargets() []string {
c.mu.RLock()
defer c.mu.RUnlock()
targets := []string{}
type nodeWithIndex struct {
address string
logIndex uint64
}
var nodes []nodeWithIndex
for _, peer := range c.knownPeers {
nodes = append(nodes, nodeWithIndex{peer.RaftAddress, peer.RaftLogIndex})
}
for i := 0; i < len(nodes)-1; i++ {
for j := i + 1; j < len(nodes); j++ {
if nodes[j].logIndex > nodes[i].logIndex {
nodes[i], nodes[j] = nodes[j], nodes[i]
}
}
}
for _, n := range nodes {
targets = append(targets, n.address)
}
return targets
}
// WaitForDiscoverySettling waits for LibP2P discovery to settle (used on concurrent startup)
func (c *ClusterDiscoveryService) WaitForDiscoverySettling(ctx context.Context) {
settleDuration := 60 * time.Second
c.logger.Info("Waiting for discovery to settle",
zap.Duration("duration", settleDuration))
select {
case <-ctx.Done():
return
case <-time.After(settleDuration):
}
c.updateClusterMembership()
c.mu.RLock()
peerCount := len(c.knownPeers)
c.mu.RUnlock()
c.logger.Info("Discovery settled",
zap.Int("peer_count", peerCount))
}
// TriggerSync manually triggers a cluster membership sync
func (c *ClusterDiscoveryService) TriggerSync() {
c.updateClusterMembership()
}
// ForceWritePeersJSON forces writing peers.json regardless of membership changes
func (c *ClusterDiscoveryService) ForceWritePeersJSON() error {
c.logger.Info("Force writing peers.json")
metadata := c.collectPeerMetadata()
c.mu.Lock()
for _, meta := range metadata {
c.knownPeers[meta.NodeID] = meta
if meta.NodeID != c.raftAddress {
if _, ok := c.peerHealth[meta.NodeID]; !ok {
c.peerHealth[meta.NodeID] = &PeerHealth{
LastSeen: time.Now(),
LastSuccessful: time.Now(),
Status: "active",
}
} else {
c.peerHealth[meta.NodeID].LastSeen = time.Now()
c.peerHealth[meta.NodeID].Status = "active"
}
}
}
peers := c.getPeersJSONUnlocked()
c.mu.Unlock()
if err := c.writePeersJSONWithData(peers); err != nil {
c.logger.Error("Failed to force write peers.json",
zap.Error(err),
zap.String("data_dir", c.dataDir),
zap.Int("peers", len(peers)))
return err
}
c.logger.Info("peers.json written",
zap.Int("peers", len(peers)))
return nil
}
// TriggerPeerExchange actively exchanges peer information with connected peers
func (c *ClusterDiscoveryService) TriggerPeerExchange(ctx context.Context) error {
if c.discoveryMgr == nil {
return fmt.Errorf("discovery manager not available")
}
collected := c.discoveryMgr.TriggerPeerExchange(ctx)
c.logger.Debug("Exchange completed", zap.Int("with_metadata", collected))
return nil
}
// UpdateOwnMetadata updates our own RQLite metadata in the peerstore
func (c *ClusterDiscoveryService) UpdateOwnMetadata() {
c.mu.RLock()
currentRaftAddr := c.raftAddress
currentHTTPAddr := c.httpAddress
c.mu.RUnlock()
metadata := &discovery.RQLiteNodeMetadata{
NodeID: currentRaftAddr,
RaftAddress: currentRaftAddr,
HTTPAddress: currentHTTPAddr,
NodeType: c.nodeType,
RaftLogIndex: c.rqliteManager.getRaftLogIndex(),
LastSeen: time.Now(),
ClusterVersion: "1.0",
}
if c.adjustSelfAdvertisedAddresses(metadata) {
c.logger.Debug("Adjusted self-advertised RQLite addresses in UpdateOwnMetadata",
zap.String("raft_address", metadata.RaftAddress),
zap.String("http_address", metadata.HTTPAddress))
}
data, err := json.Marshal(metadata)
if err != nil {
c.logger.Error("Failed to marshal own metadata", zap.Error(err))
return
}
if err := c.host.Peerstore().Put(c.host.ID(), "rqlite_metadata", data); err != nil {
c.logger.Error("Failed to store own metadata", zap.Error(err))
return
}
c.logger.Debug("Metadata updated",
zap.String("node", metadata.NodeID),
zap.Uint64("log_index", metadata.RaftLogIndex))
}
// StoreRemotePeerMetadata stores metadata received from a remote peer
func (c *ClusterDiscoveryService) StoreRemotePeerMetadata(peerID peer.ID, metadata *discovery.RQLiteNodeMetadata) error {
if metadata == nil {
return fmt.Errorf("metadata is nil")
}
if updated, stale := c.adjustPeerAdvertisedAddresses(peerID, metadata); updated && stale != "" {
c.mu.Lock()
delete(c.knownPeers, stale)
delete(c.peerHealth, stale)
c.mu.Unlock()
}
metadata.LastSeen = time.Now()
data, err := json.Marshal(metadata)
if err != nil {
return fmt.Errorf("failed to marshal metadata: %w", err)
}
if err := c.host.Peerstore().Put(peerID, "rqlite_metadata", data); err != nil {
return fmt.Errorf("failed to store metadata: %w", err)
}
c.logger.Debug("Metadata stored",
zap.String("peer", shortPeerID(peerID)),
zap.String("node", metadata.NodeID))
return nil
}

233
pkg/rqlite/cluster_discovery_utils.go Normal file
View File

@ -0,0 +1,233 @@
package rqlite
import (
"net"
"net/netip"
"strings"
"github.com/DeBrosOfficial/network/pkg/discovery"
"github.com/libp2p/go-libp2p/core/peer"
"github.com/multiformats/go-multiaddr"
"go.uber.org/zap"
)
// adjustPeerAdvertisedAddresses adjusts peer metadata addresses
func (c *ClusterDiscoveryService) adjustPeerAdvertisedAddresses(peerID peer.ID, meta *discovery.RQLiteNodeMetadata) (bool, string) {
ip := c.selectPeerIP(peerID)
if ip == "" {
return false, ""
}
changed, stale := rewriteAdvertisedAddresses(meta, ip, true)
if changed {
c.logger.Debug("Addresses normalized",
zap.String("peer", shortPeerID(peerID)),
zap.String("raft", meta.RaftAddress),
zap.String("http_address", meta.HTTPAddress))
}
return changed, stale
}
// adjustSelfAdvertisedAddresses adjusts our own metadata addresses
func (c *ClusterDiscoveryService) adjustSelfAdvertisedAddresses(meta *discovery.RQLiteNodeMetadata) bool {
ip := c.selectSelfIP()
if ip == "" {
return false
}
changed, _ := rewriteAdvertisedAddresses(meta, ip, true)
if !changed {
return false
}
c.mu.Lock()
c.raftAddress = meta.RaftAddress
c.httpAddress = meta.HTTPAddress
c.mu.Unlock()
if c.rqliteManager != nil {
c.rqliteManager.UpdateAdvertisedAddresses(meta.RaftAddress, meta.HTTPAddress)
}
return true
}
// selectPeerIP selects the best IP address for a peer
func (c *ClusterDiscoveryService) selectPeerIP(peerID peer.ID) string {
var fallback string
for _, conn := range c.host.Network().ConnsToPeer(peerID) {
if ip, public := ipFromMultiaddr(conn.RemoteMultiaddr()); ip != "" {
if shouldReplaceHost(ip) {
continue
}
if public {
return ip
}
if fallback == "" {
fallback = ip
}
}
}
for _, addr := range c.host.Peerstore().Addrs(peerID) {
if ip, public := ipFromMultiaddr(addr); ip != "" {
if shouldReplaceHost(ip) {
continue
}
if public {
return ip
}
if fallback == "" {
fallback = ip
}
}
}
return fallback
}
// selectSelfIP selects the best IP address for ourselves
func (c *ClusterDiscoveryService) selectSelfIP() string {
var fallback string
for _, addr := range c.host.Addrs() {
if ip, public := ipFromMultiaddr(addr); ip != "" {
if shouldReplaceHost(ip) {
continue
}
if public {
return ip
}
if fallback == "" {
fallback = ip
}
}
}
return fallback
}
// rewriteAdvertisedAddresses rewrites RaftAddress and HTTPAddress in metadata
func rewriteAdvertisedAddresses(meta *discovery.RQLiteNodeMetadata, newHost string, allowNodeIDRewrite bool) (bool, string) {
if meta == nil || newHost == "" {
return false, ""
}
originalNodeID := meta.NodeID
changed := false
nodeIDChanged := false
if newAddr, replaced := replaceAddressHost(meta.RaftAddress, newHost); replaced {
if meta.RaftAddress != newAddr {
meta.RaftAddress = newAddr
changed = true
}
}
if newAddr, replaced := replaceAddressHost(meta.HTTPAddress, newHost); replaced {
if meta.HTTPAddress != newAddr {
meta.HTTPAddress = newAddr
changed = true
}
}
if allowNodeIDRewrite {
if meta.RaftAddress != "" && (meta.NodeID == "" || meta.NodeID == originalNodeID || shouldReplaceHost(hostFromAddress(meta.NodeID))) {
if meta.NodeID != meta.RaftAddress {
meta.NodeID = meta.RaftAddress
nodeIDChanged = meta.NodeID != originalNodeID
if nodeIDChanged {
changed = true
}
}
}
}
if nodeIDChanged {
return changed, originalNodeID
}
return changed, ""
}
// replaceAddressHost replaces the host part of an address
func replaceAddressHost(address, newHost string) (string, bool) {
if address == "" || newHost == "" {
return address, false
}
host, port, err := net.SplitHostPort(address)
if err != nil {
return address, false
}
if !shouldReplaceHost(host) {
return address, false
}
return net.JoinHostPort(newHost, port), true
}
// shouldReplaceHost returns true if the host should be replaced
func shouldReplaceHost(host string) bool {
if host == "" {
return true
}
if strings.EqualFold(host, "localhost") {
return true
}
if addr, err := netip.ParseAddr(host); err == nil {
if addr.IsLoopback() || addr.IsUnspecified() {
return true
}
}
return false
}
// hostFromAddress extracts the host part from a host:port address
func hostFromAddress(address string) string {
host, _, err := net.SplitHostPort(address)
if err != nil {
return ""
}
return host
}
// ipFromMultiaddr extracts an IP address from a multiaddr and returns (ip, isPublic)
func ipFromMultiaddr(addr multiaddr.Multiaddr) (string, bool) {
if addr == nil {
return "", false
}
if v4, err := addr.ValueForProtocol(multiaddr.P_IP4); err == nil {
return v4, isPublicIP(v4)
}
if v6, err := addr.ValueForProtocol(multiaddr.P_IP6); err == nil {
return v6, isPublicIP(v6)
}
return "", false
}
// isPublicIP returns true if the IP is a public address
func isPublicIP(ip string) bool {
addr, err := netip.ParseAddr(ip)
if err != nil {
return false
}
if addr.IsLoopback() || addr.IsUnspecified() || addr.IsLinkLocalUnicast() || addr.IsLinkLocalMulticast() || addr.IsPrivate() {
return false
}
return true
}
// shortPeerID returns a shortened version of a peer ID
func shortPeerID(id peer.ID) string {
s := id.String()
if len(s) <= 8 {
return s
}
return s[:8] + "..."
}

61
pkg/rqlite/discovery_manager.go Normal file
View File

@ -0,0 +1,61 @@
package rqlite
import (
"fmt"
"time"
)
// SetDiscoveryService sets the cluster discovery service
func (r *RQLiteManager) SetDiscoveryService(service *ClusterDiscoveryService) {
r.discoveryService = service
}
// SetNodeType sets the node type
func (r *RQLiteManager) SetNodeType(nodeType string) {
if nodeType != "" {
r.nodeType = nodeType
}
}
// UpdateAdvertisedAddresses overrides advertised addresses
func (r *RQLiteManager) UpdateAdvertisedAddresses(raftAddr, httpAddr string) {
if r == nil || r.discoverConfig == nil {
return
}
if raftAddr != "" && r.discoverConfig.RaftAdvAddress != raftAddr {
r.discoverConfig.RaftAdvAddress = raftAddr
}
if httpAddr != "" && r.discoverConfig.HttpAdvAddress != httpAddr {
r.discoverConfig.HttpAdvAddress = httpAddr
}
}
func (r *RQLiteManager) validateNodeID() error {
for i := 0; i < 5; i++ {
nodes, err := r.getRQLiteNodes()
if err != nil {
if i < 4 {
time.Sleep(500 * time.Millisecond)
continue
}
return nil
}
expectedID := r.discoverConfig.RaftAdvAddress
if expectedID == "" || len(nodes) == 0 {
return nil
}
for _, node := range nodes {
if node.Address == expectedID {
if node.ID != expectedID {
return fmt.Errorf("node ID mismatch: %s != %s", expectedID, node.ID)
}
return nil
}
}
return nil
}
return nil
}

239
pkg/rqlite/process.go Normal file
View File

@ -0,0 +1,239 @@
package rqlite
import (
"context"
"encoding/json"
"fmt"
"io"
"net/http"
"os"
"os/exec"
"path/filepath"
"strings"
"time"
"github.com/DeBrosOfficial/network/pkg/tlsutil"
"github.com/rqlite/gorqlite"
"go.uber.org/zap"
)
// launchProcess starts the RQLite process with appropriate arguments
func (r *RQLiteManager) launchProcess(ctx context.Context, rqliteDataDir string) error {
// Build RQLite command
args := []string{
"-http-addr", fmt.Sprintf("0.0.0.0:%d", r.config.RQLitePort),
"-http-adv-addr", r.discoverConfig.HttpAdvAddress,
"-raft-adv-addr", r.discoverConfig.RaftAdvAddress,
"-raft-addr", fmt.Sprintf("0.0.0.0:%d", r.config.RQLiteRaftPort),
}
if r.config.NodeCert != "" && r.config.NodeKey != "" {
r.logger.Info("Enabling node-to-node TLS encryption",
zap.String("node_cert", r.config.NodeCert),
zap.String("node_key", r.config.NodeKey))
args = append(args, "-node-cert", r.config.NodeCert)
args = append(args, "-node-key", r.config.NodeKey)
if r.config.NodeCACert != "" {
args = append(args, "-node-ca-cert", r.config.NodeCACert)
}
if r.config.NodeNoVerify {
args = append(args, "-node-no-verify")
}
}
if r.config.RQLiteJoinAddress != "" {
r.logger.Info("Joining RQLite cluster", zap.String("join_address", r.config.RQLiteJoinAddress))
joinArg := r.config.RQLiteJoinAddress
if strings.HasPrefix(joinArg, "http://") {
joinArg = strings.TrimPrefix(joinArg, "http://")
} else if strings.HasPrefix(joinArg, "https://") {
joinArg = strings.TrimPrefix(joinArg, "https://")
}
joinTimeout := 5 * time.Minute
if err := r.waitForJoinTarget(ctx, r.config.RQLiteJoinAddress, joinTimeout); err != nil {
r.logger.Warn("Join target did not become reachable within timeout; will still attempt to join",
zap.Error(err))
}
args = append(args, "-join", joinArg, "-join-as", r.discoverConfig.RaftAdvAddress, "-join-attempts", "30", "-join-interval", "10s")
}
args = append(args, rqliteDataDir)
r.cmd = exec.Command("rqlited", args...)
nodeType := r.nodeType
if nodeType == "" {
nodeType = "node"
}
logsDir := filepath.Join(filepath.Dir(r.dataDir), "logs")
_ = os.MkdirAll(logsDir, 0755)
logPath := filepath.Join(logsDir, fmt.Sprintf("rqlite-%s.log", nodeType))
logFile, err := os.OpenFile(logPath, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0644)
if err != nil {
return fmt.Errorf("failed to open log file: %w", err)
}
r.cmd.Stdout = logFile
r.cmd.Stderr = logFile
if err := r.cmd.Start(); err != nil {
logFile.Close()
return fmt.Errorf("failed to start RQLite: %w", err)
}
logFile.Close()
return nil
}
// waitForReadyAndConnect waits for RQLite to be ready and establishes connection
func (r *RQLiteManager) waitForReadyAndConnect(ctx context.Context) error {
if err := r.waitForReady(ctx); err != nil {
if r.cmd != nil && r.cmd.Process != nil {
_ = r.cmd.Process.Kill()
}
return err
}
var conn *gorqlite.Connection
var err error
maxConnectAttempts := 10
connectBackoff := 500 * time.Millisecond
for attempt := 0; attempt < maxConnectAttempts; attempt++ {
conn, err = gorqlite.Open(fmt.Sprintf("http://localhost:%d", r.config.RQLitePort))
if err == nil {
r.connection = conn
break
}
if strings.Contains(err.Error(), "store is not open") {
time.Sleep(connectBackoff)
connectBackoff = time.Duration(float64(connectBackoff) * 1.5)
if connectBackoff > 5*time.Second {
connectBackoff = 5 * time.Second
}
continue
}
if r.cmd != nil && r.cmd.Process != nil {
_ = r.cmd.Process.Kill()
}
return fmt.Errorf("failed to connect to RQLite: %w", err)
}
if conn == nil {
return fmt.Errorf("failed to connect to RQLite after max attempts")
}
_ = r.validateNodeID()
return nil
}
// waitForReady waits for RQLite to be ready to accept connections
func (r *RQLiteManager) waitForReady(ctx context.Context) error {
url := fmt.Sprintf("http://localhost:%d/status", r.config.RQLitePort)
client := tlsutil.NewHTTPClient(2 * time.Second)
for i := 0; i < 180; i++ {
select {
case <-ctx.Done():
return ctx.Err()
case <-time.After(1 * time.Second):
}
resp, err := client.Get(url)
if err == nil && resp.StatusCode == http.StatusOK {
body, _ := io.ReadAll(resp.Body)
resp.Body.Close()
var statusResp map[string]interface{}
if err := json.Unmarshal(body, &statusResp); err == nil {
if raft, ok := statusResp["raft"].(map[string]interface{}); ok {
state, _ := raft["state"].(string)
if state == "leader" || state == "follower" {
return nil
}
} else {
return nil // Backwards compatibility
}
}
}
}
return fmt.Errorf("RQLite did not become ready within timeout")
}
// waitForSQLAvailable waits until a simple query succeeds
func (r *RQLiteManager) waitForSQLAvailable(ctx context.Context) error {
ticker := time.NewTicker(1 * time.Second)
defer ticker.Stop()
for {
select {
case <-ctx.Done():
return ctx.Err()
case <-ticker.C:
if r.connection == nil {
continue
}
_, err := r.connection.QueryOne("SELECT 1")
if err == nil {
return nil
}
}
}
}
// testJoinAddress tests if a join address is reachable
func (r *RQLiteManager) testJoinAddress(joinAddress string) error {
client := tlsutil.NewHTTPClient(5 * time.Second)
var statusURL string
if strings.HasPrefix(joinAddress, "http://") || strings.HasPrefix(joinAddress, "https://") {
statusURL = strings.TrimRight(joinAddress, "/") + "/status"
} else {
host := joinAddress
if idx := strings.Index(joinAddress, ":"); idx != -1 {
host = joinAddress[:idx]
}
statusURL = fmt.Sprintf("http://%s:%d/status", host, 5001)
}
resp, err := client.Get(statusURL)
if err != nil {
return err
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
return fmt.Errorf("leader returned status %d", resp.StatusCode)
}
return nil
}
// waitForJoinTarget waits until the join target's HTTP status becomes reachable
func (r *RQLiteManager) waitForJoinTarget(ctx context.Context, joinAddress string, timeout time.Duration) error {
deadline := time.Now().Add(timeout)
var lastErr error
for time.Now().Before(deadline) {
if err := r.testJoinAddress(joinAddress); err == nil {
return nil
} else {
lastErr = err
}
select {
case <-ctx.Done():
return ctx.Err()
case <-time.After(2 * time.Second):
}
}
return lastErr
}

1275
pkg/rqlite/rqlite.go
View File

File diff suppressed because it is too large Load Diff

58
pkg/rqlite/util.go Normal file
View File

@ -0,0 +1,58 @@
package rqlite
import (
"os"
"path/filepath"
"strings"
"time"
)
func (r *RQLiteManager) rqliteDataDirPath() (string, error) {
dataDir := os.ExpandEnv(r.dataDir)
if strings.HasPrefix(dataDir, "~") {
home, _ := os.UserHomeDir()
dataDir = filepath.Join(home, dataDir[1:])
}
return filepath.Join(dataDir, "rqlite"), nil
}
func (r *RQLiteManager) resolveMigrationsDir() (string, error) {
productionPath := "/home/debros/src/migrations"
if _, err := os.Stat(productionPath); err == nil {
return productionPath, nil
}
return "migrations", nil
}
func (r *RQLiteManager) prepareDataDir() (string, error) {
rqliteDataDir, err := r.rqliteDataDirPath()
if err != nil {
return "", err
}
if err := os.MkdirAll(rqliteDataDir, 0755); err != nil {
return "", err
}
return rqliteDataDir, nil
}
func (r *RQLiteManager) hasExistingState(rqliteDataDir string) bool {
entries, err := os.ReadDir(rqliteDataDir)
if err != nil {
return false
}
for _, e := range entries {
if e.Name() != "." && e.Name() != ".." {
return true
}
}
return false
}
func (r *RQLiteManager) exponentialBackoff(attempt int, baseDelay time.Duration, maxDelay time.Duration) time.Duration {
delay := baseDelay * time.Duration(1<<uint(attempt))
if delay > maxDelay {
delay = maxDelay
}
return delay
}