Merge branch 'nightly' into bug-fixes

Makefile

@@ -21,7 +21,7 @@ test-e2e:
 
 .PHONY: build clean test run-node run-node2 run-node3 run-example deps tidy fmt vet lint clear-ports
 
-VERSION := 0.51.0-beta
+VERSION := 0.51.1-beta
 COMMIT  ?= $(shell git rev-parse --short HEAD 2>/dev/null || echo unknown)
 DATE    ?= $(shell date -u +%Y-%m-%dT%H:%M:%SZ)
 LDFLAGS := -X 'main.version=$(VERSION)' -X 'main.commit=$(COMMIT)' -X 'main.date=$(DATE)'
@@ -59,7 +59,7 @@ run-node2:
 # Usage: make run-node3 JOINADDR=/ip4/127.0.0.1/tcp/5001 HTTP=5003 RAFT=7003 P2P=4003
 run-node3:
 	@echo "Starting regular node3 with config..."
-	go run ./cmd/node --config configs/node.yaml
+	go run ./cmd/node --config configs/node3.yaml
 
 # Run gateway HTTP server
 # Usage examples:

cmd/node/main.go

@@ -106,11 +106,13 @@ func check_if_should_open_help(help *bool) {
 }
 
 // select_data_dir selects the data directory for the node
-func select_data_dir(dataDir *string, nodeID *string) {
+// If none of (hasConfigFile, nodeID, dataDir) are present, throw an error and do not start
+func select_data_dir(dataDir *string, nodeID *string, hasConfigFile bool) {
 	logger := setup_logger(logging.ComponentNode)
 
-	if *nodeID == "" {
+	if !hasConfigFile && (*nodeID == "" || nodeID == nil) && (*dataDir == "" || dataDir == nil) {
-		*dataDir = "./data/node"
+		logger.Error("No config file, node ID, or data directory specified. Please provide at least one. Refusing to start.")
+		os.Exit(1)
 	}
 
 	logger.Info("Successfully selected Data Directory of: %s", zap.String("dataDir", *dataDir))
@@ -193,10 +195,10 @@ func load_args_into_config(cfg *config.Config, p2pPort, rqlHTTP, rqlRaft *int, r
 func main() {
 	logger := setup_logger(logging.ComponentNode)
 
-	_, dataDir, nodeID, p2pPort, rqlHTTP, rqlRaft, rqlJoinAddr, advAddr, help := parse_and_return_network_flags()
+	configPath, dataDir, nodeID, p2pPort, rqlHTTP, rqlRaft, rqlJoinAddr, advAddr, help := parse_and_return_network_flags()
 
 	check_if_should_open_help(help)
-	select_data_dir(dataDir, nodeID)
+	select_data_dir(dataDir, nodeID, *configPath != "")
 
 	// Load Node Configuration
 	var cfg *config.Config

pkg/discovery/discovery.go

@@ -2,15 +2,37 @@ package discovery
 
 import (
 	"context"
+	"encoding/json"
 	"errors"
+	"io"
 	"time"
 
 	"github.com/libp2p/go-libp2p/core/host"
 	"github.com/libp2p/go-libp2p/core/network"
 	"github.com/libp2p/go-libp2p/core/peer"
+	"github.com/multiformats/go-multiaddr"
 	"go.uber.org/zap"
 )
 
+// Protocol ID for peer exchange
+const PeerExchangeProtocol = "/debros/peer-exchange/1.0.0"
+
+// PeerExchangeRequest represents a request for peer information
+type PeerExchangeRequest struct {
+	Limit int `json:"limit"`
+}
+
+// PeerExchangeResponse represents a list of peers to exchange
+type PeerExchangeResponse struct {
+	Peers []PeerInfo `json:"peers"`
+}
+
+// PeerInfo contains peer identity and addresses
+type PeerInfo struct {
+	ID    string   `json:"id"`
+	Addrs []string `json:"addrs"`
+}
+
 // Manager handles peer discovery operations without a DHT dependency.
 // Note: The constructor intentionally accepts a second parameter of type
 // interface{} to remain source-compatible with previous call sites that
@@ -43,6 +65,75 @@ func NewManagerSimple(h host.Host, logger *zap.Logger) *Manager {
 	return NewManager(h, nil, logger)
 }
 
+// StartProtocolHandler registers the peer exchange protocol handler on the host
+func (d *Manager) StartProtocolHandler() {
+	d.host.SetStreamHandler(PeerExchangeProtocol, d.handlePeerExchangeStream)
+	d.logger.Debug("Registered peer exchange protocol handler")
+}
+
+// handlePeerExchangeStream handles incoming peer exchange requests
+func (d *Manager) handlePeerExchangeStream(s network.Stream) {
+	defer s.Close()
+
+	// Read request
+	var req PeerExchangeRequest
+	decoder := json.NewDecoder(s)
+	if err := decoder.Decode(&req); err != nil {
+		d.logger.Debug("Failed to decode peer exchange request", zap.Error(err))
+		return
+	}
+
+	// Get local peer list
+	peers := d.host.Peerstore().Peers()
+	if req.Limit <= 0 {
+		req.Limit = 10 // Default limit
+	}
+	if req.Limit > len(peers) {
+		req.Limit = len(peers)
+	}
+
+	// Build response with peer information
+	resp := PeerExchangeResponse{Peers: make([]PeerInfo, 0, req.Limit)}
+	added := 0
+
+	for _, pid := range peers {
+		if added >= req.Limit {
+			break
+		}
+		// Skip self
+		if pid == d.host.ID() {
+			continue
+		}
+
+		addrs := d.host.Peerstore().Addrs(pid)
+		if len(addrs) == 0 {
+			continue
+		}
+
+		// Convert addresses to strings
+		addrStrs := make([]string, len(addrs))
+		for i, addr := range addrs {
+			addrStrs[i] = addr.String()
+		}
+
+		resp.Peers = append(resp.Peers, PeerInfo{
+			ID:    pid.String(),
+			Addrs: addrStrs,
+		})
+		added++
+	}
+
+	// Send response
+	encoder := json.NewEncoder(s)
+	if err := encoder.Encode(&resp); err != nil {
+		d.logger.Debug("Failed to encode peer exchange response", zap.Error(err))
+		return
+	}
+
+	d.logger.Debug("Sent peer exchange response",
+		zap.Int("peer_count", len(resp.Peers)))
+}
+
 // Start begins periodic peer discovery
 func (d *Manager) Start(config Config) error {
 	ctx, cancel := context.WithCancel(context.Background())
@@ -142,7 +233,7 @@ func (d *Manager) discoverViaPeerstore(ctx context.Context, maxConnections int)
 }
 
 // discoverViaPeerExchange asks currently connected peers for addresses of other peers
-// by inspecting their peerstore entries. This is a lightweight peer-exchange approach.
+// by using an active peer exchange protocol.
 func (d *Manager) discoverViaPeerExchange(ctx context.Context, maxConnections int) int {
 	if maxConnections <= 0 {
 		return 0
@@ -150,31 +241,131 @@ func (d *Manager) discoverViaPeerExchange(ctx context.Context, maxConnections in
 
 	connected := 0
 	connectedPeers := d.host.Network().Peers()
+	if len(connectedPeers) == 0 {
+		return 0
+	}
+
+	d.logger.Debug("Starting peer exchange with connected peers",
+		zap.Int("num_peers", len(connectedPeers)))
 
 	for _, peerID := range connectedPeers {
 		if connected >= maxConnections {
 			break
 		}
 
-		peerInfo := d.host.Peerstore().PeerInfo(peerID)
+		// Request peer list from this peer
-		for _, addr := range peerInfo.Addrs {
+		peers := d.requestPeersFromPeer(ctx, peerID, maxConnections-connected)
+		if len(peers) == 0 {
+			continue
+		}
+
+		d.logger.Debug("Received peer list from peer",
+			zap.String("from_peer", peerID.String()[:8]+"..."),
+			zap.Int("peer_count", len(peers)))
+
+		// Try to connect to discovered peers
+		for _, peerInfo := range peers {
 			if connected >= maxConnections {
 				break
 			}
-			// Attempt to extract peer ID from addr is not done here; we rely on peerstore entries.
+
-			// If an address belongs to a known peer (already in peerstore), connect via that peer id.
+			// Parse peer ID and addresses
-			// No-op placeholder: actual exchange protocols would be required for richer discovery.
+			parsedID, err := peer.Decode(peerInfo.ID)
-			_ = addr
+			if err != nil {
+				d.logger.Debug("Failed to parse peer ID", zap.Error(err))
+				continue
+			}
+
+			// Skip self
+			if parsedID == d.host.ID() {
+				continue
+			}
+
+			// Skip if already connected
+			if d.host.Network().Connectedness(parsedID) != network.NotConnected {
+				continue
+			}
+
+			// Parse addresses
+			addrs := make([]multiaddr.Multiaddr, 0, len(peerInfo.Addrs))
+			for _, addrStr := range peerInfo.Addrs {
+				ma, err := multiaddr.NewMultiaddr(addrStr)
+				if err != nil {
+					d.logger.Debug("Failed to parse multiaddr", zap.Error(err))
+					continue
+				}
+				addrs = append(addrs, ma)
+			}
+
+			if len(addrs) == 0 {
+				continue
+			}
+
+			// Add to peerstore
+			d.host.Peerstore().AddAddrs(parsedID, addrs, time.Hour*24)
+
+			// Try to connect
+			connectCtx, cancel := context.WithTimeout(ctx, 15*time.Second)
+			peerAddrInfo := peer.AddrInfo{ID: parsedID, Addrs: addrs}
+
+			if err := d.host.Connect(connectCtx, peerAddrInfo); err != nil {
+				cancel()
+				d.logger.Debug("Failed to connect to discovered peer",
+					zap.String("peer_id", parsedID.String()[:8]+"..."),
+					zap.Error(err))
+				continue
+			}
+			cancel()
+
+			d.logger.Info("Successfully connected to discovered peer",
+				zap.String("peer_id", parsedID.String()[:8]+"..."),
+				zap.String("discovered_from", peerID.String()[:8]+"..."))
+			connected++
 		}
 	}
 
-	// The above is intentionally conservative (no active probing) because without an application-level
-	// peer-exchange protocol we cannot reliably learn new peer IDs from peers' addresses.
-	// Most useful discovery will come from bootstrap peers added to the peerstore by the caller.
-
 	return connected
 }
 
+// requestPeersFromPeer asks a specific peer for its peer list
+func (d *Manager) requestPeersFromPeer(ctx context.Context, peerID peer.ID, limit int) []PeerInfo {
+	// Open a stream to the peer
+	stream, err := d.host.NewStream(ctx, peerID, PeerExchangeProtocol)
+	if err != nil {
+		d.logger.Debug("Failed to open peer exchange stream",
+			zap.String("peer_id", peerID.String()[:8]+"..."),
+			zap.Error(err))
+		return nil
+	}
+	defer stream.Close()
+
+	// Send request
+	req := PeerExchangeRequest{Limit: limit}
+	encoder := json.NewEncoder(stream)
+	if err := encoder.Encode(&req); err != nil {
+		d.logger.Debug("Failed to send peer exchange request", zap.Error(err))
+		return nil
+	}
+
+	// Set read deadline
+	if err := stream.SetReadDeadline(time.Now().Add(10 * time.Second)); err != nil {
+		d.logger.Debug("Failed to set read deadline", zap.Error(err))
+		return nil
+	}
+
+	// Read response
+	var resp PeerExchangeResponse
+	decoder := json.NewDecoder(stream)
+	if err := decoder.Decode(&resp); err != nil {
+		if err != io.EOF {
+			d.logger.Debug("Failed to read peer exchange response", zap.Error(err))
+		}
+		return nil
+	}
+
+	return resp.Peers
+}
+
 // connectToPeer attempts to connect to a specific peer using its peerstore info.
 func (d *Manager) connectToPeer(ctx context.Context, peerID peer.ID) error {
 	peerInfo := d.host.Peerstore().PeerInfo(peerID)

pkg/node/node.go

@@ -6,14 +6,12 @@ import (
 	mathrand "math/rand"
 	"os"
 	"path/filepath"
-	"strings"
 	"time"
 
 	"github.com/libp2p/go-libp2p"
 	libp2ppubsub "github.com/libp2p/go-libp2p-pubsub"
 	"github.com/libp2p/go-libp2p/core/crypto"
 	"github.com/libp2p/go-libp2p/core/host"
-	"github.com/libp2p/go-libp2p/core/network"
 	"github.com/libp2p/go-libp2p/core/peer"
 
 	noise "github.com/libp2p/go-libp2p/p2p/security/noise"
@@ -22,6 +20,7 @@ import (
 	"go.uber.org/zap"
 
 	"github.com/DeBrosOfficial/network/pkg/config"
+	"github.com/DeBrosOfficial/network/pkg/discovery"
 	"github.com/DeBrosOfficial/network/pkg/encryption"
 	"github.com/DeBrosOfficial/network/pkg/logging"
 	"github.com/DeBrosOfficial/network/pkg/pubsub"
@@ -38,11 +37,13 @@ type Node struct {
 	rqliteAdapter *database.RQLiteAdapter
 
 	// Peer discovery
-	discoveryCancel context.CancelFunc
 	bootstrapCancel context.CancelFunc
 
 	// PubSub
 	pubsub *pubsub.ClientAdapter
+
+	// Discovery
+	discoveryManager *discovery.Manager
 }
 
 // NewNode creates a new network node
@@ -363,7 +364,11 @@ func (n *Node) startLibP2P() error {
 		}
 	}
 
-	n.logger.ComponentInfo(logging.ComponentNode, "LibP2P host started successfully - using bootstrap + peer exchange discovery")
+	// Initialize discovery manager with peer exchange protocol
+	n.discoveryManager = discovery.NewManager(h, nil, n.logger.Logger)
+	n.discoveryManager.StartProtocolHandler()
+
+	n.logger.ComponentInfo(logging.ComponentNode, "LibP2P host started successfully - using active peer exchange discovery")
 
 	// Start peer discovery and monitoring
 	n.startPeerDiscovery()
@@ -421,131 +426,31 @@ func (n *Node) GetPeerID() string {
 
 // startPeerDiscovery starts periodic peer discovery for the node
 func (n *Node) startPeerDiscovery() {
-	// Create a cancellation context for discovery
+	if n.discoveryManager == nil {
-	ctx, cancel := context.WithCancel(context.Background())
+		n.logger.ComponentWarn(logging.ComponentNode, "Discovery manager not initialized")
-	n.discoveryCancel = cancel
-
-	// Start bootstrap peer connections immediately
-	go func() {
-		n.connectToBootstrapPeers(ctx)
-
-		// Periodic peer discovery using interval from config
-		ticker := time.NewTicker(n.config.Discovery.DiscoveryInterval)
-		defer ticker.Stop()
-
-		for {
-			select {
-			case <-ctx.Done():
-				return
-			case <-ticker.C:
-				n.discoverPeers(ctx)
-			}
-		}
-	}()
-}
-
-// discoverPeers discovers and connects to new peers using peer exchange
-func (n *Node) discoverPeers(ctx context.Context) {
-	if n.host == nil {
 		return
 	}
 
-	connectedPeers := n.host.Network().Peers()
+	// Start the discovery manager with config from node config
-	initialCount := len(connectedPeers)
+	discoveryConfig := discovery.Config{
+		DiscoveryInterval: n.config.Discovery.DiscoveryInterval,
+		MaxConnections:    n.config.Node.MaxConnections,
+	}
 
-	if initialCount == 0 {
+	if err := n.discoveryManager.Start(discoveryConfig); err != nil {
-		// No peers connected - exponential backoff system handles bootstrap reconnection
+		n.logger.ComponentWarn(logging.ComponentNode, "Failed to start discovery manager", zap.Error(err))
-		n.logger.ComponentDebug(logging.ComponentNode, "No peers connected, relying on exponential backoff for bootstrap")
 		return
 	}
 
-	n.logger.ComponentDebug(logging.ComponentNode, "Discovering peers via peer exchange",
+	n.logger.ComponentInfo(logging.ComponentNode, "Peer discovery manager started",
-		zap.Int("current_peers", initialCount))
+		zap.Duration("interval", discoveryConfig.DiscoveryInterval),
-
+		zap.Int("max_connections", discoveryConfig.MaxConnections))
-	// Strategy: Use peer exchange through libp2p's identify protocol
-	// LibP2P automatically exchanges peer information when peers connect
-	// We just need to try connecting to peers in our peerstore
-
-	newConnections := n.discoverViaPeerExchange(ctx)
-
-	finalPeerCount := len(n.host.Network().Peers())
-
-	if newConnections > 0 {
-		n.logger.ComponentInfo(logging.ComponentNode, "Peer discovery completed",
-			zap.Int("new_connections", newConnections),
-			zap.Int("initial_peers", initialCount),
-			zap.Int("final_peers", finalPeerCount))
-	}
-}
-
-// discoverViaPeerExchange discovers new peers using peer exchange (identify protocol)
-func (n *Node) discoverViaPeerExchange(ctx context.Context) int {
-	connected := 0
-	maxConnections := 3 // Conservative limit to avoid overwhelming proxy
-
-	// Get all peers from peerstore (includes peers discovered through identify protocol)
-	allKnownPeers := n.host.Peerstore().Peers()
-
-	for _, knownPeer := range allKnownPeers {
-		if knownPeer == n.host.ID() {
-			continue
-		}
-
-		// Skip if already connected
-		if n.host.Network().Connectedness(knownPeer) == network.Connected {
-			continue
-		}
-
-		// Get addresses for this peer
-		addrs := n.host.Peerstore().Addrs(knownPeer)
-		if len(addrs) == 0 {
-			continue
-		}
-
-		// Filter to only standard P2P ports (avoid ephemeral client ports)
-		var validAddrs []multiaddr.Multiaddr
-		for _, addr := range addrs {
-			addrStr := addr.String()
-			// Keep addresses with standard P2P ports (4000-4999 range)
-			if strings.Contains(addrStr, ":400") {
-				validAddrs = append(validAddrs, addr)
-			}
-		}
-
-		if len(validAddrs) == 0 {
-			continue
-		}
-
-		// Try to connect with shorter timeout (proxy connections are slower)
-		connectCtx, cancel := context.WithTimeout(ctx, 15*time.Second)
-		peerInfo := peer.AddrInfo{ID: knownPeer, Addrs: validAddrs}
-
-		if err := n.host.Connect(connectCtx, peerInfo); err != nil {
-			cancel()
-			n.logger.ComponentDebug(logging.ComponentNode, "Failed to connect to peer via exchange",
-				zap.String("peer", knownPeer.String()),
-				zap.Error(err))
-			continue
-		}
-		cancel()
-
-		n.logger.ComponentInfo(logging.ComponentNode, "Connected to new peer via peer exchange",
-			zap.String("peer", knownPeer.String()))
-		connected++
-
-		if connected >= maxConnections {
-			break
-		}
-	}
-
-	return connected
 }
 
 // stopPeerDiscovery stops peer discovery
 func (n *Node) stopPeerDiscovery() {
-	if n.discoveryCancel != nil {
+	if n.discoveryManager != nil {
-		n.discoveryCancel()
+		n.discoveryManager.Stop()
-		n.discoveryCancel = nil
 	}
 	n.logger.ComponentInfo(logging.ComponentNode, "Peer discovery stopped")
 }