From 279c03df825249a7d1cdb24ec61d88372e0b2662 Mon Sep 17 00:00:00 2001 From: anonpenguin23 Date: Fri, 24 Oct 2025 09:52:18 +0300 Subject: [PATCH 1/6] Enhance configuration validation for nodes and gateways, implementing strict YAML decoding to reject unknown fields. Added comprehensive validation checks for node types, listen addresses, database settings, and discovery parameters. Updated README with configuration validation details and examples. --- Makefile | 2 +- README.md | 278 +++++----------- cmd/gateway/config.go | 59 ++-- cmd/node/main.go | 119 ++++--- pkg/config/config.go | 9 +- pkg/config/validate.go | 561 +++++++++++++++++++++++++++++++++ pkg/config/validate_test.go | 409 ++++++++++++++++++++++++ pkg/config/yaml.go | 19 ++ pkg/gateway/config_validate.go | 117 +++++++ 9 files changed, 1277 insertions(+), 296 deletions(-) create mode 100644 pkg/config/validate.go create mode 100644 pkg/config/validate_test.go create mode 100644 pkg/config/yaml.go create mode 100644 pkg/gateway/config_validate.go diff --git a/Makefile b/Makefile index 0a664c1..a77870b 100644 --- a/Makefile +++ b/Makefile @@ -21,7 +21,7 @@ test-e2e: .PHONY: build clean test run-node run-node2 run-node3 run-example deps tidy fmt vet lint clear-ports -VERSION := 0.51.2-beta +VERSION := 0.51.3-beta COMMIT ?= $(shell git rev-parse --short HEAD 2>/dev/null || echo unknown) DATE ?= $(shell date -u +%Y-%m-%dT%H:%M:%SZ) LDFLAGS := -X 'main.version=$(VERSION)' -X 'main.commit=$(COMMIT)' -X 'main.date=$(DATE)' diff --git a/README.md b/README.md index ef538c1..8f447fa 100644 --- a/README.md +++ b/README.md @@ -185,20 +185,21 @@ sudo journalctl -u debros-node.service -f ```yaml node: id: "" + type: "bootstrap" listen_addresses: - "/ip4/0.0.0.0/tcp/4001" data_dir: "./data/bootstrap" max_connections: 100 database: - data_dir: "./data/db" + data_dir: "./data/bootstrap/rqlite" replication_factor: 3 shard_count: 16 max_database_size: 1073741824 backup_interval: 24h rqlite_port: 5001 rqlite_raft_port: 7001 - rqlite_join_address: "" # Bootstrap node does not join + rqlite_join_address: "" discovery: bootstrap_peers: [] @@ -223,20 +224,21 @@ logging: ```yaml node: id: "node2" + type: "node" listen_addresses: - "/ip4/0.0.0.0/tcp/4002" data_dir: "./data/node2" max_connections: 50 database: - data_dir: "./data/db" + data_dir: "./data/node2/rqlite" replication_factor: 3 shard_count: 16 max_database_size: 1073741824 backup_interval: 24h rqlite_port: 5002 rqlite_raft_port: 7002 - rqlite_join_address: "http://127.0.0.1:5001" + rqlite_join_address: "127.0.0.1:7001" discovery: bootstrap_peers: @@ -280,7 +282,7 @@ database: - backup_interval (duration) e.g., "24h". Default: 24h. - rqlite_port (int) RQLite HTTP API port. Default: 5001. - rqlite_raft_port (int) RQLite Raft port. Default: 7001. -- rqlite_join_address (string) HTTP address of an existing RQLite node to join. Empty for bootstrap. +- rqlite_join_address (string) Raft address of an existing RQLite node to join (host:port format). Empty for bootstrap. discovery: @@ -310,25 +312,25 @@ Example node.yaml ```yaml node: id: "node2" + type: "node" listen_addresses: - "/ip4/0.0.0.0/tcp/4002" data_dir: "./data/node2" max_connections: 50 - disable_anonrc: true database: - data_dir: "./data/db" + data_dir: "./data/node2/rqlite" replication_factor: 3 shard_count: 16 max_database_size: 1073741824 backup_interval: 24h - rqlite_port: 5001 - rqlite_raft_port: 7001 - rqlite_join_address: "http://127.0.0.1:5001" + rqlite_port: 5002 + rqlite_raft_port: 7002 + rqlite_join_address: "127.0.0.1:7001" discovery: bootstrap_peers: - - "" + - "/ip4/127.0.0.1/tcp/4001/p2p/" discovery_interval: 15s bootstrap_port: 4001 http_adv_address: "127.0.0.1" @@ -339,7 +341,6 @@ security: enable_tls: false private_key_file: "" certificate_file: "" - auth_enabled: false logging: level: "info" @@ -382,6 +383,64 @@ bootstrap_peers: - **Database endpoints**: Set in config or via `RQLITE_NODES` env var. - **Development mode**: Use `NETWORK_DEV_LOCAL=1` for localhost defaults. +### Configuration Validation + +DeBros Network performs strict validation of all configuration files at startup. This ensures invalid configurations are caught immediately rather than causing silent failures later. + +#### Validation Features + +- **Strict YAML Parsing:** Unknown configuration keys are rejected with helpful error messages +- **Format Validation:** Multiaddrs, ports, durations, and other formats are validated for correctness +- **Cross-Field Validation:** Configuration constraints (e.g., bootstrap nodes don't join clusters) are enforced +- **Aggregated Error Reporting:** All validation errors are reported together, not one-by-one + +#### Common Validation Errors + +**Missing or Invalid `node.type`** +``` +node.type: must be one of [bootstrap node]; got "invalid" +``` +Solution: Set `type: "bootstrap"` or `type: "node"` + +**Invalid Bootstrap Peer Format** +``` +discovery.bootstrap_peers[0]: invalid multiaddr; expected /ip{4,6}/.../tcp//p2p/ +discovery.bootstrap_peers[0]: missing /p2p/ component +``` +Solution: Use full multiaddr format: `/ip4/127.0.0.1/tcp/4001/p2p/12D3KooW...` + +**Port Conflicts** +``` +database.rqlite_raft_port: must differ from database.rqlite_port (5001) +``` +Solution: Use different ports for HTTP and Raft (e.g., 5001 and 7001) + +**RQLite Join Address Issues (Nodes)** +``` +database.rqlite_join_address: required for node type (non-bootstrap) +database.rqlite_join_address: invalid format; expected host:port +``` +Solution: Non-bootstrap nodes must specify where to join the cluster. Use Raft port: `127.0.0.1:7001` + +**Bootstrap Nodes Cannot Join** +``` +database.rqlite_join_address: must be empty for bootstrap type +``` +Solution: Bootstrap nodes should have `rqlite_join_address: ""` + +**Invalid Listen Addresses** +``` +node.listen_addresses[0]: invalid TCP port 99999; port must be between 1 and 65535 +``` +Solution: Use valid ports [1-65535], e.g., `/ip4/0.0.0.0/tcp/4001` + +**Unknown Configuration Keys** +``` +invalid config: yaml: unmarshal errors: + line 42: field migrations_path not found in type config.DatabaseConfig +``` +Solution: Remove unsupported keys. Supported keys are documented in the YAML Reference section above. + --- ## CLI Usage @@ -744,8 +803,6 @@ ws.send("Hello, network!"); ## Development - - ### Project Structure ``` @@ -923,195 +980,4 @@ if err := client.FindOneBy(ctx, &u, "users", map[string]any{"email": "alice@exam `Save` inserts if PK is zero, otherwise updates by PK. `Remove` deletes by PK. -```go -// Insert (ID is zero) -u := &User{Email: "bob@example.com", FirstName: "Bob"} -_ = client.Save(ctx, u) // INSERT; sets u.ID if autoincrement - -// Update (ID is non-zero) -u.FirstName = "Bobby" -_ = client.Save(ctx, u) // UPDATE ... WHERE id = ? - -// Remove -_ = client.Remove(ctx, u) // DELETE ... WHERE id = ? - -``` - -### transactions - -Run multiple operations atomically. If your function returns an error, the transaction is rolled back; otherwise it commits. - -```go -err := client.Tx(ctx, func(tx rqlite.Tx) error { - // Read inside the same transaction - var me User - if err := tx.Query(ctx, &me, "SELECT * FROM users WHERE id = ?", 1); err != nil { - return err - } - - // Write inside the same transaction - me.LastName = "Updated" - if err := tx.Save(ctx, &me); err != nil { - return err - } - - // Complex query via builder - var recent []User - if err := tx.CreateQueryBuilder("users"). - OrderBy("created_at DESC"). - Limit(5). - GetMany(ctx, &recent); err != nil { - return err - } - - return nil // commit -}) - -``` - -### Repositories (optional, generic) - -Strongly-typed convenience layer bound to a table: - -```go -repo := client.Repository[User]("users") - -var many []User -_ = repo.Find(ctx, &many, map[string]any{"last_name": "A"}, rqlite.WithOrderBy("created_at DESC"), rqlite.WithLimit(10)) - -var one User -_ = repo.FindOne(ctx, &one, map[string]any{"email": "alice@example.com"}) - -_ = repo.Save(ctx, &one) -_ = repo.Remove(ctx, &one) - -``` - -### Migrations - -Option A: From the node (after rqlite is ready) - -```go -ctx := context.Background() -dirs := []string{ - "network/migrations", // default - "path/to/your/app/migrations", // extra -} - -if err := rqliteManager.ApplyMigrationsDirs(ctx, dirs); err != nil { - logger.Fatal("apply migrations failed", zap.Error(err)) -} -``` - -Option B: Using the adapter sql.DB - -```go -ctx := context.Background() -db := adapter.GetSQLDB() -dirs := []string{"network/migrations", "app/migrations"} - -if err := rqlite.ApplyMigrationsDirs(ctx, db, dirs, logger); err != nil { - logger.Fatal("apply migrations failed", zap.Error(err)) -} -``` - ---- - -## Troubleshooting - -### Common Issues - -#### Bootstrap Connection Failed - -- **Symptoms:** `Failed to connect to bootstrap peer` -- **Solutions:** Check node is running, firewall settings, peer ID validity. - -#### Database Operations Timeout - -- **Symptoms:** `Query timeout` or `No RQLite connection available` -- **Solutions:** Ensure RQLite ports are open, leader election completed, cluster join config correct. - -#### Message Delivery Failures - -- **Symptoms:** Messages not received by subscribers -- **Solutions:** Verify topic names, active subscriptions, network connectivity. - -#### High Memory Usage - -- **Symptoms:** Memory usage grows continuously -- **Solutions:** Unsubscribe when done, monitor connection pool, review message retention. - -#### Authentication Issues - -- **Symptoms:** `Authentication failed`, `Invalid wallet signature`, `JWT token expired` -- **Solutions:** - - Check wallet signature format (65-byte r||s||v hex) - - Ensure nonce matches exactly during wallet verification - - Verify wallet address case-insensitivity - - Use refresh endpoint or re-authenticate for expired tokens - - Clear credential cache if multi-wallet conflicts occur: `rm -rf ~/.debros/credentials` - -#### Gateway Issues - -- **Symptoms:** `Gateway connection refused`, `CORS errors`, `WebSocket disconnections` -- **Solutions:** - - Verify gateway is running and accessible on configured port - - Check CORS configuration for web applications - - Ensure proper authentication headers for protected endpoints - - Verify namespace configuration and enforcement - -#### Database Migration Issues - -- **Symptoms:** `Migration failed`, `SQL syntax error`, `Version conflict` -- **Solutions:** - - Check SQL syntax in migration files - - Ensure proper statement termination - - Verify migration file naming and sequential order - - Review migration logs for transaction rollbacks - -### Debugging & Health Checks - -```bash -export LOG_LEVEL=debug -./bin/network-cli health -./bin/network-cli peers -./bin/network-cli query "SELECT 1" -./bin/network-cli pubsub publish test "hello" -./bin/network-cli pubsub subscribe test 10s - -# Gateway health checks -curl http://localhost:6001/health -curl http://localhost:6001/v1/status -``` - -### Service Logs - -```bash -# Node service logs -sudo journalctl -u debros-node.service --since "1 hour ago" - -# Gateway service logs (if running as service) -sudo journalctl -u debros-gateway.service --since "1 hour ago" - -# Application logs -tail -f ./logs/gateway.log -tail -f ./logs/node.log -``` - ---- - -## License - -Distributed under the MIT License. See [LICENSE](LICENSE) for details. - ---- - -## Further Reading - -- [DeBros Network Documentation](https://network.debros.io/docs/) -- [RQLite Documentation](https://github.com/rqlite/rqlite) -- [LibP2P Documentation](https://libp2p.io) - ---- - -_This README reflects the latest architecture, configuration, and operational practices for the DeBros Network. For questions or contributions, please open an issue or pull request._ +``` \ No newline at end of file diff --git a/cmd/gateway/config.go b/cmd/gateway/config.go index acfbdcc..a32619e 100644 --- a/cmd/gateway/config.go +++ b/cmd/gateway/config.go @@ -2,13 +2,14 @@ package main import ( "flag" + "fmt" "os" "strings" + "github.com/DeBrosOfficial/network/pkg/config" "github.com/DeBrosOfficial/network/pkg/gateway" "github.com/DeBrosOfficial/network/pkg/logging" "go.uber.org/zap" - "gopkg.in/yaml.v3" ) // For transition, alias main.GatewayConfig to pkg/gateway.Config @@ -59,30 +60,32 @@ func parseGatewayConfig(logger *logging.ColoredLogger) *gateway.Config { const path = "configs/gateway.yaml" if data, err := os.ReadFile(path); err == nil { var y yamlCfg - if err := yaml.Unmarshal(data, &y); err != nil { - logger.ComponentWarn(logging.ComponentGeneral, "failed to parse configs/gateway.yaml; ignoring", zap.Error(err)) - } else { - if v := strings.TrimSpace(y.ListenAddr); v != "" { - cfg.ListenAddr = v - } - if v := strings.TrimSpace(y.ClientNamespace); v != "" { - cfg.ClientNamespace = v - } - if v := strings.TrimSpace(y.RQLiteDSN); v != "" { - cfg.RQLiteDSN = v - } - if len(y.BootstrapPeers) > 0 { - var bp []string - for _, p := range y.BootstrapPeers { - p = strings.TrimSpace(p) - if p != "" { - bp = append(bp, p) - } - } - if len(bp) > 0 { - cfg.BootstrapPeers = bp + // Use strict YAML decoding to reject unknown fields + if err := config.DecodeStrict(strings.NewReader(string(data)), &y); err != nil { + logger.ComponentError(logging.ComponentGeneral, "failed to parse configs/gateway.yaml", zap.Error(err)) + fmt.Fprintf(os.Stderr, "Configuration load error: %v\n", err) + os.Exit(1) + } + if v := strings.TrimSpace(y.ListenAddr); v != "" { + cfg.ListenAddr = v + } + if v := strings.TrimSpace(y.ClientNamespace); v != "" { + cfg.ClientNamespace = v + } + if v := strings.TrimSpace(y.RQLiteDSN); v != "" { + cfg.RQLiteDSN = v + } + if len(y.BootstrapPeers) > 0 { + var bp []string + for _, p := range y.BootstrapPeers { + p = strings.TrimSpace(p) + if p != "" { + bp = append(bp, p) } } + if len(bp) > 0 { + cfg.BootstrapPeers = bp + } } } } @@ -135,6 +138,16 @@ func parseGatewayConfig(logger *logging.ColoredLogger) *gateway.Config { cfg.BootstrapPeers = bp } + // Validate configuration + if errs := cfg.ValidateConfig(); len(errs) > 0 { + fmt.Fprintf(os.Stderr, "\nGateway configuration errors (%d):\n", len(errs)) + for _, err := range errs { + fmt.Fprintf(os.Stderr, " - %s\n", err) + } + fmt.Fprintf(os.Stderr, "\nPlease fix the configuration and try again.\n") + os.Exit(1) + } + logger.ComponentInfo(logging.ComponentGeneral, "Loaded gateway configuration", zap.String("addr", cfg.ListenAddr), zap.String("namespace", cfg.ClientNamespace), diff --git a/cmd/node/main.go b/cmd/node/main.go index 5c9f30c..70646ef 100644 --- a/cmd/node/main.go +++ b/cmd/node/main.go @@ -4,8 +4,6 @@ import ( "context" "flag" "fmt" - "io/ioutil" - "log" "os" "os/signal" "path/filepath" @@ -15,7 +13,6 @@ import ( "github.com/DeBrosOfficial/network/pkg/logging" "github.com/DeBrosOfficial/network/pkg/node" "go.uber.org/zap" - "gopkg.in/yaml.v3" ) // setup_logger initializes a logger for the given component. @@ -24,16 +21,15 @@ func setup_logger(component logging.Component) (logger *logging.ColoredLogger) { logger, err = logging.NewColoredLogger(component, true) if err != nil { - log.Fatalf("Failed to create logger: %v", err) + fmt.Fprintf(os.Stderr, "Failed to create logger: %v\n", err) + os.Exit(1) } return logger } -// parse_and_return_network_flags it initializes all the network flags coming from the .yaml files -func parse_and_return_network_flags() (configPath *string, dataDir, nodeID *string, p2pPort, rqlHTTP, rqlRaft *int, rqlJoinAddr *string, advAddr *string, help *bool) { - logger := setup_logger(logging.ComponentNode) - +// parse_flags parses command-line flags and returns them. +func parse_flags() (configPath, dataDir, nodeID *string, p2pPort, rqlHTTP, rqlRaft *int, rqlJoinAddr, advAddr *string, help *bool) { configPath = flag.String("config", "", "Path to config YAML file (overrides defaults)") dataDir = flag.String("data", "", "Data directory (auto-detected if not provided)") nodeID = flag.String("id", "", "Node identifier (for running multiple local nodes)") @@ -45,54 +41,20 @@ func parse_and_return_network_flags() (configPath *string, dataDir, nodeID *stri help = flag.Bool("help", false, "Show help") flag.Parse() - logger.Info("Successfully parsed all flags and arguments.") - - if *configPath != "" { - cfg, err := LoadConfigFromYAML(*configPath) - if err != nil { - logger.Error("Failed to load config from YAML", zap.Error(err)) - os.Exit(1) - } - logger.ComponentInfo(logging.ComponentNode, "Configuration loaded from YAML file", zap.String("path", *configPath)) - - // Instead of returning flag values, return config values - // For ListenAddresses, extract port from multiaddr string if possible, else use default - var p2pPortVal int - if len(cfg.Node.ListenAddresses) > 0 { - // Try to parse port from multiaddr string - var port int - _, err := fmt.Sscanf(cfg.Node.ListenAddresses[0], "/ip4/0.0.0.0/tcp/%d", &port) - if err == nil { - p2pPortVal = port - } else { - p2pPortVal = 4001 - } - } else { - p2pPortVal = 4001 - } - return configPath, - &cfg.Node.DataDir, - &cfg.Node.ID, - &p2pPortVal, - &cfg.Database.RQLitePort, - &cfg.Database.RQLiteRaftPort, - &cfg.Database.RQLiteJoinAddress, - &cfg.Discovery.HttpAdvAddress, - help - } - return } -// LoadConfigFromYAML loads a config from a YAML file +// LoadConfigFromYAML loads a config from a YAML file using strict decoding. func LoadConfigFromYAML(path string) (*config.Config, error) { - data, err := ioutil.ReadFile(path) + file, err := os.Open(path) if err != nil { - return nil, fmt.Errorf("failed to read config file: %w", err) + return nil, fmt.Errorf("failed to open config file: %w", err) } + defer file.Close() + var cfg config.Config - if err := yaml.Unmarshal(data, &cfg); err != nil { - return nil, fmt.Errorf("failed to unmarshal YAML: %w", err) + if err := config.DecodeStrict(file, &cfg); err != nil { + return nil, err } return &cfg, nil } @@ -101,7 +63,7 @@ func LoadConfigFromYAML(path string) (*config.Config, error) { func check_if_should_open_help(help *bool) { if *help { flag.Usage() - return + os.Exit(0) } } @@ -115,7 +77,9 @@ func select_data_dir(dataDir *string, nodeID *string, hasConfigFile bool) { os.Exit(1) } - logger.Info("Successfully selected Data Directory of: %s", zap.String("dataDir", *dataDir)) + if *dataDir != "" { + logger.Info("Data directory selected: %s", zap.String("dataDir", *dataDir)) + } } // startNode starts the node with the given configuration and port @@ -125,10 +89,12 @@ func startNode(ctx context.Context, cfg *config.Config, port int) error { n, err := node.NewNode(cfg) if err != nil { logger.Error("failed to create node: %v", zap.Error(err)) + return err } if err := n.Start(ctx); err != nil { logger.Error("failed to start node: %v", zap.Error(err)) + return err } // Save the peer ID to a file for CLI access (especially useful for bootstrap) @@ -152,8 +118,8 @@ func startNode(ctx context.Context, cfg *config.Config, port int) error { return n.Stop() } -// load_args_into_config applies command line argument overrides to the config -func load_args_into_config(cfg *config.Config, p2pPort, rqlHTTP, rqlRaft *int, rqlJoinAddr *string, advAddr *string, dataDir *string) { +// apply_flag_overrides applies command line argument overrides to the config +func apply_flag_overrides(cfg *config.Config, p2pPort, rqlHTTP, rqlRaft *int, rqlJoinAddr *string, advAddr *string, dataDir *string) { logger := setup_logger(logging.ComponentNode) // Apply RQLite HTTP port override @@ -183,8 +149,8 @@ func load_args_into_config(cfg *config.Config, p2pPort, rqlHTTP, rqlRaft *int, r } if *advAddr != "" { - cfg.Discovery.HttpAdvAddress = fmt.Sprintf("%s:%d", *advAddr, *rqlHTTP) - cfg.Discovery.RaftAdvAddress = fmt.Sprintf("%s:%d", *advAddr, *rqlRaft) + cfg.Discovery.HttpAdvAddress = fmt.Sprintf("%s:%d", *advAddr, cfg.Database.RQLitePort) + cfg.Discovery.RaftAdvAddress = fmt.Sprintf("%s:%d", *advAddr, cfg.Database.RQLiteRaftPort) } if *dataDir != "" { @@ -192,23 +158,52 @@ func load_args_into_config(cfg *config.Config, p2pPort, rqlHTTP, rqlRaft *int, r } } +// printValidationErrors prints aggregated validation errors and exits. +func printValidationErrors(errs []error) { + fmt.Fprintf(os.Stderr, "\nConfiguration errors (%d):\n", len(errs)) + for _, err := range errs { + fmt.Fprintf(os.Stderr, " - %s\n", err) + } + fmt.Fprintf(os.Stderr, "\nPlease fix the configuration and try again.\n") + os.Exit(1) +} + func main() { logger := setup_logger(logging.ComponentNode) - configPath, dataDir, nodeID, p2pPort, rqlHTTP, rqlRaft, rqlJoinAddr, advAddr, help := parse_and_return_network_flags() + // Parse command-line flags + configPath, dataDir, nodeID, p2pPort, rqlHTTP, rqlRaft, rqlJoinAddr, advAddr, help := parse_flags() check_if_should_open_help(help) select_data_dir(dataDir, nodeID, *configPath != "") - // Load Node Configuration + // Load configuration var cfg *config.Config - cfg = config.DefaultConfig() - logger.ComponentInfo(logging.ComponentNode, "Default configuration loaded successfully") + if *configPath != "" { + // Load from YAML with strict decoding + var err error + cfg, err = LoadConfigFromYAML(*configPath) + if err != nil { + logger.Error("Failed to load config from YAML", zap.Error(err)) + fmt.Fprintf(os.Stderr, "Configuration load error: %v\n", err) + os.Exit(1) + } + logger.ComponentInfo(logging.ComponentNode, "Configuration loaded from YAML file", zap.String("path", *configPath)) + } else { + // Use default configuration + cfg = config.DefaultConfig() + logger.ComponentInfo(logging.ComponentNode, "Default configuration loaded successfully") + } - // Apply command line argument overrides - load_args_into_config(cfg, p2pPort, rqlHTTP, rqlRaft, rqlJoinAddr, advAddr, dataDir) + // Apply command-line flag overrides + apply_flag_overrides(cfg, p2pPort, rqlHTTP, rqlRaft, rqlJoinAddr, advAddr, dataDir) logger.ComponentInfo(logging.ComponentNode, "Command line arguments applied to configuration") + // Validate configuration + if errs := cfg.Validate(); len(errs) > 0 { + printValidationErrors(errs) + } + // LibP2P uses configurable port (default 4001); RQLite uses 5001 (HTTP) and 7001 (Raft) port := *p2pPort @@ -219,7 +214,7 @@ func main() { zap.Int("p2p_port", port), zap.Strings("bootstrap_peers", cfg.Discovery.BootstrapPeers), zap.String("rqlite_join_address", cfg.Database.RQLiteJoinAddress), - zap.String("data_directory", *dataDir)) + zap.String("data_directory", cfg.Node.DataDir)) // Create context for graceful shutdown ctx, cancel := context.WithCancel(context.Background()) diff --git a/pkg/config/config.go b/pkg/config/config.go index 94da2c7..85e595d 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -109,10 +109,11 @@ func DefaultConfig() *Config { }, Discovery: DiscoveryConfig{ BootstrapPeers: []string{ - "/ip4/217.76.54.178/tcp/4001/p2p/12D3KooWKZnirPwNT4URtNSWK45f6vLkEs4xyUZ792F8Uj1oYnm1", - "/ip4/51.83.128.181/tcp/4001/p2p/12D3KooWBn2Zf1R8v9pEfmz7hDZ5b3oADxfejA3zJBYzKRCzgvhR", - "/ip4/155.133.27.199/tcp/4001/p2p/12D3KooWC69SBzM5QUgrLrfLWUykE8au32X5LwT7zwv9bixrQPm1", - "/ip4/217.76.56.2/tcp/4001/p2p/12D3KooWEiqJHvznxqJ5p2y8mUs6Ky6dfU1xTYFQbyKRCABfcZz4", + "/ip4/127.0.0.1/tcp/4001/p2p/12D3KooWHbcFcrGPXKUrHcxvd8MXEeUzRYyvY8fQcpEBxncSUwhj", + // "/ip4/217.76.54.178/tcp/4001/p2p/12D3KooWKZnirPwNT4URtNSWK45f6vLkEs4xyUZ792F8Uj1oYnm1", + // "/ip4/51.83.128.181/tcp/4001/p2p/12D3KooWBn2Zf1R8v9pEfmz7hDZ5b3oADxfejA3zJBYzKRCzgvhR", + // "/ip4/155.133.27.199/tcp/4001/p2p/12D3KooWC69SBzM5QUgrLrfLWUykE8au32X5LwT7zwv9bixrQPm1", + // "/ip4/217.76.56.2/tcp/4001/p2p/12D3KooWEiqJHvznxqJ5p2y8mUs6Ky6dfU1xTYFQbyKRCABfcZz4", }, BootstrapPort: 4001, // Default LibP2P port DiscoveryInterval: time.Second * 15, // Back to 15 seconds for testing diff --git a/pkg/config/validate.go b/pkg/config/validate.go new file mode 100644 index 0000000..6921ab4 --- /dev/null +++ b/pkg/config/validate.go @@ -0,0 +1,561 @@ +package config + +import ( + "fmt" + "net" + "os" + "path/filepath" + "strconv" + "strings" + + "github.com/multiformats/go-multiaddr" + manet "github.com/multiformats/go-multiaddr/net" +) + +// ValidationError represents a single validation error with context. +type ValidationError struct { + Path string // e.g., "discovery.bootstrap_peers[0]" + Message string // e.g., "invalid multiaddr" + Hint string // e.g., "expected /ip{4,6}/.../tcp//p2p/" +} + +func (e ValidationError) Error() string { + if e.Hint != "" { + return fmt.Sprintf("%s: %s; %s", e.Path, e.Message, e.Hint) + } + return fmt.Sprintf("%s: %s", e.Path, e.Message) +} + +// Validate performs comprehensive validation of the entire config. +// It aggregates all errors and returns them, allowing the caller to print all issues at once. +func (c *Config) Validate() []error { + var errs []error + + // Validate node config + errs = append(errs, c.validateNode()...) + // Validate database config + errs = append(errs, c.validateDatabase()...) + // Validate discovery config + errs = append(errs, c.validateDiscovery()...) + // Validate security config + errs = append(errs, c.validateSecurity()...) + // Validate logging config + errs = append(errs, c.validateLogging()...) + // Cross-field validations + errs = append(errs, c.validateCrossFields()...) + + return errs +} + +func (c *Config) validateNode() []error { + var errs []error + nc := c.Node + + // Validate type + if nc.Type != "bootstrap" && nc.Type != "node" { + errs = append(errs, ValidationError{ + Path: "node.type", + Message: fmt.Sprintf("must be one of [bootstrap node]; got %q", nc.Type), + }) + } + + // Validate listen_addresses + if len(nc.ListenAddresses) == 0 { + errs = append(errs, ValidationError{ + Path: "node.listen_addresses", + Message: "must not be empty", + }) + } + + seen := make(map[string]bool) + for i, addr := range nc.ListenAddresses { + path := fmt.Sprintf("node.listen_addresses[%d]", i) + + // Parse as multiaddr + ma, err := multiaddr.NewMultiaddr(addr) + if err != nil { + errs = append(errs, ValidationError{ + Path: path, + Message: fmt.Sprintf("invalid multiaddr: %v", err), + Hint: "expected /ip{4,6}/.../ tcp/", + }) + continue + } + + // Check for TCP and valid port + tcpAddr, err := manet.ToNetAddr(ma) + if err != nil { + errs = append(errs, ValidationError{ + Path: path, + Message: fmt.Sprintf("cannot convert multiaddr to network address: %v", err), + Hint: "ensure multiaddr contains /tcp/", + }) + continue + } + + tcpPort := tcpAddr.(*net.TCPAddr).Port + if tcpPort < 1 || tcpPort > 65535 { + errs = append(errs, ValidationError{ + Path: path, + Message: fmt.Sprintf("invalid TCP port %d", tcpPort), + Hint: "port must be between 1 and 65535", + }) + } + + if seen[addr] { + errs = append(errs, ValidationError{ + Path: path, + Message: "duplicate listen address", + }) + } + seen[addr] = true + } + + // Validate data_dir + if nc.DataDir == "" { + errs = append(errs, ValidationError{ + Path: "node.data_dir", + Message: "must not be empty", + }) + } else { + if err := validateDataDir(nc.DataDir); err != nil { + errs = append(errs, ValidationError{ + Path: "node.data_dir", + Message: err.Error(), + }) + } + } + + // Validate max_connections + if nc.MaxConnections <= 0 { + errs = append(errs, ValidationError{ + Path: "node.max_connections", + Message: fmt.Sprintf("must be > 0; got %d", nc.MaxConnections), + }) + } + + return errs +} + +func (c *Config) validateDatabase() []error { + var errs []error + dc := c.Database + + // Validate data_dir + if dc.DataDir == "" { + errs = append(errs, ValidationError{ + Path: "database.data_dir", + Message: "must not be empty", + }) + } else { + if err := validateDataDir(dc.DataDir); err != nil { + errs = append(errs, ValidationError{ + Path: "database.data_dir", + Message: err.Error(), + }) + } + } + + // Validate replication_factor + if dc.ReplicationFactor < 1 { + errs = append(errs, ValidationError{ + Path: "database.replication_factor", + Message: fmt.Sprintf("must be >= 1; got %d", dc.ReplicationFactor), + }) + } else if dc.ReplicationFactor%2 == 0 { + // Warn about even replication factor (Raft best practice: odd) + // For now we log a note but don't error + _ = fmt.Sprintf("note: database.replication_factor %d is even; Raft recommends odd numbers for quorum", dc.ReplicationFactor) + } + + // Validate shard_count + if dc.ShardCount < 1 { + errs = append(errs, ValidationError{ + Path: "database.shard_count", + Message: fmt.Sprintf("must be >= 1; got %d", dc.ShardCount), + }) + } + + // Validate max_database_size + if dc.MaxDatabaseSize < 0 { + errs = append(errs, ValidationError{ + Path: "database.max_database_size", + Message: fmt.Sprintf("must be >= 0; got %d", dc.MaxDatabaseSize), + }) + } + + // Validate rqlite_port + if dc.RQLitePort < 1 || dc.RQLitePort > 65535 { + errs = append(errs, ValidationError{ + Path: "database.rqlite_port", + Message: fmt.Sprintf("must be between 1 and 65535; got %d", dc.RQLitePort), + }) + } + + // Validate rqlite_raft_port + if dc.RQLiteRaftPort < 1 || dc.RQLiteRaftPort > 65535 { + errs = append(errs, ValidationError{ + Path: "database.rqlite_raft_port", + Message: fmt.Sprintf("must be between 1 and 65535; got %d", dc.RQLiteRaftPort), + }) + } + + // Ports must differ + if dc.RQLitePort == dc.RQLiteRaftPort { + errs = append(errs, ValidationError{ + Path: "database.rqlite_raft_port", + Message: fmt.Sprintf("must differ from database.rqlite_port (%d)", dc.RQLitePort), + }) + } + + // Validate rqlite_join_address context-dependently + if c.Node.Type == "node" { + if dc.RQLiteJoinAddress == "" { + errs = append(errs, ValidationError{ + Path: "database.rqlite_join_address", + Message: "required for node type (non-bootstrap)", + }) + } else { + if err := validateHostPort(dc.RQLiteJoinAddress); err != nil { + errs = append(errs, ValidationError{ + Path: "database.rqlite_join_address", + Message: err.Error(), + Hint: "expected format: host:port", + }) + } + } + } else if c.Node.Type == "bootstrap" { + if dc.RQLiteJoinAddress != "" { + errs = append(errs, ValidationError{ + Path: "database.rqlite_join_address", + Message: "must be empty for bootstrap type", + }) + } + } + + return errs +} + +func (c *Config) validateDiscovery() []error { + var errs []error + disc := c.Discovery + + // Validate discovery_interval + if disc.DiscoveryInterval <= 0 { + errs = append(errs, ValidationError{ + Path: "discovery.discovery_interval", + Message: fmt.Sprintf("must be > 0; got %v", disc.DiscoveryInterval), + }) + } + + // Validate bootstrap_port + if disc.BootstrapPort < 1 || disc.BootstrapPort > 65535 { + errs = append(errs, ValidationError{ + Path: "discovery.bootstrap_port", + Message: fmt.Sprintf("must be between 1 and 65535; got %d", disc.BootstrapPort), + }) + } + + // Validate bootstrap_peers context-dependently + if c.Node.Type == "node" { + if len(disc.BootstrapPeers) == 0 { + errs = append(errs, ValidationError{ + Path: "discovery.bootstrap_peers", + Message: "required for node type (must not be empty)", + }) + } + } + + // Validate each bootstrap peer multiaddr + seenPeers := make(map[string]bool) + for i, peer := range disc.BootstrapPeers { + path := fmt.Sprintf("discovery.bootstrap_peers[%d]", i) + + _, err := multiaddr.NewMultiaddr(peer) + if err != nil { + errs = append(errs, ValidationError{ + Path: path, + Message: fmt.Sprintf("invalid multiaddr: %v", err), + Hint: "expected /ip{4,6}/.../tcp//p2p/", + }) + continue + } + + // Check for /p2p/ component + if !strings.Contains(peer, "/p2p/") { + errs = append(errs, ValidationError{ + Path: path, + Message: "missing /p2p/ component", + Hint: "expected /ip{4,6}/.../tcp//p2p/", + }) + } + + // Extract TCP port by parsing the multiaddr string directly + // Look for /tcp/ in the peer string + tcpPortStr := extractTCPPort(peer) + if tcpPortStr == "" { + errs = append(errs, ValidationError{ + Path: path, + Message: "missing /tcp/ component", + Hint: "expected /ip{4,6}/.../tcp//p2p/", + }) + continue + } + + tcpPort, err := strconv.Atoi(tcpPortStr) + if err != nil || tcpPort < 1 || tcpPort > 65535 { + errs = append(errs, ValidationError{ + Path: path, + Message: fmt.Sprintf("invalid TCP port %s", tcpPortStr), + Hint: "port must be between 1 and 65535", + }) + } + + if seenPeers[peer] { + errs = append(errs, ValidationError{ + Path: path, + Message: "duplicate bootstrap peer", + }) + } + seenPeers[peer] = true + } + + // Validate http_adv_address + if disc.HttpAdvAddress != "" { + if err := validateHostOrHostPort(disc.HttpAdvAddress); err != nil { + errs = append(errs, ValidationError{ + Path: "discovery.http_adv_address", + Message: err.Error(), + Hint: "expected format: host or host:port", + }) + } + } + + // Validate raft_adv_address + if disc.RaftAdvAddress != "" { + if err := validateHostOrHostPort(disc.RaftAdvAddress); err != nil { + errs = append(errs, ValidationError{ + Path: "discovery.raft_adv_address", + Message: err.Error(), + Hint: "expected format: host or host:port", + }) + } + } + + return errs +} + +func (c *Config) validateSecurity() []error { + var errs []error + sec := c.Security + + // Validate logging level + if sec.EnableTLS { + if sec.PrivateKeyFile == "" { + errs = append(errs, ValidationError{ + Path: "security.private_key_file", + Message: "required when enable_tls is true", + }) + } else { + if err := validateFileReadable(sec.PrivateKeyFile); err != nil { + errs = append(errs, ValidationError{ + Path: "security.private_key_file", + Message: err.Error(), + }) + } + } + + if sec.CertificateFile == "" { + errs = append(errs, ValidationError{ + Path: "security.certificate_file", + Message: "required when enable_tls is true", + }) + } else { + if err := validateFileReadable(sec.CertificateFile); err != nil { + errs = append(errs, ValidationError{ + Path: "security.certificate_file", + Message: err.Error(), + }) + } + } + } + + return errs +} + +func (c *Config) validateLogging() []error { + var errs []error + log := c.Logging + + // Validate level + validLevels := map[string]bool{"debug": true, "info": true, "warn": true, "error": true} + if !validLevels[log.Level] { + errs = append(errs, ValidationError{ + Path: "logging.level", + Message: fmt.Sprintf("invalid value %q", log.Level), + Hint: "allowed values: debug, info, warn, error", + }) + } + + // Validate format + validFormats := map[string]bool{"json": true, "console": true} + if !validFormats[log.Format] { + errs = append(errs, ValidationError{ + Path: "logging.format", + Message: fmt.Sprintf("invalid value %q", log.Format), + Hint: "allowed values: json, console", + }) + } + + // Validate output_file + if log.OutputFile != "" { + dir := filepath.Dir(log.OutputFile) + if dir != "" && dir != "." { + if err := validateDirWritable(dir); err != nil { + errs = append(errs, ValidationError{ + Path: "logging.output_file", + Message: fmt.Sprintf("parent directory not writable: %v", err), + }) + } + } + } + + return errs +} + +func (c *Config) validateCrossFields() []error { + var errs []error + + // If node.type is invalid, don't run cross-checks + if c.Node.Type != "bootstrap" && c.Node.Type != "node" { + return errs + } + + // Cross-check rqlite_join_address vs node type + if c.Node.Type == "bootstrap" && c.Database.RQLiteJoinAddress != "" { + errs = append(errs, ValidationError{ + Path: "database.rqlite_join_address", + Message: "must be empty for bootstrap node type", + }) + } + + if c.Node.Type == "node" && c.Database.RQLiteJoinAddress == "" { + errs = append(errs, ValidationError{ + Path: "database.rqlite_join_address", + Message: "required for non-bootstrap node type", + }) + } + + return errs +} + +// Helper validation functions + +func validateDataDir(path string) error { + if path == "" { + return fmt.Errorf("must not be empty") + } + + if info, err := os.Stat(path); err == nil { + // Directory exists; check if it's a directory and writable + if !info.IsDir() { + return fmt.Errorf("path exists but is not a directory") + } + // Try to write a test file to check permissions + testFile := filepath.Join(path, ".write_test") + if err := os.WriteFile(testFile, []byte(""), 0644); err != nil { + return fmt.Errorf("directory not writable: %v", err) + } + os.Remove(testFile) + } else if os.IsNotExist(err) { + // Directory doesn't exist; check if parent is writable + parent := filepath.Dir(path) + if parent == "" || parent == "." { + parent = "." + } + if err := validateDirWritable(parent); err != nil { + return fmt.Errorf("parent directory not writable: %v", err) + } + } else { + return fmt.Errorf("cannot access path: %v", err) + } + + return nil +} + +func validateDirWritable(path string) error { + info, err := os.Stat(path) + if err != nil { + return fmt.Errorf("cannot access directory: %v", err) + } + if !info.IsDir() { + return fmt.Errorf("path is not a directory") + } + + // Try to write a test file + testFile := filepath.Join(path, ".write_test") + if err := os.WriteFile(testFile, []byte(""), 0644); err != nil { + return fmt.Errorf("directory not writable: %v", err) + } + os.Remove(testFile) + + return nil +} + +func validateFileReadable(path string) error { + _, err := os.Stat(path) + if err != nil { + return fmt.Errorf("cannot read file: %v", err) + } + return nil +} + +func validateHostPort(hostPort string) error { + parts := strings.Split(hostPort, ":") + if len(parts) != 2 { + return fmt.Errorf("expected format host:port") + } + + host := parts[0] + port := parts[1] + + if host == "" { + return fmt.Errorf("host must not be empty") + } + + portNum, err := strconv.Atoi(port) + if err != nil || portNum < 1 || portNum > 65535 { + return fmt.Errorf("port must be a number between 1 and 65535; got %q", port) + } + + return nil +} + +func validateHostOrHostPort(addr string) error { + // Try to parse as host:port first + if strings.Contains(addr, ":") { + return validateHostPort(addr) + } + + // Otherwise just check if it's a valid hostname/IP + if addr == "" { + return fmt.Errorf("address must not be empty") + } + + return nil +} + +func extractTCPPort(multiaddrStr string) string { + // Look for the /tcp/ protocol code + parts := strings.Split(multiaddrStr, "/") + for i := 0; i < len(parts); i++ { + if parts[i] == "tcp" { + // The port is the next part + if i+1 < len(parts) { + return parts[i+1] + } + break + } + } + return "" +} diff --git a/pkg/config/validate_test.go b/pkg/config/validate_test.go new file mode 100644 index 0000000..33de810 --- /dev/null +++ b/pkg/config/validate_test.go @@ -0,0 +1,409 @@ +package config + +import ( + "testing" + "time" +) + +func TestValidateNodeType(t *testing.T) { + tests := []struct { + name string + nodeType string + shouldError bool + }{ + {"bootstrap", "bootstrap", false}, + {"node", "node", false}, + {"invalid", "invalid-type", true}, + {"empty", "", true}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + cfg := &Config{ + Node: NodeConfig{Type: tt.nodeType, ListenAddresses: []string{"/ip4/0.0.0.0/tcp/4001"}, DataDir: ".", MaxConnections: 50}, + Database: DatabaseConfig{DataDir: ".", ReplicationFactor: 3, ShardCount: 16, MaxDatabaseSize: 1024, BackupInterval: 1 * time.Hour, RQLitePort: 5001, RQLiteRaftPort: 7001}, + Discovery: DiscoveryConfig{BootstrapPeers: []string{"/ip4/127.0.0.1/tcp/4001/p2p/12D3KooWHbcFcrGPXKUrHcxvd8MXEeUzRYyvY8fQcpEBxncSUwhj"}, DiscoveryInterval: 15 * time.Second, BootstrapPort: 4001, NodeNamespace: "default"}, + Logging: LoggingConfig{Level: "info", Format: "console"}, + } + errs := cfg.Validate() + if tt.shouldError && len(errs) == 0 { + t.Errorf("expected error, got none") + } + if !tt.shouldError && len(errs) > 0 { + t.Errorf("unexpected errors: %v", errs) + } + }) + } +} + +func TestValidateListenAddresses(t *testing.T) { + tests := []struct { + name string + addresses []string + shouldError bool + }{ + {"valid single", []string{"/ip4/0.0.0.0/tcp/4001"}, false}, + {"valid ipv6", []string{"/ip6/::/tcp/4001"}, false}, + {"invalid port", []string{"/ip4/0.0.0.0/tcp/99999"}, true}, + {"invalid port zero", []string{"/ip4/0.0.0.0/tcp/0"}, true}, + {"invalid multiaddr", []string{"invalid"}, true}, + {"empty", []string{}, true}, + {"duplicate", []string{"/ip4/0.0.0.0/tcp/4001", "/ip4/0.0.0.0/tcp/4001"}, true}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + cfg := &Config{ + Node: NodeConfig{Type: "node", ListenAddresses: tt.addresses, DataDir: ".", MaxConnections: 50}, + Database: DatabaseConfig{DataDir: ".", ReplicationFactor: 3, ShardCount: 16, MaxDatabaseSize: 1024, BackupInterval: 1 * time.Hour, RQLitePort: 5001, RQLiteRaftPort: 7001, RQLiteJoinAddress: "localhost:7001"}, + Discovery: DiscoveryConfig{BootstrapPeers: []string{"/ip4/127.0.0.1/tcp/4001/p2p/12D3KooWHbcFcrGPXKUrHcxvd8MXEeUzRYyvY8fQcpEBxncSUwhj"}, DiscoveryInterval: 15 * time.Second, BootstrapPort: 4001, NodeNamespace: "default"}, + Logging: LoggingConfig{Level: "info", Format: "console"}, + } + errs := cfg.Validate() + if tt.shouldError && len(errs) == 0 { + t.Errorf("expected error, got none") + } + if !tt.shouldError && len(errs) > 0 { + t.Errorf("unexpected errors: %v", errs) + } + }) + } +} + +func TestValidateReplicationFactor(t *testing.T) { + tests := []struct { + name string + replication int + shouldError bool + }{ + {"valid 1", 1, false}, + {"valid 3", 3, false}, + {"valid even", 2, false}, // warn but not error + {"invalid zero", 0, true}, + {"invalid negative", -1, true}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + cfg := &Config{ + Node: NodeConfig{Type: "node", ListenAddresses: []string{"/ip4/0.0.0.0/tcp/4001"}, DataDir: ".", MaxConnections: 50}, + Database: DatabaseConfig{DataDir: ".", ReplicationFactor: tt.replication, ShardCount: 16, MaxDatabaseSize: 1024, BackupInterval: 1 * time.Hour, RQLitePort: 5001, RQLiteRaftPort: 7001, RQLiteJoinAddress: "localhost:7001"}, + Discovery: DiscoveryConfig{BootstrapPeers: []string{"/ip4/127.0.0.1/tcp/4001/p2p/12D3KooWHbcFcrGPXKUrHcxvd8MXEeUzRYyvY8fQcpEBxncSUwhj"}, DiscoveryInterval: 15 * time.Second, BootstrapPort: 4001, NodeNamespace: "default"}, + Logging: LoggingConfig{Level: "info", Format: "console"}, + } + errs := cfg.Validate() + if tt.shouldError && len(errs) == 0 { + t.Errorf("expected error, got none") + } + if !tt.shouldError && len(errs) > 0 { + t.Errorf("unexpected errors: %v", errs) + } + }) + } +} + +func TestValidateRQLitePorts(t *testing.T) { + tests := []struct { + name string + httpPort int + raftPort int + shouldError bool + }{ + {"valid different", 5001, 7001, false}, + {"invalid same", 5001, 5001, true}, + {"invalid http port zero", 0, 7001, true}, + {"invalid raft port zero", 5001, 0, true}, + {"invalid http port too high", 99999, 7001, true}, + {"invalid raft port too high", 5001, 99999, true}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + cfg := &Config{ + Node: NodeConfig{Type: "node", ListenAddresses: []string{"/ip4/0.0.0.0/tcp/4001"}, DataDir: ".", MaxConnections: 50}, + Database: DatabaseConfig{DataDir: ".", ReplicationFactor: 3, ShardCount: 16, MaxDatabaseSize: 1024, BackupInterval: 1 * time.Hour, RQLitePort: tt.httpPort, RQLiteRaftPort: tt.raftPort, RQLiteJoinAddress: "localhost:7001"}, + Discovery: DiscoveryConfig{BootstrapPeers: []string{"/ip4/127.0.0.1/tcp/4001/p2p/12D3KooWHbcFcrGPXKUrHcxvd8MXEeUzRYyvY8fQcpEBxncSUwhj"}, DiscoveryInterval: 15 * time.Second, BootstrapPort: 4001, NodeNamespace: "default"}, + Logging: LoggingConfig{Level: "info", Format: "console"}, + } + errs := cfg.Validate() + if tt.shouldError && len(errs) == 0 { + t.Errorf("expected error, got none") + } + if !tt.shouldError && len(errs) > 0 { + t.Errorf("unexpected errors: %v", errs) + } + }) + } +} + +func TestValidateRQLiteJoinAddress(t *testing.T) { + tests := []struct { + name string + nodeType string + joinAddr string + shouldError bool + }{ + {"node with join", "node", "localhost:7001", false}, + {"node without join", "node", "", true}, + {"bootstrap with join", "bootstrap", "localhost:7001", true}, + {"bootstrap without join", "bootstrap", "", false}, + {"invalid join format", "node", "localhost", true}, + {"invalid join port", "node", "localhost:99999", true}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + cfg := &Config{ + Node: NodeConfig{Type: tt.nodeType, ListenAddresses: []string{"/ip4/0.0.0.0/tcp/4001"}, DataDir: ".", MaxConnections: 50}, + Database: DatabaseConfig{DataDir: ".", ReplicationFactor: 3, ShardCount: 16, MaxDatabaseSize: 1024, BackupInterval: 1 * time.Hour, RQLitePort: 5001, RQLiteRaftPort: 7001, RQLiteJoinAddress: tt.joinAddr}, + Discovery: DiscoveryConfig{BootstrapPeers: []string{"/ip4/127.0.0.1/tcp/4001/p2p/12D3KooWHbcFcrGPXKUrHcxvd8MXEeUzRYyvY8fQcpEBxncSUwhj"}, DiscoveryInterval: 15 * time.Second, BootstrapPort: 4001, NodeNamespace: "default"}, + Logging: LoggingConfig{Level: "info", Format: "console"}, + } + errs := cfg.Validate() + if tt.shouldError && len(errs) == 0 { + t.Errorf("expected error, got none") + } + if !tt.shouldError && len(errs) > 0 { + t.Errorf("unexpected errors: %v", errs) + } + }) + } +} + +func TestValidateBootstrapPeers(t *testing.T) { + validPeer := "/ip4/127.0.0.1/tcp/4001/p2p/12D3KooWHbcFcrGPXKUrHcxvd8MXEeUzRYyvY8fQcpEBxncSUwhj" + tests := []struct { + name string + nodeType string + peers []string + shouldError bool + }{ + {"node with peer", "node", []string{validPeer}, false}, + {"node without peer", "node", []string{}, true}, + {"bootstrap with peer", "bootstrap", []string{validPeer}, false}, + {"bootstrap without peer", "bootstrap", []string{}, false}, + {"invalid multiaddr", "node", []string{"invalid"}, true}, + {"missing p2p", "node", []string{"/ip4/127.0.0.1/tcp/4001"}, true}, + {"duplicate peer", "node", []string{validPeer, validPeer}, true}, + {"invalid port", "node", []string{"/ip4/127.0.0.1/tcp/99999/p2p/12D3KooWHbcFcrGPXKUrHcxvd8MXEeUzRYyvY8fQcpEBxncSUwhj"}, true}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + cfg := &Config{ + Node: NodeConfig{Type: tt.nodeType, ListenAddresses: []string{"/ip4/0.0.0.0/tcp/4001"}, DataDir: ".", MaxConnections: 50}, + Database: DatabaseConfig{DataDir: ".", ReplicationFactor: 3, ShardCount: 16, MaxDatabaseSize: 1024, BackupInterval: 1 * time.Hour, RQLitePort: 5001, RQLiteRaftPort: 7001, RQLiteJoinAddress: ""}, + Discovery: DiscoveryConfig{BootstrapPeers: tt.peers, DiscoveryInterval: 15 * time.Second, BootstrapPort: 4001, NodeNamespace: "default"}, + Logging: LoggingConfig{Level: "info", Format: "console"}, + } + errs := cfg.Validate() + if tt.shouldError && len(errs) == 0 { + t.Errorf("expected error, got none") + } + if !tt.shouldError && len(errs) > 0 { + t.Errorf("unexpected errors: %v", errs) + } + }) + } +} + +func TestValidateLoggingLevel(t *testing.T) { + tests := []struct { + name string + level string + shouldError bool + }{ + {"debug", "debug", false}, + {"info", "info", false}, + {"warn", "warn", false}, + {"error", "error", false}, + {"invalid", "verbose", true}, + {"empty", "", true}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + cfg := &Config{ + Node: NodeConfig{Type: "node", ListenAddresses: []string{"/ip4/0.0.0.0/tcp/4001"}, DataDir: ".", MaxConnections: 50}, + Database: DatabaseConfig{DataDir: ".", ReplicationFactor: 3, ShardCount: 16, MaxDatabaseSize: 1024, BackupInterval: 1 * time.Hour, RQLitePort: 5001, RQLiteRaftPort: 7001, RQLiteJoinAddress: "localhost:7001"}, + Discovery: DiscoveryConfig{BootstrapPeers: []string{"/ip4/127.0.0.1/tcp/4001/p2p/12D3KooWHbcFcrGPXKUrHcxvd8MXEeUzRYyvY8fQcpEBxncSUwhj"}, DiscoveryInterval: 15 * time.Second, BootstrapPort: 4001, NodeNamespace: "default"}, + Logging: LoggingConfig{Level: tt.level, Format: "console"}, + } + errs := cfg.Validate() + if tt.shouldError && len(errs) == 0 { + t.Errorf("expected error, got none") + } + if !tt.shouldError && len(errs) > 0 { + t.Errorf("unexpected errors: %v", errs) + } + }) + } +} + +func TestValidateLoggingFormat(t *testing.T) { + tests := []struct { + name string + format string + shouldError bool + }{ + {"json", "json", false}, + {"console", "console", false}, + {"invalid", "text", true}, + {"empty", "", true}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + cfg := &Config{ + Node: NodeConfig{Type: "node", ListenAddresses: []string{"/ip4/0.0.0.0/tcp/4001"}, DataDir: ".", MaxConnections: 50}, + Database: DatabaseConfig{DataDir: ".", ReplicationFactor: 3, ShardCount: 16, MaxDatabaseSize: 1024, BackupInterval: 1 * time.Hour, RQLitePort: 5001, RQLiteRaftPort: 7001, RQLiteJoinAddress: "localhost:7001"}, + Discovery: DiscoveryConfig{BootstrapPeers: []string{"/ip4/127.0.0.1/tcp/4001/p2p/12D3KooWHbcFcrGPXKUrHcxvd8MXEeUzRYyvY8fQcpEBxncSUwhj"}, DiscoveryInterval: 15 * time.Second, BootstrapPort: 4001, NodeNamespace: "default"}, + Logging: LoggingConfig{Level: "info", Format: tt.format}, + } + errs := cfg.Validate() + if tt.shouldError && len(errs) == 0 { + t.Errorf("expected error, got none") + } + if !tt.shouldError && len(errs) > 0 { + t.Errorf("unexpected errors: %v", errs) + } + }) + } +} + +func TestValidateMaxConnections(t *testing.T) { + tests := []struct { + name string + maxConn int + shouldError bool + }{ + {"valid 50", 50, false}, + {"valid 1", 1, false}, + {"invalid zero", 0, true}, + {"invalid negative", -1, true}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + cfg := &Config{ + Node: NodeConfig{Type: "node", ListenAddresses: []string{"/ip4/0.0.0.0/tcp/4001"}, DataDir: ".", MaxConnections: tt.maxConn}, + Database: DatabaseConfig{DataDir: ".", ReplicationFactor: 3, ShardCount: 16, MaxDatabaseSize: 1024, BackupInterval: 1 * time.Hour, RQLitePort: 5001, RQLiteRaftPort: 7001, RQLiteJoinAddress: "localhost:7001"}, + Discovery: DiscoveryConfig{BootstrapPeers: []string{"/ip4/127.0.0.1/tcp/4001/p2p/12D3KooWHbcFcrGPXKUrHcxvd8MXEeUzRYyvY8fQcpEBxncSUwhj"}, DiscoveryInterval: 15 * time.Second, BootstrapPort: 4001, NodeNamespace: "default"}, + Logging: LoggingConfig{Level: "info", Format: "console"}, + } + errs := cfg.Validate() + if tt.shouldError && len(errs) == 0 { + t.Errorf("expected error, got none") + } + if !tt.shouldError && len(errs) > 0 { + t.Errorf("unexpected errors: %v", errs) + } + }) + } +} + +func TestValidateDiscoveryInterval(t *testing.T) { + tests := []struct { + name string + interval time.Duration + shouldError bool + }{ + {"valid 15s", 15 * time.Second, false}, + {"valid 1s", 1 * time.Second, false}, + {"invalid zero", 0, true}, + {"invalid negative", -5 * time.Second, true}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + cfg := &Config{ + Node: NodeConfig{Type: "node", ListenAddresses: []string{"/ip4/0.0.0.0/tcp/4001"}, DataDir: ".", MaxConnections: 50}, + Database: DatabaseConfig{DataDir: ".", ReplicationFactor: 3, ShardCount: 16, MaxDatabaseSize: 1024, BackupInterval: 1 * time.Hour, RQLitePort: 5001, RQLiteRaftPort: 7001, RQLiteJoinAddress: "localhost:7001"}, + Discovery: DiscoveryConfig{BootstrapPeers: []string{"/ip4/127.0.0.1/tcp/4001/p2p/12D3KooWHbcFcrGPXKUrHcxvd8MXEeUzRYyvY8fQcpEBxncSUwhj"}, DiscoveryInterval: tt.interval, BootstrapPort: 4001, NodeNamespace: "default"}, + Logging: LoggingConfig{Level: "info", Format: "console"}, + } + errs := cfg.Validate() + if tt.shouldError && len(errs) == 0 { + t.Errorf("expected error, got none") + } + if !tt.shouldError && len(errs) > 0 { + t.Errorf("unexpected errors: %v", errs) + } + }) + } +} + +func TestValidateBootstrapPort(t *testing.T) { + tests := []struct { + name string + port int + shouldError bool + }{ + {"valid 4001", 4001, false}, + {"valid 4002", 4002, false}, + {"invalid zero", 0, true}, + {"invalid too high", 99999, true}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + cfg := &Config{ + Node: NodeConfig{Type: "node", ListenAddresses: []string{"/ip4/0.0.0.0/tcp/4001"}, DataDir: ".", MaxConnections: 50}, + Database: DatabaseConfig{DataDir: ".", ReplicationFactor: 3, ShardCount: 16, MaxDatabaseSize: 1024, BackupInterval: 1 * time.Hour, RQLitePort: 5001, RQLiteRaftPort: 7001, RQLiteJoinAddress: "localhost:7001"}, + Discovery: DiscoveryConfig{BootstrapPeers: []string{"/ip4/127.0.0.1/tcp/4001/p2p/12D3KooWHbcFcrGPXKUrHcxvd8MXEeUzRYyvY8fQcpEBxncSUwhj"}, DiscoveryInterval: 15 * time.Second, BootstrapPort: tt.port, NodeNamespace: "default"}, + Logging: LoggingConfig{Level: "info", Format: "console"}, + } + errs := cfg.Validate() + if tt.shouldError && len(errs) == 0 { + t.Errorf("expected error, got none") + } + if !tt.shouldError && len(errs) > 0 { + t.Errorf("unexpected errors: %v", errs) + } + }) + } +} + +func TestValidateCompleteConfig(t *testing.T) { + // Test a complete valid config + validCfg := &Config{ + Node: NodeConfig{ + Type: "node", + ID: "node1", + ListenAddresses: []string{"/ip4/0.0.0.0/tcp/4002"}, + DataDir: ".", + MaxConnections: 50, + }, + Database: DatabaseConfig{ + DataDir: ".", + ReplicationFactor: 3, + ShardCount: 16, + MaxDatabaseSize: 1073741824, + BackupInterval: 24 * time.Hour, + RQLitePort: 5002, + RQLiteRaftPort: 7002, + RQLiteJoinAddress: "127.0.0.1:7001", + }, + Discovery: DiscoveryConfig{ + BootstrapPeers: []string{ + "/ip4/127.0.0.1/tcp/4001/p2p/12D3KooWHbcFcrGPXKUrHcxvd8MXEeUzRYyvY8fQcpEBxncSUwhj", + }, + DiscoveryInterval: 15 * time.Second, + BootstrapPort: 4001, + HttpAdvAddress: "127.0.0.1", + NodeNamespace: "default", + }, + Security: SecurityConfig{ + EnableTLS: false, + }, + Logging: LoggingConfig{ + Level: "info", + Format: "console", + }, + } + + errs := validCfg.Validate() + if len(errs) > 0 { + t.Errorf("valid config should not have errors: %v", errs) + } +} diff --git a/pkg/config/yaml.go b/pkg/config/yaml.go new file mode 100644 index 0000000..de40a9f --- /dev/null +++ b/pkg/config/yaml.go @@ -0,0 +1,19 @@ +package config + +import ( + "fmt" + "io" + + "gopkg.in/yaml.v3" +) + +// DecodeStrict decodes YAML from a reader and rejects any unknown fields. +// This ensures the YAML only contains recognized configuration keys. +func DecodeStrict(r io.Reader, out interface{}) error { + decoder := yaml.NewDecoder(r) + decoder.KnownFields(true) + if err := decoder.Decode(out); err != nil { + return fmt.Errorf("invalid config: %w", err) + } + return nil +} diff --git a/pkg/gateway/config_validate.go b/pkg/gateway/config_validate.go new file mode 100644 index 0000000..ebb4686 --- /dev/null +++ b/pkg/gateway/config_validate.go @@ -0,0 +1,117 @@ +package gateway + +import ( + "fmt" + "net" + "net/url" + "strconv" + "strings" + + "github.com/multiformats/go-multiaddr" + manet "github.com/multiformats/go-multiaddr/net" +) + +// ValidateConfig performs comprehensive validation of gateway configuration. +// It returns aggregated errors, allowing the caller to print all issues at once. +func (c *Config) ValidateConfig() []error { + var errs []error + + // Validate listen_addr + if c.ListenAddr == "" { + errs = append(errs, fmt.Errorf("gateway.listen_addr: must not be empty")) + } else { + if err := validateListenAddr(c.ListenAddr); err != nil { + errs = append(errs, fmt.Errorf("gateway.listen_addr: %v", err)) + } + } + + // Validate client_namespace + if c.ClientNamespace == "" { + errs = append(errs, fmt.Errorf("gateway.client_namespace: must not be empty")) + } + + // Validate bootstrap_peers if provided + seenPeers := make(map[string]bool) + for i, peer := range c.BootstrapPeers { + path := fmt.Sprintf("gateway.bootstrap_peers[%d]", i) + + ma, err := multiaddr.NewMultiaddr(peer) + if err != nil { + errs = append(errs, fmt.Errorf("%s: invalid multiaddr: %v; expected /ip{4,6}/.../tcp//p2p/", path, err)) + continue + } + + // Check for /p2p/ component + if !strings.Contains(peer, "/p2p/") { + errs = append(errs, fmt.Errorf("%s: missing /p2p/ component; expected /ip{4,6}/.../tcp//p2p/", path)) + } + + // Try to extract TCP addr to validate port + tcpAddr, err := manet.ToNetAddr(ma) + if err != nil { + errs = append(errs, fmt.Errorf("%s: cannot convert to network address: %v", path, err)) + continue + } + + tcpPort := tcpAddr.(*net.TCPAddr).Port + if tcpPort < 1 || tcpPort > 65535 { + errs = append(errs, fmt.Errorf("%s: invalid TCP port %d; port must be between 1 and 65535", path, tcpPort)) + } + + if seenPeers[peer] { + errs = append(errs, fmt.Errorf("%s: duplicate bootstrap peer", path)) + } + seenPeers[peer] = true + } + + // Validate rqlite_dsn if provided + if c.RQLiteDSN != "" { + if err := validateRQLiteDSN(c.RQLiteDSN); err != nil { + errs = append(errs, fmt.Errorf("gateway.rqlite_dsn: %v", err)) + } + } + + return errs +} + +// validateListenAddr checks if a listen address is valid (host:port format) +func validateListenAddr(addr string) error { + host, port, err := net.SplitHostPort(addr) + if err != nil { + return fmt.Errorf("invalid format; expected host:port") + } + + portNum, err := strconv.Atoi(port) + if err != nil || portNum < 1 || portNum > 65535 { + return fmt.Errorf("port must be a number between 1 and 65535; got %q", port) + } + + // Allow empty host (for wildcard binds like :6001) + if host != "" && net.ParseIP(host) == nil { + // Try as hostname (may fail later during bind, but basic validation) + _, err := net.LookupHost(host) + if err != nil { + // Not an IP; assume it's a valid hostname for now + } + } + + return nil +} + +// validateRQLiteDSN checks if an RQLite DSN is a valid URL +func validateRQLiteDSN(dsn string) error { + u, err := url.Parse(dsn) + if err != nil { + return fmt.Errorf("invalid URL: %v", err) + } + + if u.Scheme != "http" && u.Scheme != "https" { + return fmt.Errorf("scheme must be http or https; got %q", u.Scheme) + } + + if u.Host == "" { + return fmt.Errorf("host must not be empty") + } + + return nil +} From 60affaec5c2220f1fe9e03ce50e5c260eaa58e85 Mon Sep 17 00:00:00 2001 From: anonpenguin23 Date: Fri, 24 Oct 2025 10:01:53 +0300 Subject: [PATCH 2/6] Enhance configuration validation for nodes and gateways, implementing strict YAML decoding to reject unknown fields. Added comprehensive validation checks for node types, listen addresses, database settings, and discovery parameters. Updated README with configuration validation details and examples. --- CHANGELOG.md | 7 ++ Makefile | 2 +- cmd/cli/main.go | 145 +++++++++++++++++++++++++++++++++ pkg/gateway/config_validate.go | 38 +++++++-- 4 files changed, 182 insertions(+), 10 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 29758a2..33d119d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,14 +8,21 @@ The format is based on [Keep a Changelog][keepachangelog] and adheres to [Semant ### Added +- Added validation for yaml files +- Added authenticaiton command on cli + ### Changed +- Updated readme + ### Deprecated ### Removed ### Fixed +- Regular nodes rqlite not starting + ## [0.51.2] - 2025-09-26 ### Added diff --git a/Makefile b/Makefile index a77870b..4e611b4 100644 --- a/Makefile +++ b/Makefile @@ -21,7 +21,7 @@ test-e2e: .PHONY: build clean test run-node run-node2 run-node3 run-example deps tidy fmt vet lint clear-ports -VERSION := 0.51.3-beta +VERSION := 0.51.4-beta COMMIT ?= $(shell git rev-parse --short HEAD 2>/dev/null || echo unknown) DATE ?= $(shell date -u +%Y-%m-%dT%H:%M:%SZ) LDFLAGS := -X 'main.version=$(VERSION)' -X 'main.commit=$(COMMIT)' -X 'main.date=$(DATE)' diff --git a/cmd/cli/main.go b/cmd/cli/main.go index e115113..f9acc47 100644 --- a/cmd/cli/main.go +++ b/cmd/cli/main.go @@ -76,6 +76,8 @@ func main() { handleConnect(args[0]) case "peer-id": handlePeerID() + case "auth": + handleAuth(args) case "help", "--help", "-h": showHelp() @@ -289,6 +291,145 @@ func handlePubSub(args []string) { } } +func handleAuth(args []string) { + if len(args) == 0 { + showAuthHelp() + return + } + + subcommand := args[0] + switch subcommand { + case "login": + handleAuthLogin() + case "logout": + handleAuthLogout() + case "whoami": + handleAuthWhoami() + case "status": + handleAuthStatus() + default: + fmt.Fprintf(os.Stderr, "Unknown auth command: %s\n", subcommand) + showAuthHelp() + os.Exit(1) + } +} + +func handleAuthLogin() { + gatewayURL := auth.GetDefaultGatewayURL() + fmt.Printf("šŸ” Authenticating with gateway at: %s\n", gatewayURL) + + // Use the wallet authentication flow + creds, err := auth.PerformWalletAuthentication(gatewayURL) + if err != nil { + fmt.Fprintf(os.Stderr, "āŒ Authentication failed: %v\n", err) + os.Exit(1) + } + + // Save credentials to file + if err := auth.SaveCredentialsForDefaultGateway(creds); err != nil { + fmt.Fprintf(os.Stderr, "āŒ Failed to save credentials: %v\n", err) + os.Exit(1) + } + + credsPath, _ := auth.GetCredentialsPath() + fmt.Printf("āœ… Authentication successful!\n") + fmt.Printf("šŸ“ Credentials saved to: %s\n", credsPath) + fmt.Printf("šŸŽÆ Wallet: %s\n", creds.Wallet) + fmt.Printf("šŸ¢ Namespace: %s\n", creds.Namespace) +} + +func handleAuthLogout() { + if err := auth.ClearAllCredentials(); err != nil { + fmt.Fprintf(os.Stderr, "āŒ Failed to clear credentials: %v\n", err) + os.Exit(1) + } + fmt.Println("āœ… Logged out successfully - all credentials have been cleared") +} + +func handleAuthWhoami() { + store, err := auth.LoadCredentials() + if err != nil { + fmt.Fprintf(os.Stderr, "āŒ Failed to load credentials: %v\n", err) + os.Exit(1) + } + + gatewayURL := auth.GetDefaultGatewayURL() + creds, exists := store.GetCredentialsForGateway(gatewayURL) + + if !exists || !creds.IsValid() { + fmt.Println("āŒ Not authenticated - run 'network-cli auth login' to authenticate") + os.Exit(1) + } + + fmt.Println("āœ… Authenticated") + fmt.Printf(" Wallet: %s\n", creds.Wallet) + fmt.Printf(" Namespace: %s\n", creds.Namespace) + fmt.Printf(" Issued At: %s\n", creds.IssuedAt.Format("2006-01-02 15:04:05")) + if !creds.ExpiresAt.IsZero() { + fmt.Printf(" Expires At: %s\n", creds.ExpiresAt.Format("2006-01-02 15:04:05")) + } + if !creds.LastUsedAt.IsZero() { + fmt.Printf(" Last Used: %s\n", creds.LastUsedAt.Format("2006-01-02 15:04:05")) + } + if creds.Plan != "" { + fmt.Printf(" Plan: %s\n", creds.Plan) + } +} + +func handleAuthStatus() { + store, err := auth.LoadCredentials() + if err != nil { + fmt.Fprintf(os.Stderr, "āŒ Failed to load credentials: %v\n", err) + os.Exit(1) + } + + gatewayURL := auth.GetDefaultGatewayURL() + creds, exists := store.GetCredentialsForGateway(gatewayURL) + + fmt.Println("šŸ” Authentication Status") + fmt.Printf(" Gateway URL: %s\n", gatewayURL) + + if !exists || creds == nil { + fmt.Println(" Status: āŒ Not authenticated") + return + } + + if !creds.IsValid() { + fmt.Println(" Status: āš ļø Credentials expired") + if !creds.ExpiresAt.IsZero() { + fmt.Printf(" Expired At: %s\n", creds.ExpiresAt.Format("2006-01-02 15:04:05")) + } + return + } + + fmt.Println(" Status: āœ… Authenticated") + fmt.Printf(" Wallet: %s\n", creds.Wallet) + fmt.Printf(" Namespace: %s\n", creds.Namespace) + if !creds.ExpiresAt.IsZero() { + fmt.Printf(" Expires: %s\n", creds.ExpiresAt.Format("2006-01-02 15:04:05")) + } + if !creds.LastUsedAt.IsZero() { + fmt.Printf(" Last Used: %s\n", creds.LastUsedAt.Format("2006-01-02 15:04:05")) + } +} + +func showAuthHelp() { + fmt.Printf("šŸ” Authentication Commands\n\n") + fmt.Printf("Usage: network-cli auth \n\n") + fmt.Printf("Subcommands:\n") + fmt.Printf(" login - Authenticate with wallet\n") + fmt.Printf(" logout - Clear stored credentials\n") + fmt.Printf(" whoami - Show current authentication status\n") + fmt.Printf(" status - Show detailed authentication info\n\n") + fmt.Printf("Examples:\n") + fmt.Printf(" network-cli auth login\n") + fmt.Printf(" network-cli auth whoami\n") + fmt.Printf(" network-cli auth status\n") + fmt.Printf(" network-cli auth logout\n\n") + fmt.Printf("Environment Variables:\n") + fmt.Printf(" DEBROS_GATEWAY_URL - Gateway URL (default: http://localhost:6001)\n") +} + func ensureAuthenticated() *auth.Credentials { gatewayURL := auth.GetDefaultGatewayURL() @@ -440,6 +581,7 @@ func showHelp() { fmt.Printf("Usage: network-cli [args...]\n\n") fmt.Printf("šŸ” Authentication: Commands requiring authentication will automatically prompt for wallet connection.\n\n") fmt.Printf("Commands:\n") + fmt.Printf(" auth šŸ” Authentication management (login, logout, whoami, status)\n") fmt.Printf(" health - Check network health\n") fmt.Printf(" peers - List connected peers\n") fmt.Printf(" status - Show network status\n") @@ -457,10 +599,13 @@ func showHelp() { fmt.Printf(" -t, --timeout - Operation timeout (default: 30s)\n") fmt.Printf(" --production - Connect to production bootstrap peers\n\n") fmt.Printf("Authentication:\n") + fmt.Printf(" Use 'network-cli auth login' to authenticate with your wallet\n") fmt.Printf(" Commands marked with šŸ” will automatically prompt for wallet authentication\n") fmt.Printf(" if no valid credentials are found. You can manage multiple wallets and\n") fmt.Printf(" choose between them during the authentication flow.\n\n") fmt.Printf("Examples:\n") + fmt.Printf(" network-cli auth login\n") + fmt.Printf(" network-cli auth whoami\n") fmt.Printf(" network-cli health\n") fmt.Printf(" network-cli peer-id\n") fmt.Printf(" network-cli peer-id --format json\n") diff --git a/pkg/gateway/config_validate.go b/pkg/gateway/config_validate.go index ebb4686..a185107 100644 --- a/pkg/gateway/config_validate.go +++ b/pkg/gateway/config_validate.go @@ -8,7 +8,6 @@ import ( "strings" "github.com/multiformats/go-multiaddr" - manet "github.com/multiformats/go-multiaddr/net" ) // ValidateConfig performs comprehensive validation of gateway configuration. @@ -35,7 +34,7 @@ func (c *Config) ValidateConfig() []error { for i, peer := range c.BootstrapPeers { path := fmt.Sprintf("gateway.bootstrap_peers[%d]", i) - ma, err := multiaddr.NewMultiaddr(peer) + _, err := multiaddr.NewMultiaddr(peer) if err != nil { errs = append(errs, fmt.Errorf("%s: invalid multiaddr: %v; expected /ip{4,6}/.../tcp//p2p/", path, err)) continue @@ -46,16 +45,16 @@ func (c *Config) ValidateConfig() []error { errs = append(errs, fmt.Errorf("%s: missing /p2p/ component; expected /ip{4,6}/.../tcp//p2p/", path)) } - // Try to extract TCP addr to validate port - tcpAddr, err := manet.ToNetAddr(ma) - if err != nil { - errs = append(errs, fmt.Errorf("%s: cannot convert to network address: %v", path, err)) + // Extract TCP port by parsing the multiaddr string directly + tcpPortStr := extractTCPPort(peer) + if tcpPortStr == "" { + errs = append(errs, fmt.Errorf("%s: missing /tcp/ component; expected /ip{4,6}/.../tcp//p2p/", path)) continue } - tcpPort := tcpAddr.(*net.TCPAddr).Port - if tcpPort < 1 || tcpPort > 65535 { - errs = append(errs, fmt.Errorf("%s: invalid TCP port %d; port must be between 1 and 65535", path, tcpPort)) + tcpPort, err := strconv.Atoi(tcpPortStr) + if err != nil || tcpPort < 1 || tcpPort > 65535 { + errs = append(errs, fmt.Errorf("%s: invalid TCP port %s; port must be between 1 and 65535", path, tcpPortStr)) } if seenPeers[peer] { @@ -115,3 +114,24 @@ func validateRQLiteDSN(dsn string) error { return nil } + +// extractTCPPort extracts the TCP port from a multiaddr string. +// It assumes the multiaddr is in the format /ip{4,6}/.../tcp//p2p/. +func extractTCPPort(multiaddrStr string) string { + // Find the last /tcp/ component + lastTCPIndex := strings.LastIndex(multiaddrStr, "/tcp/") + if lastTCPIndex == -1 { + return "" + } + + // Extract the port part after /tcp/ + portPart := multiaddrStr[lastTCPIndex+len("/tcp/"):] + + // Find the first / component after the port part + firstSlashIndex := strings.Index(portPart, "/") + if firstSlashIndex == -1 { + return portPart + } + + return portPart[:firstSlashIndex] +} From 8a9deb50ec98d0cb89c0f0138a4ad66384588837 Mon Sep 17 00:00:00 2001 From: anonpenguin23 Date: Fri, 24 Oct 2025 10:02:03 +0300 Subject: [PATCH 3/6] Enhance README with detailed authentication commands and flow. Introduced explicit commands for managing credentials, improved descriptions of authentication features, and added environment variable support for gateway URL configuration. --- README.md | 317 ++++++++---------------------------------------------- 1 file changed, 47 insertions(+), 270 deletions(-) diff --git a/README.md b/README.md index 8f447fa..c30e21e 100644 --- a/README.md +++ b/README.md @@ -445,6 +445,15 @@ Solution: Remove unsupported keys. Supported keys are documented in the YAML Ref ## CLI Usage +### Authentication Commands + +```bash +./bin/network-cli auth login # Authenticate with wallet +./bin/network-cli auth whoami # Show current authentication status +./bin/network-cli auth status # Show detailed authentication info +./bin/network-cli auth logout # Clear stored credentials +``` + ### Network Operations ```bash @@ -543,14 +552,39 @@ curl -X POST "$GW/v1/rqlite/drop-table" -H "Authorization: Bearer $API_KEY" -H ' ### Authentication -The CLI features an enhanced authentication system with automatic wallet detection and multi-wallet support: +The CLI features an enhanced authentication system with explicit command support and automatic wallet detection: -- **Automatic Authentication:** No manual auth commands required - authentication happens automatically when operations need credentials +#### Explicit Authentication Commands + +Use the `auth` command to manage your credentials: + +```bash +# Authenticate with your wallet (opens browser for signature) +./bin/network-cli auth login + +# Check if you're authenticated +./bin/network-cli auth whoami + +# View detailed authentication info +./bin/network-cli auth status + +# Clear all stored credentials +./bin/network-cli auth logout +``` + +Credentials are stored securely in `~/.debros/credentials.json` with restricted file permissions (readable only by owner). + +#### Key Features + +- **Explicit Authentication:** Use `auth login` command to authenticate with your wallet +- **Automatic Authentication:** Commands that require auth (query, pubsub, etc.) automatically prompt if needed - **Multi-Wallet Management:** Seamlessly switch between multiple wallet credentials - **Persistent Sessions:** Wallet credentials are automatically saved and restored between sessions - **Enhanced User Experience:** Streamlined authentication flow with better error handling and user feedback -When using operations that require authentication (storage, database, pubsub), the CLI will automatically: +#### Automatic Authentication Flow + +When using operations that require authentication (query, pubsub publish/subscribe), the CLI will automatically: 1. Check for existing valid credentials 2. Prompt for wallet authentication if needed @@ -564,6 +598,15 @@ When using operations that require authentication (storage, database, pubsub), t ./bin/network-cli pubsub publish notifications "Hello World" ``` +#### Environment Variables + +You can override the gateway URL used for authentication: + +```bash +export DEBROS_GATEWAY_URL="http://localhost:6001" +./bin/network-cli auth login +``` + --- ## HTTP Gateway @@ -714,270 +757,4 @@ GET /v1/pubsub/topics # List active topics - **PubSub** - WS Subscribe: `GET /v1/pubsub/ws?topic=` - Publish: `POST /v1/pubsub/publish` `{topic, data_base64}` ā†’ `{status:"ok"}` - - Topics: `GET /v1/pubsub/topics` ā†’ `{topics:[...]}` - -### Migrations - -- Add column: `ALTER TABLE users ADD COLUMN age INTEGER` -- Change type / add FK (recreate pattern): create `_new` table, copy data, drop old, rename. -- Always send as one `POST /v1/rqlite/transaction`. - -### Minimal examples - -TypeScript (Node) - -```ts -import { GatewayClient } from "../examples/sdk-typescript/src/client"; - -const client = new GatewayClient( - process.env.GATEWAY_BASE_URL!, - process.env.GATEWAY_API_KEY! -); -await client.createTable( - "CREATE TABLE users (id INTEGER PRIMARY KEY, name TEXT)" -); -const res = await client.query("SELECT name FROM users WHERE id = ?", [1]); -``` - -Python - -```python -import os, requests - -BASE = os.environ['GATEWAY_BASE_URL'] -KEY = os.environ['GATEWAY_API_KEY'] -H = { 'X-API-Key': KEY, 'Content-Type': 'application/json' } - -def query(sql, args=None): - r = requests.post(f'{BASE}/v1/rqlite/query', json={ 'sql': sql, 'args': args or [] }, headers=H, timeout=15) - r.raise_for_status() - return r.json()['rows'] -``` - -Go - -```go -req, _ := http.NewRequest(http.MethodPost, base+"/v1/rqlite/create-table", bytes.NewBufferString(`{"schema":"CREATE TABLE ..."}`)) -req.Header.Set("X-API-Key", apiKey) -req.Header.Set("Content-Type", "application/json") -resp, err := http.DefaultClient.Do(req) -``` - -### Security Features - -- **Namespace Enforcement:** All operations are automatically prefixed with namespace for isolation -- **CORS Support:** Configurable CORS policies (permissive for development, configurable for production) -- **Transport Security:** All network communications use Noise/TLS encryption -- **Authentication Middleware:** Flexible authentication with support for multiple credential types - -### Usage Examples - -#### Wallet Authentication Flow - -```bash -# 1. Get challenge (automatic) -curl -X POST http://localhost:6001/v1/auth/challenge - -# 2. Sign challenge with wallet (handled by client) -# 3. Verify signature (automatic) -curl -X POST http://localhost:6001/v1/auth/verify \ - -H "Content-Type: application/json" \ - -d '{"wallet":"0x...","nonce":"...","signature":"0x..."}' -``` - -#### Real-time Messaging - -```javascript -// WebSocket connection -const ws = new WebSocket("ws://localhost:6001/v1/pubsub/ws?topic=chat"); - -ws.onmessage = (event) => { - console.log("Received:", event.data); -}; - -// Send message -ws.send("Hello, network!"); -``` - ---- - -## Development - -### Project Structure - -``` -network/ -ā”œā”€ā”€ cmd/ -ā”‚ ā”œā”€ā”€ node/ # Network node executable -ā”‚ ā””ā”€ā”€ cli/ # Command-line interface -ā”œā”€ā”€ pkg/ -ā”‚ ā”œā”€ā”€ client/ # Client library -ā”‚ ā”œā”€ā”€ node/ # Node implementation -ā”‚ ā”œā”€ā”€ database/ # RQLite integration -ā”‚ ā”œā”€ā”€ pubsub/ # Pub/Sub messaging -ā”‚ ā”œā”€ā”€ config/ # Centralized config -ā”‚ ā””ā”€ā”€ discovery/ # Peer discovery (node only) -ā”œā”€ā”€ scripts/ # Install, test scripts -ā”œā”€ā”€ configs/ # YAML configs -ā”œā”€ā”€ bin/ # Built executables -``` - -### Build & Test - -```bash -make build # Build all executables -make test # Run unit tests -make clean # Clean build artifacts -``` - -### Local Multi-Node Testing - -```bash -scripts/test-multinode.sh -``` - ---- - -## Database Client (Go ORM-like) - -A lightweight ORM-like client over rqlite using Goā€™s `database/sql`. It provides: - -- Query/Exec for raw SQL -- A fluent QueryBuilder (`Where`, `InnerJoin`, `LeftJoin`, `OrderBy`, `GroupBy`, `Limit`, `Offset`) -- Simple repositories with `Find`/`FindOne` -- `Save`/`Remove` for entities with primary keys -- Transaction support via `Tx` - -### Installation - -- Ensure rqlite is running (the node starts and manages rqlite automatically). -- Import the client: - - Package: `github.com/DeBrosOfficial/network/pkg/rqlite` - -### Quick Start - -````go -package main - -import ( - "context" - "database/sql" - "time" - - "github.com/DeBrosOfficial/network/pkg/rqlite" - _ "github.com/rqlite/gorqlite/stdlib" -) - -type User struct { - ID int64 `db:"id,pk,auto"` - Email string `db:"email"` - FirstName string `db:"first_name"` - LastName string `db:"last_name"` - CreatedAt time.Time `db:"created_at"` -} - -func (User) TableName() string { return "users" } - -func main() { - ctx := context.Background() - - adapter, _ := rqlite.NewRQLiteAdapter(manager) - client := rqlite.NewClientFromAdapter(adapter) - - // Save (INSERT) - u := &User{Email: "alice@example.com", FirstName: "Alice", LastName: "A"} - _ = client.Save(ctx, u) // auto-sets u.ID if autoincrement is available - - // FindOneBy - var one User - _ = client.FindOneBy(ctx, &one, "users", map[string]any{"email": "alice@example.com"}) - - // QueryBuilder - var users []User - _ = client.CreateQueryBuilder("users"). - Where("email LIKE ?", "%@example.com"). - OrderBy("created_at DESC"). - Limit(10). - GetMany(ctx, &users) -} - -### Entities and Mapping - -- Use struct tags: `db:"column_name"`; the first tag value is the column name. -- Mark primary key: `db:"id,pk"` (and `auto` if autoincrement): `db:"id,pk,auto"`. -- Fallbacks: - - If no `db` tag is provided, the field name is used as the column (case-insensitive). - - If a field is named `ID`, it is treated as the primary key by default. - -```go -type Post struct { - ID int64 `db:"id,pk,auto"` - UserID int64 `db:"user_id"` - Title string `db:"title"` - Body string `db:"body"` - CreatedAt time.Time `db:"created_at"` -} -func (Post) TableName() string { return "posts" } -```` - -### Basic queries - -Raw SQL with scanning into structs or maps: - -```go -var users []User -err := client.Query(ctx, &users, "SELECT id, email, first_name, last_name, created_at FROM users WHERE email LIKE ?", "%@example.com") -if err != nil { - // handle -} - -var rows []map[string]any -_ = client.Query(ctx, &rows, "SELECT id, email FROM users WHERE id IN (?,?)", 1, 2) -``` - -### Query Buider - -Build complex SELECTs with joins, filters, grouping, ordering, and pagination. - -```go -var results []User -qb := client.CreateQueryBuilder("users u"). - InnerJoin("posts p", "p.user_id = u.id"). - Where("u.email LIKE ?", "%@example.com"). - AndWhere("p.created_at >= ?", "2024-01-01T00:00:00Z"). - GroupBy("u.id"). - OrderBy("u.created_at DESC"). - Limit(20). - Offset(0) - -if err := qb.GetMany(ctx, &results); err != nil { - // handle -} - -// Single row (LIMIT 1) -var one User -if err := qb.Limit(1).GetOne(ctx, &one); err != nil { - // handle sql.ErrNoRows, etc. -} -``` - -### FindBy / FindOneBy - -Simple map-based filters: - -```go -var active []User -_ = client.FindBy(ctx, &active, "users", map[string]any{"last_name": "A"}, rqlite.WithOrderBy("created_at DESC"), rqlite.WithLimit(50)) - -var u User -if err := client.FindOneBy(ctx, &u, "users", map[string]any{"email": "alice@example.com"}); err != nil { - // sql.ErrNoRows if not found -} -``` - -### Save / Remove - -`Save` inserts if PK is zero, otherwise updates by PK. -`Remove` deletes by PK. - -``` \ No newline at end of file + - Topics: `GET /v1/pubsub/topics` ā†’ ` \ No newline at end of file From 07172968227b52eb0f2401422f9b93923ff308c5 Mon Sep 17 00:00:00 2001 From: anonpenguin23 Date: Fri, 24 Oct 2025 11:03:50 +0300 Subject: [PATCH 4/6] Update configuration management and documentation. Added commands for generating and validating configuration files in the CLI, ensuring strict YAML decoding and improved error handling. Updated README to reflect new configuration paths and filesystem permissions. Enhanced Makefile to streamline node and gateway startup processes. --- CHANGELOG.md | 1 + Makefile | 94 ++++--- README.md | 552 +++++++++++++++++++++++++++-------------- cmd/cli/main.go | 332 +++++++++++++++++++++++++ cmd/gateway/config.go | 130 ++++------ cmd/node/main.go | 165 ++++++++---- pkg/config/config.go | 8 +- pkg/config/paths.go | 38 +++ pkg/config/validate.go | 31 ++- pkg/rqlite/rqlite.go | 30 ++- 10 files changed, 1029 insertions(+), 352 deletions(-) create mode 100644 pkg/config/paths.go diff --git a/CHANGELOG.md b/CHANGELOG.md index 33d119d..b41efd9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,6 +14,7 @@ The format is based on [Keep a Changelog][keepachangelog] and adheres to [Semant ### Changed - Updated readme +- Where we read .yaml files from and where data is saved to ~/.debros ### Deprecated diff --git a/Makefile b/Makefile index 4e611b4..2c93cdd 100644 --- a/Makefile +++ b/Makefile @@ -21,7 +21,7 @@ test-e2e: .PHONY: build clean test run-node run-node2 run-node3 run-example deps tidy fmt vet lint clear-ports -VERSION := 0.51.4-beta +VERSION := 0.51.5-beta COMMIT ?= $(shell git rev-parse --short HEAD 2>/dev/null || echo unknown) DATE ?= $(shell date -u +%Y-%m-%dT%H:%M:%SZ) LDFLAGS := -X 'main.version=$(VERSION)' -X 'main.commit=$(COMMIT)' -X 'main.date=$(DATE)' @@ -46,35 +46,35 @@ clean: # Run bootstrap node (auto-selects identity and data dir) run-node: - @echo "Starting bootstrap node with config..." - go run ./cmd/node --config configs/bootstrap.yaml + @echo "Starting bootstrap node..." + @echo "Config: ~/.debros/bootstrap.yaml" + @echo "Generate it with: network-cli config init --type bootstrap" + go run ./cmd/node --config node.yaml # Run second node (regular) - requires join address of bootstrap node # Usage: make run-node2 JOINADDR=/ip4/127.0.0.1/tcp/5001 HTTP=5002 RAFT=7002 P2P=4002 run-node2: - @echo "Starting regular node2 with config..." - go run ./cmd/node --config configs/node.yaml + @echo "Starting regular node (node.yaml)..." + @echo "Config: ~/.debros/node.yaml" + @echo "Generate it with: network-cli config init --type node --join localhost:5001 --bootstrap-peers ''" + go run ./cmd/node --config node2.yaml # Run third node (regular) - requires join address of bootstrap node # Usage: make run-node3 JOINADDR=/ip4/127.0.0.1/tcp/5001 HTTP=5003 RAFT=7003 P2P=4003 run-node3: - @echo "Starting regular node3 with config..." - go run ./cmd/node --config configs/node3.yaml + @echo "Starting regular node (node2.yaml)..." + @echo "Config: ~/.debros/node2.yaml" + @echo "Generate it with: network-cli config init --type node --name node2.yaml --join localhost:5001 --bootstrap-peers ''" + go run ./cmd/node --config node3.yaml # Run gateway HTTP server # Usage examples: -# make run-gateway # uses defaults (:8080, namespace=default) -# GATEWAY_ADDR=":8081" make run-gateway # override listen addr via env -# GATEWAY_NAMESPACE=myapp make run-gateway # set namespace -# GATEWAY_BOOTSTRAP_PEERS="/ip4/127.0.0.1/tcp/4001/p2p/" make run-gateway -# GATEWAY_REQUIRE_AUTH=1 GATEWAY_API_KEYS="key1:ns1,key2:ns2" make run-gateway +# make run-gateway # uses ~/.debros/gateway.yaml +# Config generated with: network-cli config init --type gateway run-gateway: @echo "Starting gateway HTTP server..." - GATEWAY_ADDR=$(or $(ADDR),$(GATEWAY_ADDR)) \ - GATEWAY_NAMESPACE=$(or $(NAMESPACE),$(GATEWAY_NAMESPACE)) \ - GATEWAY_BOOTSTRAP_PEERS=$(GATEWAY_BOOTSTRAP_PEERS) \ - GATEWAY_REQUIRE_AUTH=$(GATEWAY_REQUIRE_AUTH) \ - GATEWAY_API_KEYS=$(GATEWAY_API_KEYS) \ + @echo "Note: Config must be in ~/.debros/gateway.yaml" + @echo "Generate it with: network-cli config init --type gateway" go run ./cmd/gateway # Run basic usage example @@ -155,15 +155,28 @@ dev-setup: deps # Start development cluster (requires multiple terminals) dev-cluster: - @echo "To start a development cluster, run these commands in separate terminals:" - @echo "1. make run-node # Start bootstrap node (uses configs/bootstrap.yaml)" - @echo "2. make run-node2 # Start second node (uses configs/node.yaml)" - @echo "3. make run-node3 # Start third node (uses configs/node.yaml)" - @echo "4. make run-example # Test basic functionality" - @echo "5. make cli-health # Check network health" - @echo "6. make cli-peers # List peers" - @echo "7. make cli-storage-test # Test storage" - @echo "8. make cli-pubsub-test # Test messaging" + @echo "To start a development cluster with 3 nodes:" + @echo "" + @echo "1. Generate config files in ~/.debros:" + @echo " make build" + @echo " ./bin/network-cli config init --type bootstrap" + @echo " ./bin/network-cli config init --type node --name node.yaml --bootstrap-peers ''" + @echo " ./bin/network-cli config init --type node --name node2.yaml --bootstrap-peers ''" + @echo "" + @echo "2. Run in separate terminals:" + @echo " Terminal 1: make run-node # Start bootstrap node (bootstrap.yaml)" + @echo " Terminal 2: make run-node2 # Start node 1 (node.yaml)" + @echo " Terminal 3: make run-node3 # Start node 2 (node2.yaml)" + @echo " Terminal 4: make run-gateway # Start gateway" + @echo "" + @echo "3. Or run custom node with any config file:" + @echo " go run ./cmd/node --config custom-node.yaml" + @echo "" + @echo "4. Test:" + @echo " make cli-health # Check network health" + @echo " make cli-peers # List peers" + @echo " make cli-storage-test # Test storage" + @echo " make cli-pubsub-test # Test messaging" # Full development workflow dev: clean build test @@ -175,22 +188,43 @@ help: @echo " build - Build all executables" @echo " clean - Clean build artifacts" @echo " test - Run tests" + @echo "" + @echo "Configuration (NEW):" + @echo " First, generate config files in ~/.debros with:" + @echo " make build # Build CLI first" + @echo " ./bin/network-cli config init --type bootstrap # Generate bootstrap config" + @echo " ./bin/network-cli config init --type node --bootstrap-peers ''" + @echo " ./bin/network-cli config init --type gateway" + @echo "" + @echo "Network Targets (requires config files in ~/.debros):" @echo " run-node - Start bootstrap node" - @echo " run-node2 - Start second node (requires JOINADDR, optional HTTP/RAFT/P2P)" - @echo " run-node3 - Start third node (requires JOINADDR, optional HTTP/RAFT/P2P)" - @echo " run-gateway - Start HTTP gateway (flags via env: GATEWAY_ADDR, GATEWAY_NAMESPACE, GATEWAY_BOOTSTRAP_PEERS, GATEWAY_REQUIRE_AUTH, GATEWAY_API_KEYS)" + @echo " run-node2 - Start second node" + @echo " run-node3 - Start third node" + @echo " run-gateway - Start HTTP gateway" @echo " run-example - Run usage example" + @echo "" + @echo "Running Multiple Nodes:" + @echo " Nodes use --config flag to select which YAML file in ~/.debros to load:" + @echo " go run ./cmd/node --config bootstrap.yaml" + @echo " go run ./cmd/node --config node.yaml" + @echo " go run ./cmd/node --config node2.yaml" + @echo " Generate configs with: ./bin/network-cli config init --name " + @echo "" + @echo "CLI Commands:" @echo " run-cli - Run network CLI help" - @echo " show-bootstrap - Show example bootstrap usage with flags" @echo " cli-health - Check network health" @echo " cli-peers - List network peers" @echo " cli-status - Get network status" @echo " cli-storage-test - Test storage operations" @echo " cli-pubsub-test - Test pub/sub operations" + @echo "" + @echo "Development:" @echo " test-multinode - Full multi-node test with 1 bootstrap + 2 nodes" @echo " test-peer-discovery - Test peer discovery (requires running nodes)" @echo " test-replication - Test data replication (requires running nodes)" @echo " test-consensus - Test database consensus (requires running nodes)" + @echo "" + @echo "Maintenance:" @echo " deps - Download dependencies" @echo " tidy - Tidy dependencies" @echo " fmt - Format code" diff --git a/README.md b/README.md index c30e21e..af7c92f 100644 --- a/README.md +++ b/README.md @@ -88,6 +88,75 @@ A robust, decentralized peer-to-peer network built in Go, providing distributed - **5001:** RQLite HTTP API - **7001:** RQLite Raft consensus +### Filesystem Permissions + +DeBros Network stores all configuration and data in `~/.debros/` directory. Ensure you have: + +- **Read/Write access** to your home directory (`~`) +- **Available disk space**: At least 10GB for database and logs +- **No restrictive mount options**: The home directory must not be mounted read-only +- **Unix permissions**: Standard user permissions are sufficient (no root/sudo required) + +#### Directory Structure + +DeBros automatically creates the following directory structure: + +``` +~/.debros/ +ā”œā”€ā”€ bootstrap.yaml # Bootstrap node config +ā”œā”€ā”€ node.yaml # Node config +ā”œā”€ā”€ gateway.yaml # Gateway config +ā”œā”€ā”€ bootstrap/ # Bootstrap node data (auto-created) +ā”‚ ā”œā”€ā”€ rqlite/ # RQLite database files +ā”‚ ā”‚ ā”œā”€ā”€ db.sqlite # Main database +ā”‚ ā”‚ ā”œā”€ā”€ raft/ # Raft consensus data +ā”‚ ā”‚ ā””ā”€ā”€ rsnapshots/ # Raft snapshots +ā”‚ ā”œā”€ā”€ peer.info # Node multiaddr (created at startup) +ā”‚ ā””ā”€ā”€ identity.key # Node private key (created at startup) +ā”œā”€ā”€ node/ # Node data (auto-created) +ā”‚ ā”œā”€ā”€ rqlite/ # RQLite database files +ā”‚ ā”œā”€ā”€ raft/ # Raft data +ā”‚ ā”œā”€ā”€ peer.info # Node multiaddr (created at startup) +ā”‚ ā””ā”€ā”€ identity.key # Node private key (created at startup) +ā””ā”€ā”€ node2/ # Additional node configs (if running multiple) + ā””ā”€ā”€ rqlite/ # RQLite database files +``` + +**Files Created at Startup:** +- `identity.key` - LibP2P private key for the node (generated once, reused) +- `peer.info` - The node's multiaddr (e.g., `/ip4/0.0.0.0/tcp/4001/p2p/12D3KooW...`) + +**Automatic Creation**: The node automatically creates all necessary data directories when started. You only need to ensure: +1. `~/.debros/` is writable +2. Sufficient disk space available +3. Correct config files exist + +**Permission Check:** + +```bash +# Verify home directory is writable +touch ~/test-write && rm ~/test-write && echo "āœ“ Home directory is writable" + +# Check available disk space +df -h ~ +``` + +**If you get permission errors:** + +``` +Error: Failed to create/access config directory +Please ensure: + 1. Home directory is accessible + 2. You have write permissions to home directory + 3. Disk space is available +``` + +**Solution:** + +- Ensure you're not running with overly restrictive umask: `umask` should show `0022` or similar +- Check home directory permissions: `ls -ld ~` should show your user as owner +- For sandboxed/containerized environments: Ensure `/home/` is writable + --- ## Quick Start @@ -110,7 +179,7 @@ make build ```bash make run-node # Or manually: -go run ./cmd/node --config configs/bootstrap.yaml +go run ./cmd/node --config configs/node.yaml ``` ### 4. Start Additional Nodes @@ -178,212 +247,141 @@ sudo journalctl -u debros-node.service -f ## Configuration -### Example Configuration Files +### Configuration Files Location -#### `configs/bootstrap.yaml` +All configuration files are stored in `~/.debros/` for both local development and production deployments: -```yaml -node: - id: "" - type: "bootstrap" - listen_addresses: - - "/ip4/0.0.0.0/tcp/4001" - data_dir: "./data/bootstrap" - max_connections: 100 +- `~/.debros/node.yaml` - Node configuration +- `~/.debros/node.yaml` - Bootstrap node configuration +- `~/.debros/gateway.yaml` - Gateway configuration -database: - data_dir: "./data/bootstrap/rqlite" - replication_factor: 3 - shard_count: 16 - max_database_size: 1073741824 - backup_interval: 24h - rqlite_port: 5001 - rqlite_raft_port: 7001 - rqlite_join_address: "" +The system will **only** load config from `~/.debros/` and will error if required config files are missing. -discovery: - bootstrap_peers: [] - discovery_interval: 15s - bootstrap_port: 4001 - http_adv_address: "127.0.0.1" - raft_adv_address: "" +### Generating Configuration Files -security: - enable_tls: false - private_key_file: "" - certificate_file: "" +Use the `network-cli config init` command to generate configuration files: -logging: - level: "info" - format: "console" - output_file: "" +#### Generate a Node Config + +```bash +# Generate basic node config with bootstrap peers +network-cli config init --type node --bootstrap-peers "/ip4/127.0.0.1/tcp/4001/p2p/QmXxx,/ip4/127.0.0.1/tcp/4002/p2p/QmYyy" + +# With custom ports +network-cli config init --type node --name node2.yaml --listen-port 4002 --rqlite-http-port 5002 --rqlite-raft-port 7002 --join localhost:5001 --bootstrap-peers "/ip4/127.0.0.1/tcp/4001/p2p/QmXxx" + +# Force overwrite existing config +network-cli config init --type node --force ``` -#### `configs/node.yaml` +#### Generate a Bootstrap Node Config -```yaml -node: - id: "node2" - type: "node" - listen_addresses: - - "/ip4/0.0.0.0/tcp/4002" - data_dir: "./data/node2" - max_connections: 50 +```bash +# Generate bootstrap node (no join address required) +network-cli config init --type bootstrap -database: - data_dir: "./data/node2/rqlite" - replication_factor: 3 - shard_count: 16 - max_database_size: 1073741824 - backup_interval: 24h - rqlite_port: 5002 - rqlite_raft_port: 7002 - rqlite_join_address: "127.0.0.1:7001" - -discovery: - bootstrap_peers: - - "/ip4/127.0.0.1/tcp/4001/p2p/" - discovery_interval: 15s - bootstrap_port: 4002 - http_adv_address: "127.0.0.1" - raft_adv_address: "" - -security: - enable_tls: false - private_key_file: "" - certificate_file: "" - -logging: - level: "info" - format: "console" - output_file: "" +# With custom ports +network-cli config init --type bootstrap --listen-port 4001 --rqlite-http-port 5001 --rqlite-raft-port 7001 ``` -### YAML Reference +#### Generate a Gateway Config -#### Node YAML (configs/node.yaml or configs/bootstrap.yaml) +```bash +# Generate gateway config +network-cli config init --type gateway -The .yaml files are required in order for the nodes and the gateway to run correctly. - -node: - -- id (string) Optional node ID. Auto-generated if empty. -- type (string) "bootstrap" or "node". Default: "node". -- listen_addresses (string[]) LibP2P listen multiaddrs. Default: ["/ip4/0.0.0.0/tcp/4001"]. -- data_dir (string) Data directory. Default: "./data". -- max_connections (int) Max peer connections. Default: 50. - -database: - -- data_dir (string) Directory for database files. Default: "./data/db". -- replication_factor (int) Number of replicas. Default: 3. -- shard_count (int) Shards for data distribution. Default: 16. -- max_database_size (int64 bytes) Max DB size. Default: 1073741824 (1GB). -- backup_interval (duration) e.g., "24h". Default: 24h. -- rqlite_port (int) RQLite HTTP API port. Default: 5001. -- rqlite_raft_port (int) RQLite Raft port. Default: 7001. -- rqlite_join_address (string) Raft address of an existing RQLite node to join (host:port format). Empty for bootstrap. - -discovery: - -- bootstrap_peers (string[]) List of LibP2P multiaddrs of bootstrap peers. -- discovery_interval (duration) How often to announce/discover peers. Default: 15s. -- bootstrap_port (int) Default port for bootstrap nodes. Default: 4001. -- http_adv_address (string) Advertised HTTP address for RQLite (host:port). -- raft_adv_address (string) Advertised Raft address (host:port). -- node_namespace (string) Namespace for node identifiers. Default: "default". - -security: - -- enable_tls (bool) Enable TLS for externally exposed services. Default: false. -- private_key_file (string) Path to TLS private key (if TLS enabled). -- certificate_file (string) Path to TLS certificate (if TLS enabled). - -logging: - -- level (string) one of "debug", "info", "warn", "error". Default: "info". -- format (string) "json" or "console". Default: "console". -- output_file (string) Empty for stdout; otherwise path to log file. - -Precedence (node): Flags > YAML > Defaults. - -Example node.yaml - -```yaml -node: - id: "node2" - type: "node" - listen_addresses: - - "/ip4/0.0.0.0/tcp/4002" - data_dir: "./data/node2" - max_connections: 50 - -database: - data_dir: "./data/node2/rqlite" - replication_factor: 3 - shard_count: 16 - max_database_size: 1073741824 - backup_interval: 24h - rqlite_port: 5002 - rqlite_raft_port: 7002 - rqlite_join_address: "127.0.0.1:7001" - -discovery: - bootstrap_peers: - - "/ip4/127.0.0.1/tcp/4001/p2p/" - discovery_interval: 15s - bootstrap_port: 4001 - http_adv_address: "127.0.0.1" - raft_adv_address: "" - node_namespace: "default" - -security: - enable_tls: false - private_key_file: "" - certificate_file: "" - -logging: - level: "info" - format: "console" - output_file: "" +# With bootstrap peers +network-cli config init --type gateway --bootstrap-peers "/ip4/127.0.0.1/tcp/4001/p2p/QmXxx" ``` -#### Gateway YAML (configs/gateway.yaml) +### Running Multiple Nodes on the Same Machine -- listen_addr (string) HTTP listen address, e.g., ":6001". Default: ":6001". -- client_namespace (string) Namespace used by the gateway client. Default: "default". -- bootstrap_peers (string[]) List of bootstrap peer multiaddrs. Default: empty. +You can run multiple nodes on a single machine by creating separate configuration files and using the `--config` flag: -Precedence (gateway): Flags > Environment Variables > YAML > Defaults. -Environment variables: +#### Create Multiple Node Configs -- GATEWAY_ADDR -- GATEWAY_NAMESPACE -- GATEWAY_BOOTSTRAP_PEERS (comma-separated) +```bash +# Node 1 +./bin/network-cli config init --type node --name node1.yaml \ + --listen-port 4001 --rqlite-http-port 5001 --rqlite-raft-port 7001 \ + --bootstrap-peers "/ip4/127.0.0.1/tcp/4001/p2p/" -Example gateway.yaml +# Node 2 +./bin/network-cli config init --type node --name node2.yaml \ + --listen-port 4002 --rqlite-http-port 5002 --rqlite-raft-port 7002 \ + --join localhost:5001 \ + --bootstrap-peers "/ip4/127.0.0.1/tcp/4001/p2p/" -```yaml -listen_addr: ":6001" -client_namespace: "default" -bootstrap_peers: - - "" +# Node 3 +./bin/network-cli config init --type node --name node3.yaml \ + --listen-port 4003 --rqlite-http-port 5003 --rqlite-raft-port 7003 \ + --join localhost:5001 \ + --bootstrap-peers "/ip4/127.0.0.1/tcp/4001/p2p/" ``` -### Flags & Environment Variables +#### Run Multiple Nodes in Separate Terminals -- **Flags**: Override config at startup (`--data`, `--p2p-port`, `--rqlite-http-port`, etc.) -- **Env Vars**: Override config and flags (`NODE_ID`, `RQLITE_PORT`, `BOOTSTRAP_PEERS`, etc.) -- **Precedence (gateway)**: Flags > Env Vars > YAML > Defaults -- **Precedence (node)**: Flags > YAML > Defaults +```bash +# Terminal 1 - Bootstrap node +go run ./cmd/node --config bootstrap.yaml -### Bootstrap & Database Endpoints +# Terminal 2 - Node 1 +go run ./cmd/node --config node1.yaml -- **Bootstrap peers**: Set in config or via `BOOTSTRAP_PEERS` env var. -- **Database endpoints**: Set in config or via `RQLITE_NODES` env var. -- **Development mode**: Use `NETWORK_DEV_LOCAL=1` for localhost defaults. +# Terminal 3 - Node 2 +go run ./cmd/node --config node2.yaml -### Configuration Validation +# Terminal 4 - Node 3 +go run ./cmd/node --config node3.yaml +``` + +#### Or Use Makefile Targets + +```bash +# Terminal 1 +make run-node # Runs: go run ./cmd/node --config bootstrap.yaml + +# Terminal 2 +make run-node2 # Runs: go run ./cmd/node --config node.yaml + +# Terminal 3 +make run-node3 # Runs: go run ./cmd/node --config node2.yaml +``` + +#### Key Points for Multiple Nodes + +- **Each node needs unique ports**: P2P port, RQLite HTTP port, and RQLite Raft port must all be different +- **Join address**: Non-bootstrap nodes need `rqlite_join_address` pointing to the bootstrap or an existing node +- **Bootstrap peers**: All nodes need the bootstrap node's multiaddr in `discovery.bootstrap_peers` +- **Config files**: Store all configs in `~/.debros/` with different filenames +- **--config flag**: Specify which config file to load (defaults to `node.yaml`) + +āš ļø **Common Mistake - Same Ports:** +If all nodes use the same ports (e.g., 5001, 7001), they will try to bind to the same addresses and fail to communicate. Verify each node has unique ports: + +```bash +# Bootstrap +grep "rqlite_port\|rqlite_raft_port" ~/.debros/bootstrap.yaml +# Should show: rqlite_port: 5001, rqlite_raft_port: 7001 + +# Node 2 +grep "rqlite_port\|rqlite_raft_port" ~/.debros/node.yaml +# Should show: rqlite_port: 5002, rqlite_raft_port: 7002 + +# Node 3 +grep "rqlite_port\|rqlite_raft_port" ~/.debros/node2.yaml +# Should show: rqlite_port: 5003, rqlite_raft_port: 7003 +``` + +If ports are wrong, regenerate the config with `--force`: + +```bash +./bin/network-cli config init --type node --name node.yaml \ + --listen-port 4002 --rqlite-http-port 5002 --rqlite-raft-port 7002 \ + --join localhost:5001 --bootstrap-peers '' --force +``` + +### Validating Configuration DeBros Network performs strict validation of all configuration files at startup. This ensures invalid configurations are caught immediately rather than causing silent failures later. @@ -757,4 +755,196 @@ GET /v1/pubsub/topics # List active topics - **PubSub** - WS Subscribe: `GET /v1/pubsub/ws?topic=` - Publish: `POST /v1/pubsub/publish` `{topic, data_base64}` ā†’ `{status:"ok"}` - - Topics: `GET /v1/pubsub/topics` ā†’ ` \ No newline at end of file + - Topics: `GET /v1/pubsub/topics` ā†’ `{topics:[...]}` + +--- + +## Troubleshooting + +### Configuration & Permissions + +**Error: "Failed to create/access config directory"** + +This happens when DeBros cannot access or create `~/.debros/` directory. + +**Causes:** +1. Home directory is not writable +2. Home directory doesn't exist +3. Filesystem is read-only (sandboxed/containerized environment) +4. Permission denied (running with wrong user/umask) + +**Solutions:** + +```bash +# Check home directory exists and is writable +ls -ld ~ +touch ~/test-write && rm ~/test-write + +# Check umask (should be 0022 or 0002) +umask + +# If umask is too restrictive, change it +umask 0022 + +# Check disk space +df -h ~ + +# For containerized environments, ensure /home/ is mounted with write permissions +docker run -v /home:/home --user $(id -u):$(id -g) debros-network +``` + +**Error: "Config file not found at ~/.debros/node.yaml"** + +The node requires a config file to exist before starting. + +**Solution:** + +Generate config files first: + +```bash +# Build CLI +make build + +# Generate configs +./bin/network-cli config init --type bootstrap +./bin/network-cli config init --type node --bootstrap-peers '' +./bin/network-cli config init --type gateway +``` + +### Node Startup Issues + +**Error: "node.data_dir: parent directory not writable"** + +The data directory parent is not accessible. + +**Solution:** + +Ensure `~/.debros` is writable and has at least 10GB free space: + +```bash +# Check permissions +ls -ld ~/.debros + +# Check available space +df -h ~/.debros + +# Recreate if corrupted +rm -rf ~/.debros +./bin/network-cli config init --type bootstrap +``` + +**Error: "failed to create data directory"** + +The node cannot create its data directory in `~/.debros`. + +**Causes:** +1. `~/.debros` is not writable +2. Parent directory path in config uses `~` which isn't expanded properly +3. Disk is full + +**Solutions:** + +```bash +# Check ~/.debros exists and is writable +mkdir -p ~/.debros +ls -ld ~/.debros + +# Verify data_dir in config uses ~ (e.g., ~/.debros/node) +cat ~/.debros/node.yaml | grep data_dir + +# Check disk space +df -h ~ + +# Ensure user owns ~/.debros +chown -R $(whoami) ~/.debros + +# Retry node startup +make run-node +``` + +**Error: "stat ~/.debros: no such file or directory"** + +**Port Already in Use** + +If you get "address already in use" errors: + +```bash +# Find processes using ports +lsof -i :4001 # P2P port +lsof -i :5001 # RQLite HTTP +lsof -i :7001 # RQLite Raft + +# Kill if needed +kill -9 + +# Or use different ports in config +./bin/network-cli config init --type node --listen-port 4002 --rqlite-http-port 5002 --rqlite-raft-port 7002 +``` + +### Common Configuration Errors + +**Error: "discovery.bootstrap_peers: required for node type"** + +Nodes (non-bootstrap) must specify bootstrap peers to discover the network. + +**Solution:** + +Generate node config with bootstrap peers: + +```bash +./bin/network-cli config init --type node --bootstrap-peers '/ip4/127.0.0.1/tcp/4001/p2p/12D3KooW...' +``` + +**Error: "database.rqlite_join_address: required for node type"** + +Non-bootstrap nodes must specify which node to join in the Raft cluster. + +**Solution:** + +Generate config with join address: + +```bash +./bin/network-cli config init --type node --join localhost:5001 +``` + +**Error: "database.rqlite_raft_port: must differ from database.rqlite_port"** + +HTTP and Raft ports cannot be the same. + +**Solution:** + +Use different ports (RQLite HTTP and Raft must be on different ports): + +```bash +./bin/network-cli config init --type node \ + --rqlite-http-port 5001 \ + --rqlite-raft-port 7001 +``` + +### Peer Discovery Issues + +If nodes can't find each other: + +1. **Verify bootstrap node is running:** + ```bash + ./bin/network-cli health + ./bin/network-cli peers + ``` + +2. **Check bootstrap peer multiaddr is correct:** + ```bash + cat ~/.debros/bootstrap/peer.info # On bootstrap node + # Should match value in other nodes' discovery.bootstrap_peers + ``` + +3. **Ensure all nodes have same bootstrap peers in config** + +4. **Check firewall/network:** + ```bash + # Verify P2P port is open + nc -zv 127.0.0.1 4001 + ``` + +--- + +## License \ No newline at end of file diff --git a/cmd/cli/main.go b/cmd/cli/main.go index f9acc47..0399ec0 100644 --- a/cmd/cli/main.go +++ b/cmd/cli/main.go @@ -7,12 +7,14 @@ import ( "log" "os" "os/exec" + "path/filepath" "strconv" "strings" "time" "github.com/DeBrosOfficial/network/pkg/auth" "github.com/DeBrosOfficial/network/pkg/client" + "github.com/DeBrosOfficial/network/pkg/config" "github.com/libp2p/go-libp2p/core/crypto" "github.com/libp2p/go-libp2p/core/peer" ) @@ -78,6 +80,8 @@ func main() { handlePeerID() case "auth": handleAuth(args) + case "config": + handleConfig(args) case "help", "--help", "-h": showHelp() @@ -576,6 +580,333 @@ func isPrintableText(s string) bool { return len(s) > 0 && float64(printableCount)/float64(len(s)) > 0.8 } +func handleConfig(args []string) { + if len(args) == 0 { + showConfigHelp() + return + } + + subcommand := args[0] + subargs := args[1:] + + switch subcommand { + case "init": + handleConfigInit(subargs) + case "validate": + handleConfigValidate(subargs) + case "help": + showConfigHelp() + default: + fmt.Fprintf(os.Stderr, "Unknown config subcommand: %s\n", subcommand) + showConfigHelp() + os.Exit(1) + } +} + +func showConfigHelp() { + fmt.Printf("Config Management Commands\n\n") + fmt.Printf("Usage: network-cli config [options]\n\n") + fmt.Printf("Subcommands:\n") + fmt.Printf(" init - Generate configuration files in ~/.debros\n") + fmt.Printf(" validate --name - Validate a config file\n\n") + fmt.Printf("Init Options:\n") + fmt.Printf(" --type - Config type: node, bootstrap, gateway (default: node)\n") + fmt.Printf(" --name - Output filename (default: node.yaml)\n") + fmt.Printf(" --id - Node ID for bootstrap peers\n") + fmt.Printf(" --listen-port - LibP2P listen port (default: 4001)\n") + fmt.Printf(" --rqlite-http-port - RQLite HTTP port (default: 5001)\n") + fmt.Printf(" --rqlite-raft-port - RQLite Raft port (default: 7001)\n") + fmt.Printf(" --join - RQLite address to join (required for non-bootstrap)\n") + fmt.Printf(" --bootstrap-peers - Comma-separated bootstrap peer multiaddrs\n") + fmt.Printf(" --force - Overwrite existing config\n\n") + fmt.Printf("Examples:\n") + fmt.Printf(" network-cli config init\n") + fmt.Printf(" network-cli config init --type node --bootstrap-peers /ip4/127.0.0.1/tcp/4001/p2p/QmXxx,/ip4/127.0.0.1/tcp/4002/p2p/QmYyy\n") + fmt.Printf(" network-cli config init --type bootstrap\n") + fmt.Printf(" network-cli config init --type gateway\n") + fmt.Printf(" network-cli config validate --name node.yaml\n") +} + +func handleConfigInit(args []string) { + // Parse flags + var ( + cfgType = "node" + name = "node.yaml" + id string + listenPort = 4001 + rqliteHTTPPort = 5001 + rqliteRaftPort = 7001 + joinAddr string + bootstrapPeers string + force bool + ) + + for i := 0; i < len(args); i++ { + switch args[i] { + case "--type": + if i+1 < len(args) { + cfgType = args[i+1] + i++ + } + case "--name": + if i+1 < len(args) { + name = args[i+1] + i++ + } + case "--id": + if i+1 < len(args) { + id = args[i+1] + i++ + } + case "--listen-port": + if i+1 < len(args) { + if p, err := strconv.Atoi(args[i+1]); err == nil { + listenPort = p + } + i++ + } + case "--rqlite-http-port": + if i+1 < len(args) { + if p, err := strconv.Atoi(args[i+1]); err == nil { + rqliteHTTPPort = p + } + i++ + } + case "--rqlite-raft-port": + if i+1 < len(args) { + if p, err := strconv.Atoi(args[i+1]); err == nil { + rqliteRaftPort = p + } + i++ + } + case "--join": + if i+1 < len(args) { + joinAddr = args[i+1] + i++ + } + case "--bootstrap-peers": + if i+1 < len(args) { + bootstrapPeers = args[i+1] + i++ + } + case "--force": + force = true + } + } + + // Validate type + if cfgType != "node" && cfgType != "bootstrap" && cfgType != "gateway" { + fmt.Fprintf(os.Stderr, "Invalid --type: %s (expected: node, bootstrap, or gateway)\n", cfgType) + os.Exit(1) + } + + // Ensure config directory exists + configDir, err := config.EnsureConfigDir() + if err != nil { + fmt.Fprintf(os.Stderr, "Failed to ensure config directory: %v\n", err) + os.Exit(1) + } + + configPath := filepath.Join(configDir, name) + + // Check if file exists + if !force { + if _, err := os.Stat(configPath); err == nil { + fmt.Fprintf(os.Stderr, "Config file already exists at %s (use --force to overwrite)\n", configPath) + os.Exit(1) + } + } + + // Generate config based on type + var configContent string + switch cfgType { + case "node": + configContent = generateNodeConfig(name, id, listenPort, rqliteHTTPPort, rqliteRaftPort, joinAddr, bootstrapPeers) + case "bootstrap": + configContent = generateBootstrapConfig(name, id, listenPort, rqliteHTTPPort, rqliteRaftPort) + case "gateway": + configContent = generateGatewayConfig() + } + + // Write config file + if err := os.WriteFile(configPath, []byte(configContent), 0644); err != nil { + fmt.Fprintf(os.Stderr, "Failed to write config file: %v\n", err) + os.Exit(1) + } + + fmt.Printf("āœ… Configuration file created: %s\n", configPath) + fmt.Printf(" Type: %s\n", cfgType) + fmt.Printf("\nYou can now start the %s using the generated config.\n", cfgType) +} + +func handleConfigValidate(args []string) { + var name string + for i := 0; i < len(args); i++ { + if args[i] == "--name" && i+1 < len(args) { + name = args[i+1] + i++ + } + } + + if name == "" { + fmt.Fprintf(os.Stderr, "Missing --name flag\n") + showConfigHelp() + os.Exit(1) + } + + configDir, err := config.ConfigDir() + if err != nil { + fmt.Fprintf(os.Stderr, "Failed to get config directory: %v\n", err) + os.Exit(1) + } + + configPath := filepath.Join(configDir, name) + file, err := os.Open(configPath) + if err != nil { + fmt.Fprintf(os.Stderr, "Failed to open config file: %v\n", err) + os.Exit(1) + } + defer file.Close() + + var cfg config.Config + if err := config.DecodeStrict(file, &cfg); err != nil { + fmt.Fprintf(os.Stderr, "Failed to parse config: %v\n", err) + os.Exit(1) + } + + // Run validation + errs := cfg.Validate() + if len(errs) > 0 { + fmt.Fprintf(os.Stderr, "\nāŒ Configuration errors (%d):\n", len(errs)) + for _, err := range errs { + fmt.Fprintf(os.Stderr, " - %s\n", err) + } + os.Exit(1) + } + + fmt.Printf("āœ… Config is valid: %s\n", configPath) +} + +func generateNodeConfig(name, id string, listenPort, rqliteHTTPPort, rqliteRaftPort int, joinAddr, bootstrapPeers string) string { + nodeID := id + if nodeID == "" { + nodeID = fmt.Sprintf("node-%d", time.Now().Unix()) + } + + // Parse bootstrap peers + var peers []string + if bootstrapPeers != "" { + for _, p := range strings.Split(bootstrapPeers, ",") { + if p = strings.TrimSpace(p); p != "" { + peers = append(peers, p) + } + } + } + + // Construct data_dir from name stem (remove .yaml) + dataDir := strings.TrimSuffix(name, ".yaml") + dataDir = filepath.Join(os.ExpandEnv("~"), ".debros", dataDir) + + var peersYAML strings.Builder + if len(peers) == 0 { + peersYAML.WriteString(" bootstrap_peers: []") + } else { + peersYAML.WriteString(" bootstrap_peers:\n") + for _, p := range peers { + fmt.Fprintf(&peersYAML, " - \"%s\"\n", p) + } + } + + if joinAddr == "" { + joinAddr = "localhost:5001" + } + + return fmt.Sprintf(`node: + id: "%s" + type: "node" + listen_addresses: + - "/ip4/0.0.0.0/tcp/%d" + data_dir: "%s" + max_connections: 50 + +database: + data_dir: "%s/rqlite" + replication_factor: 3 + shard_count: 16 + max_database_size: 1073741824 + backup_interval: "24h" + rqlite_port: %d + rqlite_raft_port: %d + rqlite_join_address: "%s" + +discovery: +%s + discovery_interval: "15s" + bootstrap_port: %d + http_adv_address: "127.0.0.1:%d" + raft_adv_address: "127.0.0.1:%d" + node_namespace: "default" + +security: + enable_tls: false + +logging: + level: "info" + format: "console" +`, nodeID, listenPort, dataDir, dataDir, rqliteHTTPPort, rqliteRaftPort, joinAddr, peersYAML.String(), 4001, rqliteHTTPPort, rqliteRaftPort) +} + +func generateBootstrapConfig(name, id string, listenPort, rqliteHTTPPort, rqliteRaftPort int) string { + nodeID := id + if nodeID == "" { + nodeID = "bootstrap" + } + + dataDir := filepath.Join(os.ExpandEnv("~"), ".debros", "bootstrap") + + return fmt.Sprintf(`node: + id: "%s" + type: "bootstrap" + listen_addresses: + - "/ip4/0.0.0.0/tcp/%d" + data_dir: "%s" + max_connections: 50 + +database: + data_dir: "%s/rqlite" + replication_factor: 3 + shard_count: 16 + max_database_size: 1073741824 + backup_interval: "24h" + rqlite_port: %d + rqlite_raft_port: %d + rqlite_join_address: "" + +discovery: + bootstrap_peers: [] + discovery_interval: "15s" + bootstrap_port: %d + http_adv_address: "127.0.0.1:%d" + raft_adv_address: "127.0.0.1:%d" + node_namespace: "default" + +security: + enable_tls: false + +logging: + level: "info" + format: "console" +`, nodeID, listenPort, dataDir, dataDir, rqliteHTTPPort, rqliteRaftPort, 4001, rqliteHTTPPort, rqliteRaftPort) +} + +func generateGatewayConfig() string { + return `listen_addr: ":6001" +client_namespace: "default" +rqlite_dsn: "" +bootstrap_peers: [] +` +} + func showHelp() { fmt.Printf("Network CLI - Distributed P2P Network Management Tool\n\n") fmt.Printf("Usage: network-cli [args...]\n\n") @@ -591,6 +922,7 @@ func showHelp() { fmt.Printf(" pubsub subscribe [duration] šŸ” Subscribe to topic\n") fmt.Printf(" pubsub topics šŸ” List topics\n") fmt.Printf(" connect - Connect to peer\n") + fmt.Printf(" config - Show current configuration\n") fmt.Printf(" help - Show this help\n\n") fmt.Printf("Global Flags:\n") diff --git a/cmd/gateway/config.go b/cmd/gateway/config.go index a32619e..76548da 100644 --- a/cmd/gateway/config.go +++ b/cmd/gateway/config.go @@ -1,7 +1,6 @@ package main import ( - "flag" "fmt" "os" "strings" @@ -38,10 +37,43 @@ func getEnvBoolDefault(key string, def bool) bool { } } -// parseGatewayConfig loads optional configs/gateway.yaml then applies env and flags. -// Priority: flags > env > yaml > defaults. +// parseGatewayConfig loads gateway.yaml from ~/.debros exclusively. func parseGatewayConfig(logger *logging.ColoredLogger) *gateway.Config { - // Base defaults + // Determine config path + configPath, err := config.DefaultPath("gateway.yaml") + if err != nil { + logger.ComponentError(logging.ComponentGeneral, "Failed to determine config path", zap.Error(err)) + fmt.Fprintf(os.Stderr, "Configuration error: %v\n", err) + os.Exit(1) + } + + // Load YAML + type yamlCfg struct { + ListenAddr string `yaml:"listen_addr"` + ClientNamespace string `yaml:"client_namespace"` + RQLiteDSN string `yaml:"rqlite_dsn"` + BootstrapPeers []string `yaml:"bootstrap_peers"` + } + + data, err := os.ReadFile(configPath) + if err != nil { + logger.ComponentError(logging.ComponentGeneral, "Config file not found", + zap.String("path", configPath), + zap.Error(err)) + fmt.Fprintf(os.Stderr, "\nConfig file not found at %s\n", configPath) + fmt.Fprintf(os.Stderr, "Generate it using: network-cli config init --type gateway\n") + os.Exit(1) + } + + var y yamlCfg + // Use strict YAML decoding to reject unknown fields + if err := config.DecodeStrict(strings.NewReader(string(data)), &y); err != nil { + logger.ComponentError(logging.ComponentGeneral, "Failed to parse gateway config", zap.Error(err)) + fmt.Fprintf(os.Stderr, "Configuration parse error: %v\n", err) + os.Exit(1) + } + + // Build config from YAML cfg := &gateway.Config{ ListenAddr: ":6001", ClientNamespace: "default", @@ -49,93 +81,26 @@ func parseGatewayConfig(logger *logging.ColoredLogger) *gateway.Config { RQLiteDSN: "", } - // 1) YAML (optional) - { - type yamlCfg struct { - ListenAddr string `yaml:"listen_addr"` - ClientNamespace string `yaml:"client_namespace"` - RQLiteDSN string `yaml:"rqlite_dsn"` - BootstrapPeers []string `yaml:"bootstrap_peers"` - } - const path = "configs/gateway.yaml" - if data, err := os.ReadFile(path); err == nil { - var y yamlCfg - // Use strict YAML decoding to reject unknown fields - if err := config.DecodeStrict(strings.NewReader(string(data)), &y); err != nil { - logger.ComponentError(logging.ComponentGeneral, "failed to parse configs/gateway.yaml", zap.Error(err)) - fmt.Fprintf(os.Stderr, "Configuration load error: %v\n", err) - os.Exit(1) - } - if v := strings.TrimSpace(y.ListenAddr); v != "" { - cfg.ListenAddr = v - } - if v := strings.TrimSpace(y.ClientNamespace); v != "" { - cfg.ClientNamespace = v - } - if v := strings.TrimSpace(y.RQLiteDSN); v != "" { - cfg.RQLiteDSN = v - } - if len(y.BootstrapPeers) > 0 { - var bp []string - for _, p := range y.BootstrapPeers { - p = strings.TrimSpace(p) - if p != "" { - bp = append(bp, p) - } - } - if len(bp) > 0 { - cfg.BootstrapPeers = bp - } - } - } - } - - // 2) Env overrides - if v := strings.TrimSpace(os.Getenv("GATEWAY_ADDR")); v != "" { + if v := strings.TrimSpace(y.ListenAddr); v != "" { cfg.ListenAddr = v } - if v := strings.TrimSpace(os.Getenv("GATEWAY_NAMESPACE")); v != "" { + if v := strings.TrimSpace(y.ClientNamespace); v != "" { cfg.ClientNamespace = v } - if v := strings.TrimSpace(os.Getenv("GATEWAY_RQLITE_DSN")); v != "" { + if v := strings.TrimSpace(y.RQLiteDSN); v != "" { cfg.RQLiteDSN = v } - if v := strings.TrimSpace(os.Getenv("GATEWAY_BOOTSTRAP_PEERS")); v != "" { - parts := strings.Split(v, ",") + if len(y.BootstrapPeers) > 0 { var bp []string - for _, part := range parts { - s := strings.TrimSpace(part) - if s != "" { - bp = append(bp, s) + for _, p := range y.BootstrapPeers { + p = strings.TrimSpace(p) + if p != "" { + bp = append(bp, p) } } - cfg.BootstrapPeers = bp - } - - // 3) Flags (override env) - addr := flag.String("addr", "", "HTTP listen address (e.g., :6001)") - ns := flag.String("namespace", "", "Client namespace for scoping resources") - peers := flag.String("bootstrap-peers", "", "Comma-separated bootstrap peers for network client") - - // Do not call flag.Parse() elsewhere to avoid double-parsing - flag.Parse() - - if a := strings.TrimSpace(*addr); a != "" { - cfg.ListenAddr = a - } - if n := strings.TrimSpace(*ns); n != "" { - cfg.ClientNamespace = n - } - if p := strings.TrimSpace(*peers); p != "" { - parts := strings.Split(p, ",") - var bp []string - for _, part := range parts { - s := strings.TrimSpace(part) - if s != "" { - bp = append(bp, s) - } + if len(bp) > 0 { + cfg.BootstrapPeers = bp } - cfg.BootstrapPeers = bp } // Validate configuration @@ -148,7 +113,8 @@ func parseGatewayConfig(logger *logging.ColoredLogger) *gateway.Config { os.Exit(1) } - logger.ComponentInfo(logging.ComponentGeneral, "Loaded gateway configuration", + logger.ComponentInfo(logging.ComponentGeneral, "Loaded gateway configuration from YAML", + zap.String("path", configPath), zap.String("addr", cfg.ListenAddr), zap.String("namespace", cfg.ClientNamespace), zap.Int("bootstrap_peer_count", len(cfg.BootstrapPeers)), diff --git a/cmd/node/main.go b/cmd/node/main.go index 70646ef..5d469b1 100644 --- a/cmd/node/main.go +++ b/cmd/node/main.go @@ -7,6 +7,8 @@ import ( "os" "os/signal" "path/filepath" + "strconv" + "strings" "syscall" "github.com/DeBrosOfficial/network/pkg/config" @@ -29,15 +31,8 @@ func setup_logger(component logging.Component) (logger *logging.ColoredLogger) { } // parse_flags parses command-line flags and returns them. -func parse_flags() (configPath, dataDir, nodeID *string, p2pPort, rqlHTTP, rqlRaft *int, rqlJoinAddr, advAddr *string, help *bool) { - configPath = flag.String("config", "", "Path to config YAML file (overrides defaults)") - dataDir = flag.String("data", "", "Data directory (auto-detected if not provided)") - nodeID = flag.String("id", "", "Node identifier (for running multiple local nodes)") - p2pPort = flag.Int("p2p-port", 4001, "LibP2P listen port") - rqlHTTP = flag.Int("rqlite-http-port", 5001, "RQLite HTTP API port") - rqlRaft = flag.Int("rqlite-raft-port", 7001, "RQLite Raft port") - rqlJoinAddr = flag.String("rqlite-join-address", "", "RQLite address to join (e.g., /ip4/)") - advAddr = flag.String("adv-addr", "127.0.0.1", "Default Advertise address for rqlite and rafts") +func parse_flags() (configName *string, help *bool) { + configName = flag.String("config", "node.yaml", "Config filename in ~/.debros (default: node.yaml)") help = flag.Bool("help", false, "Show help") flag.Parse() @@ -67,18 +62,39 @@ func check_if_should_open_help(help *bool) { } } -// select_data_dir selects the data directory for the node -// If none of (hasConfigFile, nodeID, dataDir) are present, throw an error and do not start -func select_data_dir(dataDir *string, nodeID *string, hasConfigFile bool) { +// select_data_dir validates that we can load the config from ~/.debros +func select_data_dir_check(configName *string) { logger := setup_logger(logging.ComponentNode) - if !hasConfigFile && (*nodeID == "" || nodeID == nil) && (*dataDir == "" || dataDir == nil) { - logger.Error("No config file, node ID, or data directory specified. Please provide at least one. Refusing to start.") + // Ensure config directory exists and is writable + _, err := config.EnsureConfigDir() + if err != nil { + logger.Error("Failed to ensure config directory", zap.Error(err)) + fmt.Fprintf(os.Stderr, "\nāŒ Configuration Error:\n") + fmt.Fprintf(os.Stderr, "Failed to create/access config directory: %v\n", err) + fmt.Fprintf(os.Stderr, "\nPlease ensure:\n") + fmt.Fprintf(os.Stderr, " 1. Home directory is accessible: %s\n", os.ExpandEnv("~")) + fmt.Fprintf(os.Stderr, " 2. You have write permissions to home directory\n") + fmt.Fprintf(os.Stderr, " 3. Disk space is available\n") os.Exit(1) } - if *dataDir != "" { - logger.Info("Data directory selected: %s", zap.String("dataDir", *dataDir)) + configPath, err := config.DefaultPath(*configName) + if err != nil { + logger.Error("Failed to determine config path", zap.Error(err)) + os.Exit(1) + } + + if _, err := os.Stat(configPath); err != nil { + logger.Error("Config file not found", + zap.String("path", configPath), + zap.Error(err)) + fmt.Fprintf(os.Stderr, "\nāŒ Configuration Error:\n") + fmt.Fprintf(os.Stderr, "Config file not found at %s\n", configPath) + fmt.Fprintf(os.Stderr, "\nGenerate it with one of:\n") + fmt.Fprintf(os.Stderr, " network-cli config init --type bootstrap\n") + fmt.Fprintf(os.Stderr, " network-cli config init --type node --bootstrap-peers ''\n") + os.Exit(1) } } @@ -97,9 +113,21 @@ func startNode(ctx context.Context, cfg *config.Config, port int) error { return err } + // Expand data directory path for peer.info file + dataDir := os.ExpandEnv(cfg.Node.DataDir) + if strings.HasPrefix(dataDir, "~") { + home, err := os.UserHomeDir() + if err != nil { + logger.Error("failed to determine home directory: %v", zap.Error(err)) + dataDir = cfg.Node.DataDir + } else { + dataDir = filepath.Join(home, dataDir[1:]) + } + } + // Save the peer ID to a file for CLI access (especially useful for bootstrap) peerID := n.GetPeerID() - peerInfoFile := filepath.Join(cfg.Node.DataDir, "peer.info") + peerInfoFile := filepath.Join(dataDir, "peer.info") peerMultiaddr := fmt.Sprintf("/ip4/0.0.0.0/tcp/%d/p2p/%s", port, peerID) if err := os.WriteFile(peerInfoFile, []byte(peerMultiaddr), 0644); err != nil { @@ -168,54 +196,107 @@ func printValidationErrors(errs []error) { os.Exit(1) } +// ensureDataDirectories ensures that all necessary data directories exist and have correct permissions. +func ensureDataDirectories(cfg *config.Config, logger *logging.ColoredLogger) error { + // Expand ~ in data_dir path + dataDir := os.ExpandEnv(cfg.Node.DataDir) + if strings.HasPrefix(dataDir, "~") { + home, err := os.UserHomeDir() + if err != nil { + return fmt.Errorf("failed to determine home directory: %w", err) + } + dataDir = filepath.Join(home, dataDir[1:]) + } + + // Ensure Node.DataDir exists and is writable + if err := os.MkdirAll(dataDir, 0755); err != nil { + return fmt.Errorf("failed to create data directory %s: %w", dataDir, err) + } + logger.ComponentInfo(logging.ComponentNode, "Data directory created/verified", zap.String("path", dataDir)) + + // Ensure RQLite data directory exists + rqliteDir := filepath.Join(dataDir, "rqlite") + if err := os.MkdirAll(rqliteDir, 0755); err != nil { + return fmt.Errorf("failed to create rqlite data directory: %w", err) + } + logger.ComponentInfo(logging.ComponentNode, "RQLite data directory created/verified", zap.String("path", rqliteDir)) + + return nil +} + func main() { logger := setup_logger(logging.ComponentNode) // Parse command-line flags - configPath, dataDir, nodeID, p2pPort, rqlHTTP, rqlRaft, rqlJoinAddr, advAddr, help := parse_flags() + configName, help := parse_flags() check_if_should_open_help(help) - select_data_dir(dataDir, nodeID, *configPath != "") - // Load configuration - var cfg *config.Config - if *configPath != "" { - // Load from YAML with strict decoding - var err error - cfg, err = LoadConfigFromYAML(*configPath) - if err != nil { - logger.Error("Failed to load config from YAML", zap.Error(err)) - fmt.Fprintf(os.Stderr, "Configuration load error: %v\n", err) - os.Exit(1) - } - logger.ComponentInfo(logging.ComponentNode, "Configuration loaded from YAML file", zap.String("path", *configPath)) - } else { - // Use default configuration - cfg = config.DefaultConfig() - logger.ComponentInfo(logging.ComponentNode, "Default configuration loaded successfully") + // Check if config file exists + select_data_dir_check(configName) + + // Load configuration from ~/.debros/node.yaml + configPath, err := config.DefaultPath(*configName) + if err != nil { + logger.Error("Failed to determine config path", zap.Error(err)) + fmt.Fprintf(os.Stderr, "Configuration error: %v\n", err) + os.Exit(1) } - // Apply command-line flag overrides - apply_flag_overrides(cfg, p2pPort, rqlHTTP, rqlRaft, rqlJoinAddr, advAddr, dataDir) - logger.ComponentInfo(logging.ComponentNode, "Command line arguments applied to configuration") + var cfg *config.Config + var cfgErr error + cfg, cfgErr = LoadConfigFromYAML(configPath) + if cfgErr != nil { + logger.Error("Failed to load config from YAML", zap.Error(cfgErr)) + fmt.Fprintf(os.Stderr, "Configuration load error: %v\n", cfgErr) + os.Exit(1) + } + logger.ComponentInfo(logging.ComponentNode, "Configuration loaded from YAML file", zap.String("path", configPath)) + + // Set default advertised addresses if empty + if cfg.Discovery.HttpAdvAddress == "" { + cfg.Discovery.HttpAdvAddress = fmt.Sprintf("127.0.0.1:%d", cfg.Database.RQLitePort) + } + if cfg.Discovery.RaftAdvAddress == "" { + cfg.Discovery.RaftAdvAddress = fmt.Sprintf("127.0.0.1:%d", cfg.Database.RQLiteRaftPort) + } // Validate configuration if errs := cfg.Validate(); len(errs) > 0 { printValidationErrors(errs) } - // LibP2P uses configurable port (default 4001); RQLite uses 5001 (HTTP) and 7001 (Raft) - port := *p2pPort + // Expand and create data directories + if err := ensureDataDirectories(cfg, logger); err != nil { + logger.Error("Failed to create data directories", zap.Error(err)) + fmt.Fprintf(os.Stderr, "\nāŒ Data Directory Error:\n") + fmt.Fprintf(os.Stderr, "%v\n", err) + os.Exit(1) + } logger.ComponentInfo(logging.ComponentNode, "Node configuration summary", zap.Strings("listen_addresses", cfg.Node.ListenAddresses), zap.Int("rqlite_http_port", cfg.Database.RQLitePort), zap.Int("rqlite_raft_port", cfg.Database.RQLiteRaftPort), - zap.Int("p2p_port", port), zap.Strings("bootstrap_peers", cfg.Discovery.BootstrapPeers), zap.String("rqlite_join_address", cfg.Database.RQLiteJoinAddress), zap.String("data_directory", cfg.Node.DataDir)) + // Extract P2P port from listen addresses + p2pPort := 4001 // default + if len(cfg.Node.ListenAddresses) > 0 { + // Parse port from multiaddr like "/ip4/0.0.0.0/tcp/4001" + parts := strings.Split(cfg.Node.ListenAddresses[0], "/") + for i, part := range parts { + if part == "tcp" && i+1 < len(parts) { + if port, err := strconv.Atoi(parts[i+1]); err == nil { + p2pPort = port + break + } + } + } + } + // Create context for graceful shutdown ctx, cancel := context.WithCancel(context.Background()) defer cancel() @@ -224,7 +305,7 @@ func main() { errChan := make(chan error, 1) doneChan := make(chan struct{}) go func() { - if err := startNode(ctx, cfg, port); err != nil { + if err := startNode(ctx, cfg, p2pPort); err != nil { errChan <- err } close(doneChan) diff --git a/pkg/config/config.go b/pkg/config/config.go index 85e595d..9ca73d9 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -108,13 +108,7 @@ func DefaultConfig() *Config { RQLiteJoinAddress: "", // Empty for bootstrap node }, Discovery: DiscoveryConfig{ - BootstrapPeers: []string{ - "/ip4/127.0.0.1/tcp/4001/p2p/12D3KooWHbcFcrGPXKUrHcxvd8MXEeUzRYyvY8fQcpEBxncSUwhj", - // "/ip4/217.76.54.178/tcp/4001/p2p/12D3KooWKZnirPwNT4URtNSWK45f6vLkEs4xyUZ792F8Uj1oYnm1", - // "/ip4/51.83.128.181/tcp/4001/p2p/12D3KooWBn2Zf1R8v9pEfmz7hDZ5b3oADxfejA3zJBYzKRCzgvhR", - // "/ip4/155.133.27.199/tcp/4001/p2p/12D3KooWC69SBzM5QUgrLrfLWUykE8au32X5LwT7zwv9bixrQPm1", - // "/ip4/217.76.56.2/tcp/4001/p2p/12D3KooWEiqJHvznxqJ5p2y8mUs6Ky6dfU1xTYFQbyKRCABfcZz4", - }, + BootstrapPeers: []string{}, BootstrapPort: 4001, // Default LibP2P port DiscoveryInterval: time.Second * 15, // Back to 15 seconds for testing HttpAdvAddress: "", diff --git a/pkg/config/paths.go b/pkg/config/paths.go new file mode 100644 index 0000000..4e8ecec --- /dev/null +++ b/pkg/config/paths.go @@ -0,0 +1,38 @@ +package config + +import ( + "fmt" + "os" + "path/filepath" +) + +// ConfigDir returns the path to the DeBros config directory (~/.debros). +func ConfigDir() (string, error) { + home, err := os.UserHomeDir() + if err != nil { + return "", fmt.Errorf("failed to determine home directory: %w", err) + } + return filepath.Join(home, ".debros"), nil +} + +// EnsureConfigDir creates the config directory if it does not exist. +func EnsureConfigDir() (string, error) { + dir, err := ConfigDir() + if err != nil { + return "", err + } + if err := os.MkdirAll(dir, 0700); err != nil { + return "", fmt.Errorf("failed to create config directory %s: %w", dir, err) + } + return dir, nil +} + +// DefaultPath returns the path to the config file for the given component name. +// component should be e.g., "node.yaml", "bootstrap.yaml", "gateway.yaml" +func DefaultPath(component string) (string, error) { + dir, err := ConfigDir() + if err != nil { + return "", err + } + return filepath.Join(dir, component), nil +} diff --git a/pkg/config/validate.go b/pkg/config/validate.go index 6921ab4..045d784 100644 --- a/pkg/config/validate.go +++ b/pkg/config/validate.go @@ -456,25 +456,46 @@ func validateDataDir(path string) error { return fmt.Errorf("must not be empty") } - if info, err := os.Stat(path); err == nil { + // Expand ~ to home directory + expandedPath := os.ExpandEnv(path) + if strings.HasPrefix(expandedPath, "~") { + home, err := os.UserHomeDir() + if err != nil { + return fmt.Errorf("cannot determine home directory: %v", err) + } + expandedPath = filepath.Join(home, expandedPath[1:]) + } + + if info, err := os.Stat(expandedPath); err == nil { // Directory exists; check if it's a directory and writable if !info.IsDir() { return fmt.Errorf("path exists but is not a directory") } // Try to write a test file to check permissions - testFile := filepath.Join(path, ".write_test") + testFile := filepath.Join(expandedPath, ".write_test") if err := os.WriteFile(testFile, []byte(""), 0644); err != nil { return fmt.Errorf("directory not writable: %v", err) } os.Remove(testFile) } else if os.IsNotExist(err) { // Directory doesn't exist; check if parent is writable - parent := filepath.Dir(path) + parent := filepath.Dir(expandedPath) if parent == "" || parent == "." { parent = "." } - if err := validateDirWritable(parent); err != nil { - return fmt.Errorf("parent directory not writable: %v", err) + // Allow parent not existing - it will be created at runtime + if info, err := os.Stat(parent); err != nil { + if !os.IsNotExist(err) { + return fmt.Errorf("parent directory not accessible: %v", err) + } + // Parent doesn't exist either - that's ok, will be created + } else if !info.IsDir() { + return fmt.Errorf("parent path is not a directory") + } else { + // Parent exists, check if writable + if err := validateDirWritable(parent); err != nil { + return fmt.Errorf("parent directory not writable: %v", err) + } } } else { return fmt.Errorf("cannot access path: %v", err) diff --git a/pkg/rqlite/rqlite.go b/pkg/rqlite/rqlite.go index eca678d..d673662 100644 --- a/pkg/rqlite/rqlite.go +++ b/pkg/rqlite/rqlite.go @@ -69,8 +69,18 @@ func NewRQLiteManager(cfg *config.DatabaseConfig, discoveryCfg *config.Discovery // Start starts the RQLite node func (r *RQLiteManager) Start(ctx context.Context) error { + // Expand ~ in data directory path + dataDir := os.ExpandEnv(r.dataDir) + if strings.HasPrefix(dataDir, "~") { + home, err := os.UserHomeDir() + if err != nil { + return fmt.Errorf("failed to determine home directory: %w", err) + } + dataDir = filepath.Join(home, dataDir[1:]) + } + // Create data directory - rqliteDataDir := filepath.Join(r.dataDir, "rqlite") + rqliteDataDir := filepath.Join(dataDir, "rqlite") if err := os.MkdirAll(rqliteDataDir, 0755); err != nil { return fmt.Errorf("failed to create RQLite data directory: %w", err) } @@ -100,7 +110,7 @@ func (r *RQLiteManager) Start(ctx context.Context) error { } // Wait for join target to become reachable to avoid forming a separate cluster (wait indefinitely) - if err := r.waitForJoinTarget(ctx, joinArg, 0); err != nil { + if err := r.waitForJoinTarget(ctx, r.config.RQLiteJoinAddress, 0); err != nil { r.logger.Warn("Join target did not become reachable within timeout; will still attempt to join", zap.String("join_address", r.config.RQLiteJoinAddress), zap.Error(err)) @@ -126,7 +136,7 @@ func (r *RQLiteManager) Start(ctx context.Context) error { // Start RQLite process (not bound to ctx for graceful Stop handling) r.cmd = exec.Command("rqlited", args...) - // Uncomment if you want to see the stdout/stderr of the RQLite process + // Enable debug logging of RQLite process to help diagnose issues // r.cmd.Stdout = os.Stdout // r.cmd.Stderr = os.Stderr @@ -166,7 +176,15 @@ func (r *RQLiteManager) Start(ctx context.Context) error { } } else { r.logger.Info("Waiting for RQLite SQL availability (leader discovery)") - if err := r.waitForSQLAvailable(ctx); err != nil { + // For joining nodes, wait longer for SQL availability + sqlCtx := ctx + if _, hasDeadline := ctx.Deadline(); !hasDeadline { + // If no deadline in context, create one for SQL availability check + var cancel context.CancelFunc + sqlCtx, cancel = context.WithTimeout(context.Background(), 2*time.Minute) + defer cancel() + } + if err := r.waitForSQLAvailable(sqlCtx); err != nil { if r.cmd != nil && r.cmd.Process != nil { _ = r.cmd.Process.Kill() } @@ -207,7 +225,9 @@ func (r *RQLiteManager) waitForReady(ctx context.Context) error { url := fmt.Sprintf("http://localhost:%d/status", r.config.RQLitePort) client := &http.Client{Timeout: 2 * time.Second} - for i := 0; i < 30; i++ { + // Give joining nodes more time (120 seconds vs 30) + maxAttempts := 30 + for i := 0; i < maxAttempts; i++ { select { case <-ctx.Done(): return ctx.Err() From 83c498892cabce23b323d4b52740c97cf4eedbb4 Mon Sep 17 00:00:00 2001 From: anonpenguin23 Date: Fri, 24 Oct 2025 11:05:34 +0300 Subject: [PATCH 5/6] Update CHANGELOG.md to include new entries for version 0.51.5, highlighting the addition of YAML file validation and a new authentication command in the CLI. --- CHANGELOG.md | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index b41efd9..2e350f8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,18 @@ The format is based on [Keep a Changelog][keepachangelog] and adheres to [Semant ### Added +### Changed + +### Deprecated + +### Removed + +### Fixed + +## [0.51.5] - 2025-10-24 + +### Added + - Added validation for yaml files - Added authenticaiton command on cli From b8af8e0c98ca1240611ef20074babc90f6800e84 Mon Sep 17 00:00:00 2001 From: anonpenguin23 Date: Fri, 24 Oct 2025 11:13:21 +0300 Subject: [PATCH 6/6] Refactor configuration handling in CLI and node package. Updated default config file naming based on node type and enhanced gateway config generation to include bootstrap peers. Improved data directory path handling by expanding environment variables and user home directory. --- cmd/cli/main.go | 43 +++++++++++++++++++++++++++++++++++++------ pkg/node/node.go | 24 +++++++++++++++++++++++- 2 files changed, 60 insertions(+), 7 deletions(-) diff --git a/cmd/cli/main.go b/cmd/cli/main.go index 0399ec0..eff39f4 100644 --- a/cmd/cli/main.go +++ b/cmd/cli/main.go @@ -631,7 +631,7 @@ func handleConfigInit(args []string) { // Parse flags var ( cfgType = "node" - name = "node.yaml" + name = "" // Will be set based on type if not provided id string listenPort = 4001 rqliteHTTPPort = 5001 @@ -700,6 +700,18 @@ func handleConfigInit(args []string) { os.Exit(1) } + // Set default name based on type if not provided + if name == "" { + switch cfgType { + case "bootstrap": + name = "bootstrap.yaml" + case "gateway": + name = "gateway.yaml" + default: + name = "node.yaml" + } + } + // Ensure config directory exists configDir, err := config.EnsureConfigDir() if err != nil { @@ -725,7 +737,7 @@ func handleConfigInit(args []string) { case "bootstrap": configContent = generateBootstrapConfig(name, id, listenPort, rqliteHTTPPort, rqliteRaftPort) case "gateway": - configContent = generateGatewayConfig() + configContent = generateGatewayConfig(bootstrapPeers) } // Write config file @@ -899,12 +911,31 @@ logging: `, nodeID, listenPort, dataDir, dataDir, rqliteHTTPPort, rqliteRaftPort, 4001, rqliteHTTPPort, rqliteRaftPort) } -func generateGatewayConfig() string { - return `listen_addr: ":6001" +func generateGatewayConfig(bootstrapPeers string) string { + var peers []string + if bootstrapPeers != "" { + for _, p := range strings.Split(bootstrapPeers, ",") { + if p = strings.TrimSpace(p); p != "" { + peers = append(peers, p) + } + } + } + + var peersYAML strings.Builder + if len(peers) == 0 { + peersYAML.WriteString(" bootstrap_peers: []") + } else { + peersYAML.WriteString(" bootstrap_peers:\n") + for _, p := range peers { + fmt.Fprintf(&peersYAML, " - \"%s\"\n", p) + } + } + + return fmt.Sprintf(`listen_addr: ":6001" client_namespace: "default" rqlite_dsn: "" -bootstrap_peers: [] -` +%s +`, peersYAML.String()) } func showHelp() { diff --git a/pkg/node/node.go b/pkg/node/node.go index b1e850b..1415471 100644 --- a/pkg/node/node.go +++ b/pkg/node/node.go @@ -6,6 +6,7 @@ import ( mathrand "math/rand" "os" "path/filepath" + "strings" "time" "github.com/libp2p/go-libp2p" @@ -384,6 +385,16 @@ func (n *Node) startLibP2P() error { func (n *Node) loadOrCreateIdentity() (crypto.PrivKey, error) { identityFile := filepath.Join(n.config.Node.DataDir, "identity.key") + // Expand ~ in data directory path + identityFile = os.ExpandEnv(identityFile) + if strings.HasPrefix(identityFile, "~") { + home, err := os.UserHomeDir() + if err != nil { + return nil, fmt.Errorf("failed to determine home directory: %w", err) + } + identityFile = filepath.Join(home, identityFile[1:]) + } + // Try to load existing identity using the shared package if _, err := os.Stat(identityFile); err == nil { info, err := encryption.LoadIdentity(identityFile) @@ -489,8 +500,19 @@ func (n *Node) Stop() error { func (n *Node) Start(ctx context.Context) error { n.logger.Info("Starting network node", zap.String("data_dir", n.config.Node.DataDir)) + // Expand ~ in data directory path + dataDir := n.config.Node.DataDir + dataDir = os.ExpandEnv(dataDir) + if strings.HasPrefix(dataDir, "~") { + home, err := os.UserHomeDir() + if err != nil { + return fmt.Errorf("failed to determine home directory: %w", err) + } + dataDir = filepath.Join(home, dataDir[1:]) + } + // Create data directory - if err := os.MkdirAll(n.config.Node.DataDir, 0755); err != nil { + if err := os.MkdirAll(dataDir, 0755); err != nil { return fmt.Errorf("failed to create data directory: %w", err) }