From 6a86592cad885b6141272b2e6c1f0e42957b6f39 Mon Sep 17 00:00:00 2001 From: anonpenguin23 Date: Mon, 10 Nov 2025 05:34:50 +0200 Subject: [PATCH] refactor: streamline development and production command structure - Consolidated development commands into a new `dev` command group for better organization. - Introduced a `prod` command group to manage production environment operations. - Updated Makefile to simplify the development environment setup and improve logging. - Enhanced README to clarify the development process and health check requirements. - Removed deprecated configuration and service management commands to streamline the CLI interface. --- .goreleaser.yaml | 22 ++-- CHANGELOG.md | 104 +++++++++++----- CONTRIBUTING.md | 12 +- Makefile | 30 ++--- PRODUCTION_INSTALL.md | 158 +++++++++++++++++++++++ README.md | 38 +++--- cmd/cli/main.go | 22 ++-- cmd/gateway/config.go | 2 +- cmd/node/main.go | 4 +- pkg/auth/simple_auth.go | 116 +++++++++++++++++ pkg/cli/auth_commands.go | 26 ++-- pkg/cli/basic_commands.go | 8 +- pkg/cli/dev_commands.go | 18 +-- pkg/cli/env_commands.go | 16 +-- pkg/cli/prod_commands.go | 14 +-- pkg/environments/development/runner.go | 166 ++++++++++++++++++------- pkg/gateway/auth_handlers.go | 102 +++++++++++++++ pkg/gateway/middleware.go | 2 +- pkg/gateway/routes.go | 1 + pkg/ipfs/client.go | 21 +++- pkg/ipfs/cluster.go | 14 ++- pkg/rqlite/rqlite.go | 103 +++++++++++++-- scripts/install-debros-network.sh | 24 ++-- 23 files changed, 814 insertions(+), 209 deletions(-) create mode 100644 PRODUCTION_INSTALL.md create mode 100644 pkg/auth/simple_auth.go diff --git a/.goreleaser.yaml b/.goreleaser.yaml index 5a1f0ff..6cebf4a 100644 --- a/.goreleaser.yaml +++ b/.goreleaser.yaml @@ -1,6 +1,6 @@ # GoReleaser Configuration for DeBros Network -# Builds and releases the network-cli binary for multiple platforms -# Other binaries (node, gateway, identity) are installed via: network-cli setup +# Builds and releases the dbn binary for multiple platforms +# Other binaries (node, gateway, identity) are installed via: dbn setup project_name: debros-network @@ -8,10 +8,10 @@ env: - GO111MODULE=on builds: - # network-cli binary - only build the CLI - - id: network-cli + # dbn binary - only build the CLI + - id: dbn main: ./cmd/cli - binary: network-cli + binary: dbn goos: - linux - darwin @@ -23,10 +23,10 @@ builds: - -X main.version={{.Version}} - -X main.commit={{.ShortCommit}} - -X main.date={{.Date}} - mod_timestamp: '{{ .CommitTimestamp }}' + mod_timestamp: "{{ .CommitTimestamp }}" archives: - # Tar.gz archives for network-cli + # Tar.gz archives for dbn - id: binaries format: tar.gz name_template: "{{ .ProjectName }}_{{ .Version }}_{{ .Os }}_{{ .Arch }}" @@ -50,10 +50,10 @@ changelog: abbrev: -1 filters: exclude: - - '^docs:' - - '^test:' - - '^chore:' - - '^ci:' + - "^docs:" + - "^test:" + - "^chore:" + - "^ci:" - Merge pull request - Merge branch diff --git a/CHANGELOG.md b/CHANGELOG.md index f0b549d..f1aa70d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,30 +13,17 @@ The format is based on [Keep a Changelog][keepachangelog] and adheres to [Semant ### Deprecated ### Fixed -## [0.60.1] - 2025-11-09 +## [0.61.0] - 2025-11-10 ### Added -- Improved IPFS Cluster startup logic in development environment to ensure proper peer discovery and configuration. +- Introduced a new simplified authentication flow (`dbn auth login`) that allows users to generate an API key directly from a wallet address without signature verification (for development/testing purposes). +- Added a new `PRODUCTION_INSTALL.md` guide for production deployment using the `dbn prod` command suite. ### Changed -- Refactored IPFS Cluster initialization in the development environment to use a multi-phase startup (bootstrap first, then followers) and explicitly clean stale cluster state (pebble, peerstore) before initialization. - -### Deprecated - -### Removed - -### Fixed -- Fixed an issue where IPFS Cluster nodes in the development environment might fail to join due to incorrect bootstrap configuration or stale state. - -## [0.60.0] - 2025-11-09 - -### Added -- Introduced comprehensive `network-cli dev` commands for managing the local development environment (start, stop, status, logs). -- Added `network-cli prod` commands for streamlined production installation, upgrade, and service management on Linux systems (requires root). - -### Changed -- Refactored `Makefile` targets (`dev` and `kill`) to use the new `network-cli dev up` and `network-cli dev down` commands, significantly simplifying the development workflow. -- Removed deprecated `network-cli config`, `network-cli setup`, `network-cli service`, and `network-cli rqlite` commands, consolidating functionality under `dev` and `prod`. +- Renamed the primary CLI binary from `network-cli` to `dbn` across all configurations, documentation, and source code. +- Refactored the IPFS configuration logic in the development environment to directly modify the IPFS config file instead of relying on shell commands, improving stability. +- Improved the IPFS Cluster peer count logic to correctly handle NDJSON streaming responses from the `/peers` endpoint. +- Enhanced RQLite connection logic to retry connecting to the database if the store is not yet open, particularly for joining nodes during recovery, improving cluster stability. ### Deprecated @@ -44,14 +31,55 @@ The format is based on [Keep a Changelog][keepachangelog] and adheres to [Semant ### Fixed \n + +## [0.60.1] - 2025-11-09 + +### Added + +- Improved IPFS Cluster startup logic in development environment to ensure proper peer discovery and configuration. + +### Changed + +- Refactored IPFS Cluster initialization in the development environment to use a multi-phase startup (bootstrap first, then followers) and explicitly clean stale cluster state (pebble, peerstore) before initialization. + +### Deprecated + +### Removed + +### Fixed + +- Fixed an issue where IPFS Cluster nodes in the development environment might fail to join due to incorrect bootstrap configuration or stale state. + +## [0.60.0] - 2025-11-09 + +### Added + +- Introduced comprehensive `dbn dev` commands for managing the local development environment (start, stop, status, logs). +- Added `dbn prod` commands for streamlined production installation, upgrade, and service management on Linux systems (requires root). + +### Changed + +- Refactored `Makefile` targets (`dev` and `kill`) to use the new `dbn dev up` and `dbn dev down` commands, significantly simplifying the development workflow. +- Removed deprecated `dbn config`, `dbn setup`, `dbn service`, and `dbn rqlite` commands, consolidating functionality under `dev` and `prod`. + +### Deprecated + +### Removed + +### Fixed + +\n + ## [0.59.2] - 2025-11-08 ### Added + - Added health checks to the installation script to verify the gateway and node services are running after setup or upgrade. - The installation script now attempts to verify the downloaded binary using checksums.txt if available. - Added checks in the CLI setup to ensure systemd is available before attempting to create service files. ### Changed + - Improved the installation script to detect existing installations, stop services before upgrading, and restart them afterward to minimize downtime. - Enhanced the CLI setup process by detecting the VPS IP address earlier and improving validation feedback for cluster secrets and swarm keys. - Modified directory setup to log warnings instead of exiting if `chown` fails, providing manual instructions for fixing ownership issues. @@ -62,12 +90,17 @@ The format is based on [Keep a Changelog][keepachangelog] and adheres to [Semant ### Removed ### Fixed + \n + ## [0.59.1] - 2025-11-08 ### Added + \n + ### Changed + - Improved interactive setup to prompt for existing IPFS Cluster secret and Swarm key, allowing easier joining of existing private networks. - Updated default IPFS API URL in configuration files from `http://localhost:9105` to the standard `http://localhost:5001`. - Updated systemd service files (debros-ipfs.service and debros-ipfs-cluster.service) to correctly determine and use the IPFS and Cluster repository paths. @@ -77,14 +110,18 @@ The format is based on [Keep a Changelog][keepachangelog] and adheres to [Semant ### Removed ### Fixed + \n + ## [0.59.0] - 2025-11-08 ### Added + - Added support for asynchronous pinning of uploaded files, improving upload speed. - Added an optional `pin` flag to the storage upload endpoint to control whether content is pinned (defaults to true). ### Changed + - Improved handling of IPFS Cluster responses during the Add operation to correctly process streaming NDJSON output. ### Deprecated @@ -92,14 +129,18 @@ The format is based on [Keep a Changelog][keepachangelog] and adheres to [Semant ### Removed ### Fixed + \n + ## [0.58.0] - 2025-11-07 ### Added + - Added default configuration for IPFS Cluster and IPFS API settings in node and gateway configurations. - Added `ipfs` configuration section to node configuration, including settings for cluster API URL, replication factor, and encryption. ### Changed + - Improved error logging for cache operations in the Gateway. ### Deprecated @@ -107,13 +148,17 @@ The format is based on [Keep a Changelog][keepachangelog] and adheres to [Semant ### Removed ### Fixed + \n + ## [0.57.0] - 2025-11-07 ### Added + - Added a new endpoint `/v1/cache/mget` to retrieve multiple keys from the distributed cache in a single request. ### Changed + - Improved API key extraction logic to prioritize the `X-API-Key` header and better handle different authorization schemes (Bearer, ApiKey) while avoiding confusion with JWTs. - Refactored cache retrieval logic to use a dedicated function for decoding values from the distributed cache. @@ -122,6 +167,7 @@ The format is based on [Keep a Changelog][keepachangelog] and adheres to [Semant ### Removed ### Fixed + \n ## [0.56.0] - 2025-11-05 @@ -314,7 +360,7 @@ The format is based on [Keep a Changelog][keepachangelog] and adheres to [Semant ### Added - **HTTPS/ACME Support**: Gateway now supports automatic HTTPS with Let's Encrypt certificates via ACME - - Interactive domain configuration during `network-cli setup` command + - Interactive domain configuration during `dbn setup` command - Automatic port availability checking for ports 80 and 443 before enabling HTTPS - DNS resolution verification to ensure domain points to the server IP - TLS certificate cache directory management (`~/.debros/tls-cache`) @@ -360,8 +406,8 @@ The format is based on [Keep a Changelog][keepachangelog] and adheres to [Semant ### Changed -- **GoReleaser**: Updated to build only `network-cli` binary (v0.52.2+) - - Other binaries (node, gateway, identity) now installed via `network-cli setup` +- **GoReleaser**: Updated to build only `dbn` binary (v0.52.2+) + - Other binaries (node, gateway, identity) now installed via `dbn setup` - Cleaner, smaller release packages - Resolves archive mismatch errors - **GitHub Actions**: Updated artifact actions from v3 to v4 (deprecated versions) @@ -379,7 +425,7 @@ The format is based on [Keep a Changelog][keepachangelog] and adheres to [Semant - **CLI Refactor**: Modularized monolithic CLI into `pkg/cli/` package structure for better maintainability - New `environment.go`: Multi-environment management system (local, devnet, testnet) - New `env_commands.go`: Environment switching commands (`env list`, `env switch`, `devnet enable`, `testnet enable`) - - New `setup.go`: Interactive VPS installation command (`network-cli setup`) that replaces bash install script + - New `setup.go`: Interactive VPS installation command (`dbn setup`) that replaces bash install script - New `service.go`: Systemd service management commands (`service start|stop|restart|status|logs`) - New `auth_commands.go`, `config_commands.go`, `basic_commands.go`: Refactored commands into modular pkg/cli - **Release Pipeline**: Complete automated release infrastructure via `.goreleaser.yaml` and GitHub Actions @@ -401,7 +447,7 @@ The format is based on [Keep a Changelog][keepachangelog] and adheres to [Semant - All business logic moved to modular `pkg/cli/` functions - Easier to test, maintain, and extend individual commands - **Installation**: `scripts/install-debros-network.sh` now APT-ready with fallback to source build -- **Setup Process**: Consolidated all installation logic into `network-cli setup` command +- **Setup Process**: Consolidated all installation logic into `dbn setup` command - Single unified installation regardless of installation method - Interactive user experience with clear progress indicators @@ -412,7 +458,7 @@ The format is based on [Keep a Changelog][keepachangelog] and adheres to [Semant ### Added - One-command `make dev` target to start full development stack (bootstrap + node2 + node3 + gateway in background) -- New `network-cli config init` (no --type) generates complete development stack with all configs and identities +- New `dbn config init` (no --type) generates complete development stack with all configs and identities - Full stack initialization with auto-generated peer identities for bootstrap and all nodes - Explicit control over LibP2P listen addresses for better localhost/development support - Production/development mode detection for NAT services (disabled for localhost, enabled for production) @@ -423,8 +469,8 @@ The format is based on [Keep a Changelog][keepachangelog] and adheres to [Semant - Simplified Makefile: removed legacy dev commands, replaced with unified `make dev` target - Updated README with clearer getting started instructions (single `make dev` command) -- Simplified `network-cli config init` behavior: defaults to generating full stack instead of single node -- `network-cli config init` now handles bootstrap peer discovery and join addresses automatically +- Simplified `dbn config init` behavior: defaults to generating full stack instead of single node +- `dbn config init` now handles bootstrap peer discovery and join addresses automatically - LibP2P configuration: removed always-on NAT services for development environments - Code formatting in pkg/node/node.go (indentation fixes in bootstrapPeerSource) @@ -620,7 +666,7 @@ The format is based on [Keep a Changelog][keepachangelog] and adheres to [Semant ### Removed -- Removed cli, network-cli binaries from project +- Removed cli, dbn binaries from project - Removed AI_CONTEXT.md - Removed Network.md - Removed unused log from monitoring.go diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index efff5c6..f93d30f 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -22,19 +22,19 @@ make deps - Test: `make test` - Format/Vet: `make fmt vet` (or `make lint`) -``` +```` Useful CLI commands: ```bash -./bin/network-cli health -./bin/network-cli peers -./bin/network-cli status -``` +./bin/dbn health +./bin/dbn peers +./bin/dbn status +```` ## Versioning -- The CLI reports its version via `network-cli version`. +- The CLI reports its version via `dbn version`. - Releases are tagged (e.g., `v0.18.0-beta`) and published via GoReleaser. ## Pull Requests diff --git a/Makefile b/Makefile index 7946d0f..227ded6 100644 --- a/Makefile +++ b/Makefile @@ -21,7 +21,7 @@ test-e2e: .PHONY: build clean test run-node run-node2 run-node3 run-example deps tidy fmt vet lint clear-ports install-hooks kill -VERSION := 0.60.1 +VERSION := 0.61.0 COMMIT ?= $(shell git rev-parse --short HEAD 2>/dev/null || echo unknown) DATE ?= $(shell date -u +%Y-%m-%dT%H:%M:%SZ) LDFLAGS := -X 'main.version=$(VERSION)' -X 'main.commit=$(COMMIT)' -X 'main.date=$(DATE)' @@ -32,10 +32,10 @@ build: deps @mkdir -p bin go build -ldflags "$(LDFLAGS)" -o bin/identity ./cmd/identity go build -ldflags "$(LDFLAGS)" -o bin/node ./cmd/node - go build -ldflags "$(LDFLAGS)" -o bin/network-cli cmd/cli/main.go + go build -ldflags "$(LDFLAGS)" -o bin/dbn cmd/cli/main.go # Inject gateway build metadata via pkg path variables go build -ldflags "$(LDFLAGS) -X 'github.com/DeBrosOfficial/network/pkg/gateway.BuildVersion=$(VERSION)' -X 'github.com/DeBrosOfficial/network/pkg/gateway.BuildCommit=$(COMMIT)' -X 'github.com/DeBrosOfficial/network/pkg/gateway.BuildTime=$(DATE)'" -o bin/gateway ./cmd/gateway - @echo "Build complete! Run ./bin/network-cli version" + @echo "Build complete! Run ./bin/dbn version" # Install git hooks install-hooks: @@ -53,7 +53,7 @@ clean: run-node: @echo "Starting bootstrap node..." @echo "Config: ~/.debros/bootstrap.yaml" - @echo "Generate it with: network-cli config init --type bootstrap" + @echo "Generate it with: dbn config init --type bootstrap" go run ./cmd/node --config node.yaml # Run second node (regular) - requires join address of bootstrap node @@ -61,7 +61,7 @@ run-node: run-node2: @echo "Starting regular node (node.yaml)..." @echo "Config: ~/.debros/node.yaml" - @echo "Generate it with: network-cli config init --type node --join localhost:5001 --bootstrap-peers ''" + @echo "Generate it with: dbn config init --type node --join localhost:5001 --bootstrap-peers ''" go run ./cmd/node --config node2.yaml # Run third node (regular) - requires join address of bootstrap node @@ -69,27 +69,27 @@ run-node2: run-node3: @echo "Starting regular node (node2.yaml)..." @echo "Config: ~/.debros/node2.yaml" - @echo "Generate it with: network-cli config init --type node --name node2.yaml --join localhost:5001 --bootstrap-peers ''" + @echo "Generate it with: dbn config init --type node --name node2.yaml --join localhost:5001 --bootstrap-peers ''" go run ./cmd/node --config node3.yaml # Run gateway HTTP server # Usage examples: # make run-gateway # uses ~/.debros/gateway.yaml -# Config generated with: network-cli config init --type gateway +# Config generated with: dbn config init --type gateway run-gateway: @echo "Starting gateway HTTP server..." @echo "Note: Config must be in ~/.debros/gateway.yaml" - @echo "Generate it with: network-cli config init --type gateway" + @echo "Generate it with: dbn config init --type gateway" go run ./cmd/gateway # Development environment target -# Uses network-cli dev up to start full stack with dependency and port checking +# Uses dbn dev up to start full stack with dependency and port checking dev: build - @./bin/network-cli dev up + @./bin/dbn dev up -# Kill all processes using network-cli dev down +# Kill all processes using dbn dev down kill: - @./bin/network-cli dev down + @./bin/dbn dev down # Help help: @@ -108,9 +108,9 @@ help: @echo " - Includes comprehensive logging" @echo " make kill - Stop all development services" @echo "" - @echo "Development Management (via network-cli):" - @echo " ./bin/network-cli dev status - Show status of all dev services" - @echo " ./bin/network-cli dev logs [--follow]" + @echo "Development Management (via dbn):" + @echo " ./bin/dbn dev status - Show status of all dev services" + @echo " ./bin/dbn dev logs [--follow]" @echo "" @echo "Individual Node Targets (advanced):" @echo " run-node - Start bootstrap node directly" diff --git a/PRODUCTION_INSTALL.md b/PRODUCTION_INSTALL.md new file mode 100644 index 0000000..e6ad959 --- /dev/null +++ b/PRODUCTION_INSTALL.md @@ -0,0 +1,158 @@ +# Production Installation Guide - DeBros Network + +This guide covers production deployment of the DeBros Network using the `dbn prod` command suite. + +## System Requirements + +- **OS**: Ubuntu 20.04 LTS or later, Debian 11+, or other Linux distributions +- **Architecture**: x86_64 (amd64) or ARM64 (aarch64) +- **RAM**: Minimum 4GB, recommended 8GB+ +- **Storage**: Minimum 50GB SSD recommended +- **Ports**: + - 4001 (P2P networking) + - 4501 (IPFS HTTP API - bootstrap), 4502/4503 (node2/node3) + - 5001-5003 (RQLite HTTP - one per node) + - 6001 (Gateway) + - 7001-7003 (RQLite Raft - one per node) + - 9094 (IPFS Cluster API - bootstrap), 9104/9114 (node2/node3) + - 3320/3322 (Olric) + - 80, 443 (for HTTPS with Let's Encrypt) + +## Installation + +### Prerequisites + +1. **Root access required**: All production operations require sudo/root privileges +2. **Supported distros**: Ubuntu, Debian, Fedora (via package manager) +3. **Basic tools**: `curl`, `git`, `make`, `build-essential`, `wget` + +### Single-Node Bootstrap Installation + +Deploy the first node (bootstrap node) on a VPS: + +```bash +sudo dbn prod install --bootstrap +``` + +This will: + +1. Check system prerequisites (OS, arch, root privileges, basic tools) +2. Provision the `debros` system user and filesystem structure at `~/.debros` +3. Download and install all required binaries (Go, RQLite, IPFS, IPFS Cluster, Olric, DeBros) +4. Generate secrets (cluster secret, swarm key, node identity) +5. Initialize repositories (IPFS, IPFS Cluster, RQLite) +6. Generate configurations for bootstrap node +7. Create and start systemd services + +All files will be under `/home/debros/.debros`: + +``` +~/.debros/ +├── bin/ # Compiled binaries +├── configs/ # YAML configurations +├── data/ +│ ├── ipfs/ # IPFS repository +│ ├── ipfs-cluster/ # IPFS Cluster state +│ └── rqlite/ # RQLite database +├── logs/ # Service logs +└── secrets/ # Keys and certificates +``` + +## Service Management + +### Check Service Status + +```bash +sudo systemctl status debros-node-bootstrap +sudo systemctl status debros-gateway +sudo systemctl status debros-rqlite-bootstrap +``` + +### View Service Logs + +```bash +# Bootstrap node logs +sudo journalctl -u debros-node-bootstrap -f + +# Gateway logs +sudo journalctl -u debros-gateway -f + +# All services +sudo journalctl -u "debros-*" -f +``` + +## Health Checks + +After installation, verify services are running: + +```bash +# Check IPFS +curl http://localhost:4501/api/v0/id + +# Check RQLite cluster +curl http://localhost:5001/status + +# Check Gateway +curl http://localhost:6001/health + +# Check Olric +curl http://localhost:3320/ping +``` + +## Port Reference + +### Development Environment (via `make dev`) + +- IPFS API: 4501 (bootstrap), 4502 (node2), 4503 (node3) +- RQLite HTTP: 5001, 5002, 5003 +- RQLite Raft: 7001, 7002, 7003 +- IPFS Cluster: 9094, 9104, 9114 +- P2P: 4001, 4002, 4003 +- Gateway: 6001 +- Olric: 3320, 3322 + +### Production Environment (via `sudo dbn prod install`) + +- Same port assignments as development for consistency + +## Configuration Files + +Key configuration files are located in `~/.debros/configs/`: + +- **bootstrap.yaml**: Bootstrap node configuration +- **node.yaml**: Regular node configuration +- **gateway.yaml**: HTTP gateway configuration +- **olric.yaml**: In-memory cache configuration + +Edit these files directly for advanced configuration, then restart services: + +```bash +sudo systemctl restart debros-node-bootstrap +``` + +## Troubleshooting + +### Port already in use + +Check which process is using the port: + +```bash +sudo lsof -i :4501 +sudo lsof -i :5001 +sudo lsof -i :7001 +``` + +Kill conflicting processes or change ports in config. + +### RQLite cluster not forming + +Ensure: + +1. Bootstrap node is running: `systemctl status debros-rqlite-bootstrap` +2. Network connectivity between nodes on ports 5001+ (HTTP) and 7001+ (Raft) +3. Check logs: `journalctl -u debros-rqlite-bootstrap -f` + +--- + +**Last Updated**: November 2024 +**Compatible with**: Network v1.0.0+ diff --git a/README.md b/README.md index 451141e..6bb7bf0 100644 --- a/README.md +++ b/README.md @@ -34,7 +34,7 @@ DeBros Network is a decentralized peer-to-peer data platform built in Go. It com 2. Generate local configuration (bootstrap, node2, node3, gateway): ```bash - ./bin/network-cli config init + ./bin/dbn config init ``` 3. Launch the full development stack: @@ -48,10 +48,10 @@ DeBros Network is a decentralized peer-to-peer data platform built in Go. It com 4. Validate the network from another terminal: ```bash - ./bin/network-cli health - ./bin/network-cli peers - ./bin/network-cli pubsub publish notifications "Hello World" - ./bin/network-cli pubsub subscribe notifications 10s + ./bin/dbn health + ./bin/dbn peers + ./bin/dbn pubsub publish notifications "Hello World" + ./bin/dbn pubsub subscribe notifications 10s ``` ## Components & Ports @@ -78,7 +78,7 @@ Validation reminders: - Bootstrap nodes cannot define a join address - Multiaddrs must end with `/p2p/` -Regenerate configs any time with `./bin/network-cli config init --force`. +Regenerate configs any time with `./bin/dbn config init --force`. ## CLI Highlights @@ -87,33 +87,33 @@ All commands accept `--format json`, `--timeout `, and `--bootstrap - ./bin/network-cli pubsub subscribe 30s - ./bin/network-cli pubsub topics + ./bin/dbn pubsub publish + ./bin/dbn pubsub subscribe 30s + ./bin/dbn pubsub topics ``` Credentials live at `~/.debros/credentials.json` with user-only permissions. @@ -145,7 +145,7 @@ Common endpoints (see `openapi/gateway.yaml` for the full spec): - **Config directory errors**: Ensure `~/.debros/` exists, is writable, and has free disk space (`touch ~/.debros/test && rm ~/.debros/test`). - **Port conflicts**: Inspect with `lsof -i :4001` (or other ports) and stop conflicting processes or regenerate configs with new ports. -- **Missing configs**: Run `./bin/network-cli config init` before starting nodes. +- **Missing configs**: Run `./bin/dbn config init` before starting nodes. - **Cluster join issues**: Confirm the bootstrap node is running, `peer.info` multiaddr matches `bootstrap_peers`, and firewall rules allow the P2P ports. ## Resources diff --git a/cmd/cli/main.go b/cmd/cli/main.go index c9db844..2d4ac60 100644 --- a/cmd/cli/main.go +++ b/cmd/cli/main.go @@ -34,7 +34,7 @@ func main() { switch command { case "version": - fmt.Printf("network-cli %s", version) + fmt.Printf("dbn %s", version) if commit != "" { fmt.Printf(" (commit %s)", commit) } @@ -60,7 +60,7 @@ func main() { fmt.Printf(" Gateway URL: %s\n", env.GatewayURL) } } else { - fmt.Fprintf(os.Stderr, "Usage: network-cli %s enable\n", command) + fmt.Fprintf(os.Stderr, "Usage: dbn %s enable\n", command) os.Exit(1) } @@ -89,7 +89,7 @@ func main() { // Query command case "query": if len(args) == 0 { - fmt.Fprintf(os.Stderr, "Usage: network-cli query \n") + fmt.Fprintf(os.Stderr, "Usage: dbn query \n") os.Exit(1) } cli.HandleQueryCommand(args[0], format, timeout) @@ -101,7 +101,7 @@ func main() { // Connect command case "connect": if len(args) == 0 { - fmt.Fprintf(os.Stderr, "Usage: network-cli connect \n") + fmt.Fprintf(os.Stderr, "Usage: dbn connect \n") os.Exit(1) } cli.HandleConnectCommand(args[0], timeout) @@ -136,7 +136,7 @@ func parseGlobalFlags(args []string) { func showHelp() { fmt.Printf("Network CLI - Distributed P2P Network Management Tool\n\n") - fmt.Printf("Usage: network-cli [args...]\n\n") + fmt.Printf("Usage: dbn [args...]\n\n") fmt.Printf("🌍 Environment Management:\n") fmt.Printf(" env list - List available environments\n") @@ -187,16 +187,16 @@ func showHelp() { fmt.Printf("Examples:\n") fmt.Printf(" # Switch to devnet\n") - fmt.Printf(" network-cli devnet enable\n\n") + fmt.Printf(" dbn devnet enable\n\n") fmt.Printf(" # Authenticate and query\n") - fmt.Printf(" network-cli auth login\n") - fmt.Printf(" network-cli query \"SELECT * FROM users LIMIT 10\"\n\n") + fmt.Printf(" dbn auth login\n") + fmt.Printf(" dbn query \"SELECT * FROM users LIMIT 10\"\n\n") fmt.Printf(" # Setup VPS (Linux only)\n") - fmt.Printf(" sudo network-cli setup\n\n") + fmt.Printf(" sudo dbn setup\n\n") fmt.Printf(" # Manage services\n") - fmt.Printf(" sudo network-cli service status all\n") - fmt.Printf(" sudo network-cli service logs node --follow\n") + fmt.Printf(" sudo dbn service status all\n") + fmt.Printf(" sudo dbn service logs node --follow\n") } diff --git a/cmd/gateway/config.go b/cmd/gateway/config.go index cf71959..8973963 100644 --- a/cmd/gateway/config.go +++ b/cmd/gateway/config.go @@ -72,7 +72,7 @@ func parseGatewayConfig(logger *logging.ColoredLogger) *gateway.Config { zap.String("path", configPath), zap.Error(err)) fmt.Fprintf(os.Stderr, "\nConfig file not found at %s\n", configPath) - fmt.Fprintf(os.Stderr, "Generate it using: network-cli config init --type gateway\n") + fmt.Fprintf(os.Stderr, "Generate it using: dbn config init --type gateway\n") os.Exit(1) } diff --git a/cmd/node/main.go b/cmd/node/main.go index 949ecd3..451c8f8 100644 --- a/cmd/node/main.go +++ b/cmd/node/main.go @@ -92,8 +92,8 @@ func select_data_dir_check(configName *string) { fmt.Fprintf(os.Stderr, "\n❌ Configuration Error:\n") fmt.Fprintf(os.Stderr, "Config file not found at %s\n", configPath) fmt.Fprintf(os.Stderr, "\nGenerate it with one of:\n") - fmt.Fprintf(os.Stderr, " network-cli config init --type bootstrap\n") - fmt.Fprintf(os.Stderr, " network-cli config init --type node --bootstrap-peers ''\n") + fmt.Fprintf(os.Stderr, " dbn config init --type bootstrap\n") + fmt.Fprintf(os.Stderr, " dbn config init --type node --bootstrap-peers ''\n") os.Exit(1) } } diff --git a/pkg/auth/simple_auth.go b/pkg/auth/simple_auth.go new file mode 100644 index 0000000..246ed80 --- /dev/null +++ b/pkg/auth/simple_auth.go @@ -0,0 +1,116 @@ +package auth + +import ( + "bufio" + "bytes" + "encoding/json" + "fmt" + "io" + "net/http" + "os" + "strings" + "time" +) + +// PerformSimpleAuthentication performs a simple authentication flow where the user +// provides a wallet address and receives an API key without signature verification +func PerformSimpleAuthentication(gatewayURL string) (*Credentials, error) { + reader := bufio.NewReader(os.Stdin) + + fmt.Println("\n🔐 Simple Wallet Authentication") + fmt.Println("================================") + + // Read wallet address + fmt.Print("Enter your wallet address (0x...): ") + walletInput, err := reader.ReadString('\n') + if err != nil { + return nil, fmt.Errorf("failed to read wallet address: %w", err) + } + + wallet := strings.TrimSpace(walletInput) + if wallet == "" { + return nil, fmt.Errorf("wallet address cannot be empty") + } + + // Validate wallet format (basic check) + if !strings.HasPrefix(wallet, "0x") && !strings.HasPrefix(wallet, "0X") { + wallet = "0x" + wallet + } + + if !ValidateWalletAddress(wallet) { + return nil, fmt.Errorf("invalid wallet address format") + } + + // Read namespace (optional) + fmt.Print("Enter namespace (press Enter for 'default'): ") + nsInput, err := reader.ReadString('\n') + if err != nil { + return nil, fmt.Errorf("failed to read namespace: %w", err) + } + + namespace := strings.TrimSpace(nsInput) + if namespace == "" { + namespace = "default" + } + + fmt.Printf("\n✅ Wallet: %s\n", wallet) + fmt.Printf("✅ Namespace: %s\n", namespace) + fmt.Println("⏳ Requesting API key from gateway...") + + // Request API key from gateway + apiKey, err := requestAPIKeyFromGateway(gatewayURL, wallet, namespace) + if err != nil { + return nil, fmt.Errorf("failed to request API key: %w", err) + } + + // Create credentials + creds := &Credentials{ + APIKey: apiKey, + Namespace: namespace, + UserID: wallet, + Wallet: wallet, + IssuedAt: time.Now(), + } + + fmt.Printf("\n🎉 Authentication successful!\n") + fmt.Printf("📝 API Key: %s\n", creds.APIKey) + + return creds, nil +} + +// requestAPIKeyFromGateway calls the gateway's simple-key endpoint to generate an API key +func requestAPIKeyFromGateway(gatewayURL, wallet, namespace string) (string, error) { + reqBody := map[string]string{ + "wallet": wallet, + "namespace": namespace, + } + + payload, err := json.Marshal(reqBody) + if err != nil { + return "", fmt.Errorf("failed to marshal request: %w", err) + } + + endpoint := gatewayURL + "/v1/auth/simple-key" + resp, err := http.Post(endpoint, "application/json", bytes.NewReader(payload)) + if err != nil { + return "", fmt.Errorf("failed to call gateway: %w", err) + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + body, _ := io.ReadAll(resp.Body) + return "", fmt.Errorf("gateway returned status %d: %s", resp.StatusCode, string(body)) + } + + var respBody map[string]interface{} + if err := json.NewDecoder(resp.Body).Decode(&respBody); err != nil { + return "", fmt.Errorf("failed to decode response: %w", err) + } + + apiKey, ok := respBody["api_key"].(string) + if !ok || apiKey == "" { + return "", fmt.Errorf("no api_key in response") + } + + return apiKey, nil +} diff --git a/pkg/cli/auth_commands.go b/pkg/cli/auth_commands.go index 5e795c6..3474288 100644 --- a/pkg/cli/auth_commands.go +++ b/pkg/cli/auth_commands.go @@ -33,29 +33,34 @@ func HandleAuthCommand(args []string) { func showAuthHelp() { fmt.Printf("🔐 Authentication Commands\n\n") - fmt.Printf("Usage: network-cli auth \n\n") + fmt.Printf("Usage: dbn auth \n\n") fmt.Printf("Subcommands:\n") - fmt.Printf(" login - Authenticate with wallet\n") + fmt.Printf(" login - Authenticate by providing your wallet address\n") fmt.Printf(" logout - Clear stored credentials\n") fmt.Printf(" whoami - Show current authentication status\n") fmt.Printf(" status - Show detailed authentication info\n\n") fmt.Printf("Examples:\n") - fmt.Printf(" network-cli auth login\n") - fmt.Printf(" network-cli auth whoami\n") - fmt.Printf(" network-cli auth status\n") - fmt.Printf(" network-cli auth logout\n\n") + fmt.Printf(" dbn auth login # Enter wallet address interactively\n") + fmt.Printf(" dbn auth whoami # Check who you're logged in as\n") + fmt.Printf(" dbn auth status # View detailed authentication info\n") + fmt.Printf(" dbn auth logout # Clear all stored credentials\n\n") fmt.Printf("Environment Variables:\n") fmt.Printf(" DEBROS_GATEWAY_URL - Gateway URL (overrides environment config)\n\n") + fmt.Printf("Authentication Flow:\n") + fmt.Printf(" 1. Run 'dbn auth login'\n") + fmt.Printf(" 2. Enter your wallet address when prompted\n") + fmt.Printf(" 3. Enter your namespace (or press Enter for 'default')\n") + fmt.Printf(" 4. An API key will be generated and saved to ~/.debros/credentials.json\n\n") fmt.Printf("Note: Authentication uses the currently active environment.\n") - fmt.Printf(" Use 'network-cli env current' to see your active environment.\n") + fmt.Printf(" Use 'dbn env current' to see your active environment.\n") } func handleAuthLogin() { gatewayURL := getGatewayURL() fmt.Printf("🔐 Authenticating with gateway at: %s\n", gatewayURL) - // Use the wallet authentication flow - creds, err := auth.PerformWalletAuthentication(gatewayURL) + // Use the simple authentication flow + creds, err := auth.PerformSimpleAuthentication(gatewayURL) if err != nil { fmt.Fprintf(os.Stderr, "❌ Authentication failed: %v\n", err) os.Exit(1) @@ -72,6 +77,7 @@ func handleAuthLogin() { fmt.Printf("📁 Credentials saved to: %s\n", credsPath) fmt.Printf("🎯 Wallet: %s\n", creds.Wallet) fmt.Printf("🏢 Namespace: %s\n", creds.Namespace) + fmt.Printf("🔑 API Key: %s\n", creds.APIKey) } func handleAuthLogout() { @@ -93,7 +99,7 @@ func handleAuthWhoami() { creds, exists := store.GetCredentialsForGateway(gatewayURL) if !exists || !creds.IsValid() { - fmt.Println("❌ Not authenticated - run 'network-cli auth login' to authenticate") + fmt.Println("❌ Not authenticated - run 'dbn auth login' to authenticate") os.Exit(1) } diff --git a/pkg/cli/basic_commands.go b/pkg/cli/basic_commands.go index 368160b..d3db10d 100644 --- a/pkg/cli/basic_commands.go +++ b/pkg/cli/basic_commands.go @@ -158,7 +158,7 @@ func HandlePeerIDCommand(format string, timeout time.Duration) { // HandlePubSubCommand handles pubsub commands func HandlePubSubCommand(args []string, format string, timeout time.Duration) { if len(args) == 0 { - fmt.Fprintf(os.Stderr, "Usage: network-cli pubsub [args...]\n") + fmt.Fprintf(os.Stderr, "Usage: dbn pubsub [args...]\n") os.Exit(1) } @@ -179,7 +179,7 @@ func HandlePubSubCommand(args []string, format string, timeout time.Duration) { switch subcommand { case "publish": if len(args) < 3 { - fmt.Fprintf(os.Stderr, "Usage: network-cli pubsub publish \n") + fmt.Fprintf(os.Stderr, "Usage: dbn pubsub publish \n") os.Exit(1) } err := cli.PubSub().Publish(ctx, args[1], []byte(args[2])) @@ -191,7 +191,7 @@ func HandlePubSubCommand(args []string, format string, timeout time.Duration) { case "subscribe": if len(args) < 2 { - fmt.Fprintf(os.Stderr, "Usage: network-cli pubsub subscribe [duration]\n") + fmt.Fprintf(os.Stderr, "Usage: dbn pubsub subscribe [duration]\n") os.Exit(1) } duration := 30 * time.Second @@ -243,7 +243,7 @@ func HandlePubSubCommand(args []string, format string, timeout time.Duration) { // Helper functions func createClient() (client.NetworkClient, error) { - config := client.DefaultClientConfig("network-cli") + config := client.DefaultClientConfig("dbn") // Check for existing credentials using enhanced authentication creds, err := auth.GetValidEnhancedCredentials() diff --git a/pkg/cli/dev_commands.go b/pkg/cli/dev_commands.go index 8d087b3..7486447 100644 --- a/pkg/cli/dev_commands.go +++ b/pkg/cli/dev_commands.go @@ -40,7 +40,7 @@ func HandleDevCommand(args []string) { func showDevHelp() { fmt.Printf("🚀 Development Environment Commands\n\n") - fmt.Printf("Usage: network-cli dev [options]\n\n") + fmt.Printf("Usage: dbn dev [options]\n\n") fmt.Printf("Subcommands:\n") fmt.Printf(" up - Start development environment (bootstrap + 2 nodes + gateway)\n") fmt.Printf(" down - Stop all development services\n") @@ -48,10 +48,10 @@ func showDevHelp() { fmt.Printf(" logs - Tail logs for a component\n") fmt.Printf(" help - Show this help\n\n") fmt.Printf("Examples:\n") - fmt.Printf(" network-cli dev up\n") - fmt.Printf(" network-cli dev down\n") - fmt.Printf(" network-cli dev status\n") - fmt.Printf(" network-cli dev logs bootstrap --follow\n") + fmt.Printf(" dbn dev up\n") + fmt.Printf(" dbn dev down\n") + fmt.Printf(" dbn dev status\n") + fmt.Printf(" dbn dev logs bootstrap --follow\n") } func handleDevUp(args []string) { @@ -114,9 +114,9 @@ func handleDevUp(args []string) { fmt.Printf(" Anon SOCKS: 127.0.0.1:9050\n") fmt.Printf(" Olric Cache: http://localhost:3320\n\n") fmt.Printf("Useful commands:\n") - fmt.Printf(" network-cli dev status - Show status\n") - fmt.Printf(" network-cli dev logs bootstrap - Bootstrap logs\n") - fmt.Printf(" network-cli dev down - Stop all services\n\n") + fmt.Printf(" dbn dev status - Show status\n") + fmt.Printf(" dbn dev logs bootstrap - Bootstrap logs\n") + fmt.Printf(" dbn dev down - Stop all services\n\n") fmt.Printf("Logs directory: %s/logs\n\n", debrosDir) } @@ -152,7 +152,7 @@ func handleDevStatus(args []string) { func handleDevLogs(args []string) { if len(args) == 0 { - fmt.Fprintf(os.Stderr, "Usage: network-cli dev logs [--follow]\n") + fmt.Fprintf(os.Stderr, "Usage: dbn dev logs [--follow]\n") fmt.Fprintf(os.Stderr, "\nComponents: bootstrap, node2, node3, gateway, ipfs-bootstrap, ipfs-node2, ipfs-node3, olric, anon\n") os.Exit(1) } diff --git a/pkg/cli/env_commands.go b/pkg/cli/env_commands.go index 064f871..4094a06 100644 --- a/pkg/cli/env_commands.go +++ b/pkg/cli/env_commands.go @@ -35,7 +35,7 @@ func HandleEnvCommand(args []string) { func showEnvHelp() { fmt.Printf("🌍 Environment Management Commands\n\n") - fmt.Printf("Usage: network-cli env \n\n") + fmt.Printf("Usage: dbn env \n\n") fmt.Printf("Subcommands:\n") fmt.Printf(" list - List all available environments\n") fmt.Printf(" current - Show current active environment\n") @@ -46,12 +46,12 @@ func showEnvHelp() { fmt.Printf(" devnet - Development network (https://devnet.debros.network)\n") fmt.Printf(" testnet - Test network (https://testnet.debros.network)\n\n") fmt.Printf("Examples:\n") - fmt.Printf(" network-cli env list\n") - fmt.Printf(" network-cli env current\n") - fmt.Printf(" network-cli env switch devnet\n") - fmt.Printf(" network-cli env enable testnet\n") - fmt.Printf(" network-cli devnet enable # Shorthand for switch to devnet\n") - fmt.Printf(" network-cli testnet enable # Shorthand for switch to testnet\n") + fmt.Printf(" dbn env list\n") + fmt.Printf(" dbn env current\n") + fmt.Printf(" dbn env switch devnet\n") + fmt.Printf(" dbn env enable testnet\n") + fmt.Printf(" dbn devnet enable # Shorthand for switch to devnet\n") + fmt.Printf(" dbn testnet enable # Shorthand for switch to testnet\n") } func handleEnvList() { @@ -99,7 +99,7 @@ func handleEnvCurrent() { func handleEnvSwitch(args []string) { if len(args) == 0 { - fmt.Fprintf(os.Stderr, "Usage: network-cli env switch \n") + fmt.Fprintf(os.Stderr, "Usage: dbn env switch \n") fmt.Fprintf(os.Stderr, "Available: local, devnet, testnet\n") os.Exit(1) } diff --git a/pkg/cli/prod_commands.go b/pkg/cli/prod_commands.go index 262317f..dc5fbdd 100644 --- a/pkg/cli/prod_commands.go +++ b/pkg/cli/prod_commands.go @@ -43,7 +43,7 @@ func HandleProdCommand(args []string) { func showProdHelp() { fmt.Printf("Production Environment Commands\n\n") - fmt.Printf("Usage: network-cli prod [options]\n\n") + fmt.Printf("Usage: dbn prod [options]\n\n") fmt.Printf("Subcommands:\n") fmt.Printf(" install - Full production bootstrap (requires root/sudo)\n") fmt.Printf(" Options:\n") @@ -59,10 +59,10 @@ func showProdHelp() { fmt.Printf(" --follow - Follow logs in real-time\n") fmt.Printf(" uninstall - Remove production services (requires root/sudo)\n\n") fmt.Printf("Examples:\n") - fmt.Printf(" sudo network-cli prod install --bootstrap\n") - fmt.Printf(" sudo network-cli prod install --peers /ip4/1.2.3.4/tcp/4001/p2p/Qm...\n") - fmt.Printf(" network-cli prod status\n") - fmt.Printf(" network-cli prod logs node --follow\n") + fmt.Printf(" sudo dbn prod install --bootstrap\n") + fmt.Printf(" sudo dbn prod install --peers /ip4/1.2.3.4/tcp/4001/p2p/Qm...\n") + fmt.Printf(" dbn prod status\n") + fmt.Printf(" dbn prod logs node --follow\n") } func handleProdInstall(args []string) { @@ -236,12 +236,12 @@ func handleProdStatus() { fmt.Printf(" ❌ %s not found\n", debrosDir) } - fmt.Printf("\nView logs with: network-cli prod logs \n") + fmt.Printf("\nView logs with: dbn prod logs \n") } func handleProdLogs(args []string) { if len(args) == 0 { - fmt.Fprintf(os.Stderr, "Usage: network-cli prod logs [--follow]\n") + fmt.Fprintf(os.Stderr, "Usage: dbn prod logs [--follow]\n") os.Exit(1) } diff --git a/pkg/environments/development/runner.go b/pkg/environments/development/runner.go index b37c10d..178d86d 100644 --- a/pkg/environments/development/runner.go +++ b/pkg/environments/development/runner.go @@ -226,6 +226,107 @@ func readIPFSConfigValue(ctx context.Context, repoPath string, key string) (stri return "", fmt.Errorf("key %s not found in IPFS config", key) } +// configureIPFSRepo directly modifies IPFS config JSON to set addresses, bootstrap, and CORS headers +// This avoids shell commands which fail on some systems and instead manipulates the config directly +// Returns the peer ID from the config +func configureIPFSRepo(repoPath string, apiPort, gatewayPort, swarmPort int) (string, error) { + configPath := filepath.Join(repoPath, "config") + + // Read existing config + data, err := os.ReadFile(configPath) + if err != nil { + return "", fmt.Errorf("failed to read IPFS config: %w", err) + } + + var config map[string]interface{} + if err := json.Unmarshal(data, &config); err != nil { + return "", fmt.Errorf("failed to parse IPFS config: %w", err) + } + + // Set Addresses + config["Addresses"] = map[string]interface{}{ + "API": []string{fmt.Sprintf("/ip4/127.0.0.1/tcp/%d", apiPort)}, + "Gateway": []string{fmt.Sprintf("/ip4/127.0.0.1/tcp/%d", gatewayPort)}, + "Swarm": []string{ + fmt.Sprintf("/ip4/0.0.0.0/tcp/%d", swarmPort), + fmt.Sprintf("/ip6/::/tcp/%d", swarmPort), + }, + } + + // Disable AutoConf for private swarm + config["AutoConf"] = map[string]interface{}{ + "Enabled": false, + } + + // Clear Bootstrap (will be set via HTTP API after startup) + config["Bootstrap"] = []string{} + + // Clear DNS Resolvers + if dns, ok := config["DNS"].(map[string]interface{}); ok { + dns["Resolvers"] = map[string]interface{}{} + } else { + config["DNS"] = map[string]interface{}{ + "Resolvers": map[string]interface{}{}, + } + } + + // Clear Routing DelegatedRouters + if routing, ok := config["Routing"].(map[string]interface{}); ok { + routing["DelegatedRouters"] = []string{} + } else { + config["Routing"] = map[string]interface{}{ + "DelegatedRouters": []string{}, + } + } + + // Clear IPNS DelegatedPublishers + if ipns, ok := config["Ipns"].(map[string]interface{}); ok { + ipns["DelegatedPublishers"] = []string{} + } else { + config["Ipns"] = map[string]interface{}{ + "DelegatedPublishers": []string{}, + } + } + + // Set API HTTPHeaders with CORS (must be map[string][]string) + if api, ok := config["API"].(map[string]interface{}); ok { + api["HTTPHeaders"] = map[string][]string{ + "Access-Control-Allow-Origin": {"*"}, + "Access-Control-Allow-Methods": {"GET", "PUT", "POST", "DELETE", "OPTIONS"}, + "Access-Control-Allow-Headers": {"Content-Type", "X-Requested-With"}, + "Access-Control-Expose-Headers": {"Content-Length", "Content-Range"}, + } + } else { + config["API"] = map[string]interface{}{ + "HTTPHeaders": map[string][]string{ + "Access-Control-Allow-Origin": {"*"}, + "Access-Control-Allow-Methods": {"GET", "PUT", "POST", "DELETE", "OPTIONS"}, + "Access-Control-Allow-Headers": {"Content-Type", "X-Requested-With"}, + "Access-Control-Expose-Headers": {"Content-Length", "Content-Range"}, + }, + } + } + + // Write config back + updatedData, err := json.MarshalIndent(config, "", " ") + if err != nil { + return "", fmt.Errorf("failed to marshal IPFS config: %w", err) + } + + if err := os.WriteFile(configPath, updatedData, 0644); err != nil { + return "", fmt.Errorf("failed to write IPFS config: %w", err) + } + + // Extract and return peer ID + if id, ok := config["Identity"].(map[string]interface{}); ok { + if peerID, ok := id["PeerID"].(string); ok { + return peerID, nil + } + } + + return "", fmt.Errorf("could not extract peer ID from config") +} + // seedIPFSPeersWithHTTP configures each IPFS node to bootstrap with its local peers using HTTP API func (pm *ProcessManager) seedIPFSPeersWithHTTP(ctx context.Context, nodes []ipfsNodeInfo) error { fmt.Fprintf(pm.logWriter, " Seeding IPFS local bootstrap peers via HTTP API...\n") @@ -332,44 +433,11 @@ func (pm *ProcessManager) startIPFS(ctx context.Context) error { } } - // Always reapply address settings to ensure correct ports (before daemon starts) - apiAddr := fmt.Sprintf("/ip4/127.0.0.1/tcp/%d", nodes[i].apiPort) - gatewayAddr := fmt.Sprintf("/ip4/127.0.0.1/tcp/%d", nodes[i].gatewayPort) - swarmAddrs := fmt.Sprintf("[\"/ip4/0.0.0.0/tcp/%d\", \"/ip6/::/tcp/%d\"]", nodes[i].swarmPort, nodes[i].swarmPort) - - if err := exec.CommandContext(ctx, "ipfs", "config", "--repo-dir="+nodes[i].ipfsPath, "Addresses.API", apiAddr).Run(); err != nil { - fmt.Fprintf(pm.logWriter, " Warning: failed to set API address: %v\n", err) - } - if err := exec.CommandContext(ctx, "ipfs", "config", "--repo-dir="+nodes[i].ipfsPath, "Addresses.Gateway", gatewayAddr).Run(); err != nil { - fmt.Fprintf(pm.logWriter, " Warning: failed to set Gateway address: %v\n", err) - } - if err := exec.CommandContext(ctx, "ipfs", "config", "--repo-dir="+nodes[i].ipfsPath, "--json", "Addresses.Swarm", swarmAddrs).Run(); err != nil { - fmt.Fprintf(pm.logWriter, " Warning: failed to set Swarm addresses: %v\n", err) - } - - // Ensure AutoConf is disabled for private swarm repos to avoid mainnet autoconf error - if err := exec.CommandContext(ctx, "ipfs", "config", "--repo-dir="+nodes[i].ipfsPath, "--json", "AutoConf.Enabled", "false").Run(); err != nil { - fmt.Fprintf(pm.logWriter, " Warning: failed to disable AutoConf for %s: %v\n", nodes[i].name, err) - } - - // Clear 'auto' placeholders that are invalid when AutoConf is disabled - if err := exec.CommandContext(ctx, "ipfs", "config", "--repo-dir="+nodes[i].ipfsPath, "--json", "Bootstrap", "[]").Run(); err != nil { - fmt.Fprintf(pm.logWriter, " Warning: failed to clear Bootstrap for %s: %v\n", nodes[i].name, err) - } - if err := exec.CommandContext(ctx, "ipfs", "config", "--repo-dir="+nodes[i].ipfsPath, "--json", "DNS.Resolvers", "[]").Run(); err != nil { - fmt.Fprintf(pm.logWriter, " Warning: failed to clear DNS.Resolvers for %s: %v\n", nodes[i].name, err) - } - if err := exec.CommandContext(ctx, "ipfs", "config", "--repo-dir="+nodes[i].ipfsPath, "--json", "Routing.DelegatedRouters", "[]").Run(); err != nil { - fmt.Fprintf(pm.logWriter, " Warning: failed to clear Routing.DelegatedRouters for %s: %v\n", nodes[i].name, err) - } - if err := exec.CommandContext(ctx, "ipfs", "config", "--repo-dir="+nodes[i].ipfsPath, "--json", "Ipns.DelegatedPublishers", "[]").Run(); err != nil { - fmt.Fprintf(pm.logWriter, " Warning: failed to clear Ipns.DelegatedPublishers for %s: %v\n", nodes[i].name, err) - } - - // Read peer ID from config BEFORE daemon starts - peerID, err := readIPFSConfigValue(ctx, nodes[i].ipfsPath, "PeerID") + // Configure the IPFS config directly (addresses, bootstrap, DNS, routing, CORS headers) + // This replaces shell commands which can fail on some systems + peerID, err := configureIPFSRepo(nodes[i].ipfsPath, nodes[i].apiPort, nodes[i].gatewayPort, nodes[i].swarmPort) if err != nil { - fmt.Fprintf(pm.logWriter, " Warning: failed to read peer ID for %s: %v\n", nodes[i].name, err) + fmt.Fprintf(pm.logWriter, " Warning: failed to configure IPFS repo for %s: %v\n", nodes[i].name, err) } else { nodes[i].peerID = peerID fmt.Fprintf(pm.logWriter, " Peer ID for %s: %s\n", nodes[i].name, peerID) @@ -662,14 +730,24 @@ func (pm *ProcessManager) waitClusterFormed(ctx context.Context, bootstrapRestAP httpURL := fmt.Sprintf("http://127.0.0.1:%d/peers", bootstrapRestAPIPort) resp, err := http.Get(httpURL) if err == nil && resp.StatusCode == 200 { - var peers []interface{} - if err := json.NewDecoder(resp.Body).Decode(&peers); err == nil { - resp.Body.Close() - if len(peers) >= requiredPeers { - return nil // All peers have formed + // The /peers endpoint returns NDJSON (newline-delimited JSON), not a JSON array + // We need to stream-read each peer object + dec := json.NewDecoder(resp.Body) + peerCount := 0 + for { + var peer interface{} + err := dec.Decode(&peer) + if err != nil { + if err == io.EOF { + break + } + break // Stop on parse error } - } else { - resp.Body.Close() + peerCount++ + } + resp.Body.Close() + if peerCount >= requiredPeers { + return nil // All peers have formed } } if resp != nil { diff --git a/pkg/gateway/auth_handlers.go b/pkg/gateway/auth_handlers.go index dd6ab1a..ceaccaa 100644 --- a/pkg/gateway/auth_handlers.go +++ b/pkg/gateway/auth_handlers.go @@ -1125,6 +1125,108 @@ func (g *Gateway) logoutHandler(w http.ResponseWriter, r *http.Request) { writeError(w, http.StatusBadRequest, "nothing to revoke: provide refresh_token or all=true") } +// simpleAPIKeyHandler creates an API key directly from a wallet address without signature verification +// This is a simplified flow for development/testing +// Requires: POST { wallet, namespace } +func (g *Gateway) simpleAPIKeyHandler(w http.ResponseWriter, r *http.Request) { + if g.client == nil { + writeError(w, http.StatusServiceUnavailable, "client not initialized") + return + } + if r.Method != http.MethodPost { + writeError(w, http.StatusMethodNotAllowed, "method not allowed") + return + } + + var req struct { + Wallet string `json:"wallet"` + Namespace string `json:"namespace"` + } + + if err := json.NewDecoder(r.Body).Decode(&req); err != nil { + writeError(w, http.StatusBadRequest, "invalid json body") + return + } + + if strings.TrimSpace(req.Wallet) == "" { + writeError(w, http.StatusBadRequest, "wallet is required") + return + } + + ns := strings.TrimSpace(req.Namespace) + if ns == "" { + ns = strings.TrimSpace(g.cfg.ClientNamespace) + if ns == "" { + ns = "default" + } + } + + ctx := r.Context() + internalCtx := client.WithInternalAuth(ctx) + db := g.client.Database() + + // Resolve or create namespace + if _, err := db.Query(internalCtx, "INSERT OR IGNORE INTO namespaces(name) VALUES (?)", ns); err != nil { + writeError(w, http.StatusInternalServerError, err.Error()) + return + } + + nres, err := db.Query(internalCtx, "SELECT id FROM namespaces WHERE name = ? LIMIT 1", ns) + if err != nil || nres == nil || nres.Count == 0 || len(nres.Rows) == 0 || len(nres.Rows[0]) == 0 { + writeError(w, http.StatusInternalServerError, "failed to resolve namespace") + return + } + nsID := nres.Rows[0][0] + + // Check if api key already exists for (namespace, wallet) + var apiKey string + r1, err := db.Query(internalCtx, + "SELECT api_keys.key FROM wallet_api_keys JOIN api_keys ON wallet_api_keys.api_key_id = api_keys.id WHERE wallet_api_keys.namespace_id = ? AND LOWER(wallet_api_keys.wallet) = LOWER(?) LIMIT 1", + nsID, req.Wallet, + ) + if err == nil && r1 != nil && r1.Count > 0 && len(r1.Rows) > 0 && len(r1.Rows[0]) > 0 { + if s, ok := r1.Rows[0][0].(string); ok { + apiKey = s + } else { + b, _ := json.Marshal(r1.Rows[0][0]) + _ = json.Unmarshal(b, &apiKey) + } + } + + // If no existing key, create a new one + if strings.TrimSpace(apiKey) == "" { + buf := make([]byte, 18) + if _, err := rand.Read(buf); err != nil { + writeError(w, http.StatusInternalServerError, "failed to generate api key") + return + } + apiKey = "ak_" + base64.RawURLEncoding.EncodeToString(buf) + ":" + ns + + if _, err := db.Query(internalCtx, "INSERT INTO api_keys(key, name, namespace_id) VALUES (?, ?, ?)", apiKey, "", nsID); err != nil { + writeError(w, http.StatusInternalServerError, err.Error()) + return + } + + // Link wallet to api key + rid, err := db.Query(internalCtx, "SELECT id FROM api_keys WHERE key = ? LIMIT 1", apiKey) + if err == nil && rid != nil && rid.Count > 0 && len(rid.Rows) > 0 && len(rid.Rows[0]) > 0 { + apiKeyID := rid.Rows[0][0] + _, _ = db.Query(internalCtx, "INSERT OR IGNORE INTO wallet_api_keys(namespace_id, wallet, api_key_id) VALUES (?, ?, ?)", nsID, strings.ToLower(req.Wallet), apiKeyID) + } + } + + // Record ownerships (best-effort) + _, _ = db.Query(internalCtx, "INSERT OR IGNORE INTO namespace_ownership(namespace_id, owner_type, owner_id) VALUES (?, 'api_key', ?)", nsID, apiKey) + _, _ = db.Query(internalCtx, "INSERT OR IGNORE INTO namespace_ownership(namespace_id, owner_type, owner_id) VALUES (?, 'wallet', ?)", nsID, req.Wallet) + + writeJSON(w, http.StatusOK, map[string]any{ + "api_key": apiKey, + "namespace": ns, + "wallet": strings.ToLower(strings.TrimPrefix(strings.TrimPrefix(req.Wallet, "0x"), "0X")), + "created": time.Now().Format(time.RFC3339), + }) +} + // base58Decode decodes a base58-encoded string (Bitcoin alphabet) // Used for decoding Solana public keys (base58-encoded 32-byte ed25519 public keys) func base58Decode(encoded string) ([]byte, error) { diff --git a/pkg/gateway/middleware.go b/pkg/gateway/middleware.go index d92f9ac..198a831 100644 --- a/pkg/gateway/middleware.go +++ b/pkg/gateway/middleware.go @@ -179,7 +179,7 @@ func extractAPIKey(r *http.Request) string { // isPublicPath returns true for routes that should be accessible without API key auth func isPublicPath(p string) bool { switch p { - case "/health", "/v1/health", "/status", "/v1/status", "/v1/auth/jwks", "/.well-known/jwks.json", "/v1/version", "/v1/auth/login", "/v1/auth/challenge", "/v1/auth/verify", "/v1/auth/register", "/v1/auth/refresh", "/v1/auth/logout", "/v1/auth/api-key": + case "/health", "/v1/health", "/status", "/v1/status", "/v1/auth/jwks", "/.well-known/jwks.json", "/v1/version", "/v1/auth/login", "/v1/auth/challenge", "/v1/auth/verify", "/v1/auth/register", "/v1/auth/refresh", "/v1/auth/logout", "/v1/auth/api-key", "/v1/auth/simple-key": return true default: return false diff --git a/pkg/gateway/routes.go b/pkg/gateway/routes.go index 531bf24..3037b4d 100644 --- a/pkg/gateway/routes.go +++ b/pkg/gateway/routes.go @@ -22,6 +22,7 @@ func (g *Gateway) Routes() http.Handler { // New: issue JWT from API key; new: create or return API key for a wallet after verification mux.HandleFunc("/v1/auth/token", g.apiKeyToJWTHandler) mux.HandleFunc("/v1/auth/api-key", g.issueAPIKeyHandler) + mux.HandleFunc("/v1/auth/simple-key", g.simpleAPIKeyHandler) mux.HandleFunc("/v1/auth/register", g.registerHandler) mux.HandleFunc("/v1/auth/refresh", g.refreshHandler) mux.HandleFunc("/v1/auth/logout", g.logoutHandler) diff --git a/pkg/ipfs/client.go b/pkg/ipfs/client.go index ff41c42..adbef91 100644 --- a/pkg/ipfs/client.go +++ b/pkg/ipfs/client.go @@ -130,14 +130,23 @@ func (c *Client) GetPeerCount(ctx context.Context) (int, error) { return 0, fmt.Errorf("peers request failed with status: %d", resp.StatusCode) } - var peers []struct { - ID string `json:"id"` - } - if err := json.NewDecoder(resp.Body).Decode(&peers); err != nil { - return 0, fmt.Errorf("failed to decode peers response: %w", err) + // The /peers endpoint returns NDJSON (newline-delimited JSON), not a JSON array + // We need to stream-read each peer object + dec := json.NewDecoder(resp.Body) + peerCount := 0 + for { + var peer map[string]interface{} + err := dec.Decode(&peer) + if err != nil { + if errors.Is(err, io.EOF) { + break + } + return 0, fmt.Errorf("failed to decode peers response: %w", err) + } + peerCount++ } - return len(peers), nil + return peerCount, nil } // Add adds content to IPFS and returns the CID diff --git a/pkg/ipfs/cluster.go b/pkg/ipfs/cluster.go index 0ab5e58..dd19133 100644 --- a/pkg/ipfs/cluster.go +++ b/pkg/ipfs/cluster.go @@ -1,6 +1,7 @@ package ipfs import ( + "bytes" "crypto/rand" "encoding/hex" "encoding/json" @@ -533,19 +534,22 @@ func parseIPFSPort(apiURL string) (int, error) { func getBootstrapPeerID(apiURL string) (string, error) { // Simple HTTP client to query /peers endpoint client := &standardHTTPClient{} - peersResp, err := client.Get(fmt.Sprintf("%s/peers", apiURL)) + resp, err := client.Get(fmt.Sprintf("%s/peers", apiURL)) if err != nil { return "", err } - var peersData struct { + // The /peers endpoint returns NDJSON (newline-delimited JSON) + // We need to read the first peer object to get the bootstrap peer ID + dec := json.NewDecoder(bytes.NewReader(resp)) + var firstPeer struct { ID string `json:"id"` } - if err := json.Unmarshal(peersResp, &peersData); err != nil { - return "", err + if err := dec.Decode(&firstPeer); err != nil { + return "", fmt.Errorf("failed to decode first peer: %w", err) } - return peersData.ID, nil + return firstPeer.ID, nil } // loadOrGenerateClusterSecret loads cluster secret or generates a new one diff --git a/pkg/rqlite/rqlite.go b/pkg/rqlite/rqlite.go index dc84881..a2cb00c 100644 --- a/pkg/rqlite/rqlite.go +++ b/pkg/rqlite/rqlite.go @@ -2,8 +2,10 @@ package rqlite import ( "context" + "encoding/json" "errors" "fmt" + "io" "net/http" "os" "os/exec" @@ -202,6 +204,7 @@ func (r *RQLiteManager) launchProcess(ctx context.Context, rqliteDataDir string) } // waitForReadyAndConnect waits for RQLite to be ready and establishes connection +// For joining nodes, retries if gorqlite.Open fails with "store is not open" error func (r *RQLiteManager) waitForReadyAndConnect(ctx context.Context) error { // Wait for RQLite to be ready if err := r.waitForReady(ctx); err != nil { @@ -211,15 +214,55 @@ func (r *RQLiteManager) waitForReadyAndConnect(ctx context.Context) error { return fmt.Errorf("RQLite failed to become ready: %w", err) } - // Create connection - conn, err := gorqlite.Open(fmt.Sprintf("http://localhost:%d", r.config.RQLitePort)) - if err != nil { + // For joining nodes, retry gorqlite.Open if store is not yet open + // This handles recovery scenarios where the store opens after HTTP is responsive + var conn *gorqlite.Connection + var err error + maxConnectAttempts := 10 + connectBackoff := 500 * time.Millisecond + + for attempt := 0; attempt < maxConnectAttempts; attempt++ { + // Create connection + conn, err = gorqlite.Open(fmt.Sprintf("http://localhost:%d", r.config.RQLitePort)) + if err == nil { + // Success + r.connection = conn + r.logger.Debug("Successfully connected to RQLite", zap.Int("attempt", attempt+1)) + break + } + + // Check if error is "store is not open" (recovery scenario) + if strings.Contains(err.Error(), "store is not open") { + if attempt < maxConnectAttempts-1 { + // Only retry for joining nodes; bootstrap nodes should fail fast + if r.config.RQLiteJoinAddress != "" { + if attempt%3 == 0 { + r.logger.Debug("RQLite store not yet accessible for connection, retrying...", + zap.Int("attempt", attempt+1), zap.Error(err)) + } + time.Sleep(connectBackoff) + connectBackoff = time.Duration(float64(connectBackoff) * 1.5) + if connectBackoff > 5*time.Second { + connectBackoff = 5 * time.Second + } + continue + } + } + } + + // For any other error or final attempt, fail if r.cmd != nil && r.cmd.Process != nil { _ = r.cmd.Process.Kill() } return fmt.Errorf("failed to connect to RQLite: %w", err) } - r.connection = conn + + if conn == nil { + if r.cmd != nil && r.cmd.Process != nil { + _ = r.cmd.Process.Kill() + } + return fmt.Errorf("failed to establish RQLite connection after %d attempts", maxConnectAttempts) + } // Sanity check: verify rqlite's node ID matches our configured raft address if err := r.validateNodeID(); err != nil { @@ -336,12 +379,24 @@ func (r *RQLiteManager) hasExistingState(rqliteDataDir string) bool { } // waitForReady waits for RQLite to be ready to accept connections +// It checks for HTTP 200 + valid raft state (leader/follower) +// The store may not be fully open initially during recovery, but connection retries will handle it +// For joining nodes in recovery, this may take longer (up to 3 minutes) func (r *RQLiteManager) waitForReady(ctx context.Context) error { url := fmt.Sprintf("http://localhost:%d/status", r.config.RQLitePort) client := &http.Client{Timeout: 2 * time.Second} - // Give joining nodes more time (120 seconds vs 30) - maxAttempts := 30 + // Determine timeout based on whether this is a joining node + // Joining nodes in recovery may take longer to open the store + var maxAttempts int + if r.config.RQLiteJoinAddress != "" { + // Joining node: allow up to 180 seconds (3 minutes) for recovery + maxAttempts = 180 + } else { + // Bootstrap node: allow 30 seconds + maxAttempts = 30 + } + for i := 0; i < maxAttempts; i++ { select { case <-ctx.Done(): @@ -350,11 +405,41 @@ func (r *RQLiteManager) waitForReady(ctx context.Context) error { } resp, err := client.Get(url) - if err == nil { + if err == nil && resp.StatusCode == http.StatusOK { + // Parse the response to check for valid raft state + body, err := io.ReadAll(resp.Body) resp.Body.Close() - if resp.StatusCode == http.StatusOK { - return nil + if err == nil { + var statusResp map[string]interface{} + if err := json.Unmarshal(body, &statusResp); err == nil { + // Check for valid raft state (leader or follower) + // If raft is established, we consider the node ready even if store.open is false + // The store will eventually open during recovery, and connection retries will handle it + if raft, ok := statusResp["raft"].(map[string]interface{}); ok { + state, ok := raft["state"].(string) + if ok && (state == "leader" || state == "follower") { + r.logger.Debug("RQLite raft ready", zap.String("state", state), zap.Int("attempt", i+1)) + return nil + } + // Raft not yet ready (likely in candidate state) + if i%10 == 0 { + r.logger.Debug("RQLite raft not yet ready", zap.String("state", state), zap.Int("attempt", i+1)) + } + } else { + // If no raft field, fall back to treating HTTP 200 as ready + // (for backwards compatibility with older RQLite versions) + r.logger.Debug("RQLite HTTP responsive (no raft field)", zap.Int("attempt", i+1)) + return nil + } + } else { + resp.Body.Close() + } } + } else if err != nil && i%20 == 0 { + // Log connection errors only periodically (every ~20s) + r.logger.Debug("RQLite not yet reachable", zap.Int("attempt", i+1), zap.Error(err)) + } else if resp != nil { + resp.Body.Close() } time.Sleep(1 * time.Second) diff --git a/scripts/install-debros-network.sh b/scripts/install-debros-network.sh index 654edac..fcba825 100755 --- a/scripts/install-debros-network.sh +++ b/scripts/install-debros-network.sh @@ -1,7 +1,7 @@ #!/bin/bash # DeBros Network Installation Script -# Downloads network-cli from GitHub releases and runs the new 'network-cli prod install' flow +# Downloads dbn from GitHub releases and runs the new 'dbn prod install' flow # # Supported: Ubuntu 20.04+, Debian 11+ # @@ -124,18 +124,18 @@ download_and_install_cli() { BINARY_NAME="network-cli_${LATEST_RELEASE#v}_linux_${GITHUB_ARCH}" DOWNLOAD_URL="$GITHUB_REPO/releases/download/$LATEST_RELEASE/$BINARY_NAME" - log "Downloading network-cli from GitHub releases..." - if ! curl -fsSL -o /tmp/network-cli "https://github.com/$DOWNLOAD_URL"; then - error "Failed to download network-cli" + log "Downloading dbn from GitHub releases..." + if ! curl -fsSL -o /tmp/dbn "https://github.com/$DOWNLOAD_URL"; then + error "Failed to download dbn" exit 1 fi - chmod +x /tmp/network-cli + chmod +x /tmp/dbn - log "Installing network-cli to $INSTALL_DIR..." - mv /tmp/network-cli "$INSTALL_DIR/network-cli" + log "Installing dbn to $INSTALL_DIR..." + mv /tmp/dbn "$INSTALL_DIR/dbn" - success "network-cli installed successfully" + success "dbn installed successfully" } # Main flow @@ -157,14 +157,14 @@ echo "" echo -e "${CYAN}Next, run the production setup:${NOCOLOR}" echo "" echo "Bootstrap node (first node):" -echo -e " ${BLUE}sudo network-cli prod install --bootstrap${NOCOLOR}" +echo -e " ${BLUE}sudo dbn prod install --bootstrap${NOCOLOR}" echo "" echo "Secondary node (join existing cluster):" -echo -e " ${BLUE}sudo network-cli prod install --vps-ip --peers ${NOCOLOR}" +echo -e " ${BLUE}sudo dbn prod install --vps-ip --peers ${NOCOLOR}" echo "" echo "With HTTPS/domain:" -echo -e " ${BLUE}sudo network-cli prod install --bootstrap --domain example.com${NOCOLOR}" +echo -e " ${BLUE}sudo dbn prod install --bootstrap --domain example.com${NOCOLOR}" echo "" echo "For more help:" -echo -e " ${BLUE}network-cli prod --help${NOCOLOR}" +echo -e " ${BLUE}dbn prod --help${NOCOLOR}" echo ""