diff --git a/.githooks/pre-commit b/.githooks/pre-commit index 74f323d..d9e2bad 100644 --- a/.githooks/pre-commit +++ b/.githooks/pre-commit @@ -30,6 +30,15 @@ if [ -z "$OTHER_FILES" ]; then exit 0 fi +# Check for skip flag +# To skip changelog generation, set SKIP_CHANGELOG=1 before committing: +# SKIP_CHANGELOG=1 git commit -m "your message" +# SKIP_CHANGELOG=1 git commit +if [ "$SKIP_CHANGELOG" = "1" ] || [ "$SKIP_CHANGELOG" = "true" ]; then + echo -e "${YELLOW}Skipping changelog update (SKIP_CHANGELOG is set)${NOCOLOR}" + exit 0 +fi + # Update changelog before commit if [ -f "$CHANGELOG_SCRIPT" ]; then echo -e "\n${CYAN}Updating changelog...${NOCOLOR}" diff --git a/CHANGELOG.md b/CHANGELOG.md index 20b5347..8502398 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,14 +13,17 @@ The format is based on [Keep a Changelog][keepachangelog] and adheres to [Semant ### Deprecated ### Fixed + ## [0.56.0] - 2025-11-05 ### Added + - Added IPFS storage endpoints to the Gateway for content upload, pinning, status, retrieval, and unpinning. - Introduced `StorageClient` interface and implementation in the Go client library for interacting with the new IPFS storage endpoints. - Added support for automatically starting IPFS daemon, IPFS Cluster daemon, and Olric cache server in the `dev` environment setup. ### Changed + - Updated Gateway configuration to include settings for IPFS Cluster API URL, IPFS API URL, timeout, and replication factor. - Refactored Olric configuration generation to use a simpler, local-environment focused setup. - Improved IPFS content retrieval (`Get`) to fall back to the IPFS Gateway (port 8080) if the IPFS API (port 5001) returns a 404. @@ -30,34 +33,18 @@ The format is based on [Keep a Changelog][keepachangelog] and adheres to [Semant ### Removed ### Fixed -\n -## [0.55.0] - 2025-11-05 -### Added -- Added IPFS storage endpoints to the Gateway for content upload, pinning, status, retrieval, and unpinning. -- Introduced `StorageClient` interface and implementation in the Go client library for interacting with the new IPFS storage endpoints. -- Added support for automatically starting IPFS daemon, IPFS Cluster daemon, and Olric cache server in the `dev` environment setup. - -### Changed -- Updated Gateway configuration to include settings for IPFS Cluster API URL, IPFS API URL, timeout, and replication factor. -- Refactored Olric configuration generation to use a simpler, local-environment focused setup. -- Improved `dev` environment logging to include logs from IPFS and Olric services when running. - -### Deprecated - -### Removed - -### Fixed -\n ## [0.54.0] - 2025-11-03 ### Added + - Integrated Olric distributed cache for high-speed key-value storage and caching. - Added new HTTP Gateway endpoints for cache operations (GET, PUT, DELETE, SCAN) via `/v1/cache/`. - Added `olric_servers` and `olric_timeout` configuration options to the Gateway. - Updated the automated installation script (`install-debros-network.sh`) to include Olric installation, configuration, and firewall rules (ports 3320, 3322). ### Changed + - Refactored README for better clarity and organization, focusing on quick start and core features. ### Deprecated @@ -65,12 +52,17 @@ The format is based on [Keep a Changelog][keepachangelog] and adheres to [Semant ### Removed ### Fixed + \n + ## [0.53.18] - 2025-11-03 ### Added + \n + ### Changed + - Increased the connection timeout during peer discovery from 15 seconds to 20 seconds to improve connection reliability. - Removed unnecessary debug logging related to filtering out ephemeral port addresses during peer exchange. @@ -79,13 +71,17 @@ The format is based on [Keep a Changelog][keepachangelog] and adheres to [Semant ### Removed ### Fixed + \n + ## [0.53.17] - 2025-11-03 ### Added + - Added a new Git `pre-commit` hook to automatically update the changelog and version before committing, ensuring version consistency. ### Changed + - Refactored the `update_changelog.sh` script to support different execution contexts (pre-commit vs. pre-push), allowing it to analyze only staged changes during commit. - The Git `pre-push` hook was simplified by removing the changelog update logic, which is now handled by the `pre-commit` hook. @@ -94,12 +90,17 @@ The format is based on [Keep a Changelog][keepachangelog] and adheres to [Semant ### Removed ### Fixed + \n + ## [0.53.16] - 2025-11-03 ### Added + \n + ### Changed + - Improved the changelog generation script to prevent infinite loops when the only unpushed commit is a previous changelog update. ### Deprecated @@ -107,12 +108,17 @@ The format is based on [Keep a Changelog][keepachangelog] and adheres to [Semant ### Removed ### Fixed + \n + ## [0.53.15] - 2025-11-03 ### Added + \n + ### Changed + - Improved the pre-push git hook to automatically commit updated changelog and Makefile after generation. - Updated the changelog generation script to load the OpenRouter API key from the .env file or environment variables for better security. - Modified the pre-push hook to read user confirmation from /dev/tty for better compatibility. @@ -124,12 +130,17 @@ The format is based on [Keep a Changelog][keepachangelog] and adheres to [Semant ### Removed ### Fixed + \n + ## [0.53.15] - 2025-11-03 ### Added + \n + ### Changed + - Improved the pre-push git hook to automatically commit updated changelog and Makefile after generation. - Updated the changelog generation script to load the OpenRouter API key from the .env file or environment variables for better security. - Modified the pre-push hook to read user confirmation from /dev/tty for better compatibility. @@ -141,14 +152,18 @@ The format is based on [Keep a Changelog][keepachangelog] and adheres to [Semant ### Removed ### Fixed + \n + ## [0.53.14] - 2025-11-03 ### Added + - Added a new `install-hooks` target to the Makefile to easily set up git hooks. - Added a script (`scripts/install-hooks.sh`) to copy git hooks from `.githooks` to `.git/hooks`. ### Changed + - Improved the pre-push git hook to automatically commit the updated `CHANGELOG.md` and `Makefile` after generating the changelog. - Updated the changelog generation script (`scripts/update_changelog.sh`) to load the OpenRouter API key from the `.env` file or environment variables, improving security and configuration. - Modified the pre-push hook to read user confirmation from `/dev/tty` for better compatibility in various terminal environments. @@ -160,14 +175,18 @@ The format is based on [Keep a Changelog][keepachangelog] and adheres to [Semant ### Removed ### Fixed + \n + ## [0.53.14] - 2025-11-03 ### Added + - Added a new `install-hooks` target to the Makefile to easily set up git hooks. - Added a script (`scripts/install-hooks.sh`) to copy git hooks from `.githooks` to `.git/hooks`. ### Changed + - Improved the pre-push git hook to automatically commit the updated `CHANGELOG.md` and `Makefile` after generating the changelog. - Updated the changelog generation script (`scripts/update_changelog.sh`) to load the OpenRouter API key from the `.env` file or environment variables, improving security and configuration. - Modified the pre-push hook to read user confirmation from `/dev/tty` for better compatibility in various terminal environments. @@ -177,6 +196,7 @@ The format is based on [Keep a Changelog][keepachangelog] and adheres to [Semant ### Removed ### Fixed + \n ## [0.53.8] - 2025-10-31 diff --git a/Makefile b/Makefile index bc9bbb2..712948d 100644 --- a/Makefile +++ b/Makefile @@ -19,7 +19,7 @@ test-e2e: # Network - Distributed P2P Database System # Makefile for development and build tasks -.PHONY: build clean test run-node run-node2 run-node3 run-example deps tidy fmt vet lint clear-ports install-hooks +.PHONY: build clean test run-node run-node2 run-node3 run-example deps tidy fmt vet lint clear-ports install-hooks kill VERSION := 0.56.0 COMMIT ?= $(shell git rev-parse --short HEAD 2>/dev/null || echo unknown) @@ -109,6 +109,102 @@ dev: build echo " ⚠️ systemctl not found - skipping Anon"; \ fi; \ fi + @echo "Initializing IPFS and Cluster for all nodes..." + @if command -v ipfs >/dev/null 2>&1 && command -v ipfs-cluster-service >/dev/null 2>&1; then \ + CLUSTER_SECRET=$$HOME/.debros/cluster-secret; \ + if [ ! -f $$CLUSTER_SECRET ]; then \ + echo " Generating shared cluster secret..."; \ + ipfs-cluster-service --version >/dev/null 2>&1 && openssl rand -hex 32 > $$CLUSTER_SECRET || echo "0000000000000000000000000000000000000000000000000000000000000000" > $$CLUSTER_SECRET; \ + fi; \ + SECRET=$$(cat $$CLUSTER_SECRET); \ + echo " Setting up bootstrap node (IPFS: 5001, Cluster: 9094)..."; \ + if [ ! -d $$HOME/.debros/bootstrap/ipfs/repo ]; then \ + echo " Initializing IPFS..."; \ + mkdir -p $$HOME/.debros/bootstrap/ipfs; \ + IPFS_PATH=$$HOME/.debros/bootstrap/ipfs/repo ipfs init --profile=server 2>&1 | grep -v "generating" | grep -v "peer identity" || true; \ + IPFS_PATH=$$HOME/.debros/bootstrap/ipfs/repo ipfs config --json Addresses.API '["/ip4/127.0.0.1/tcp/5001"]' 2>&1 | grep -v "generating" || true; \ + IPFS_PATH=$$HOME/.debros/bootstrap/ipfs/repo ipfs config --json Addresses.Gateway '["/ip4/127.0.0.1/tcp/8080"]' 2>&1 | grep -v "generating" || true; \ + IPFS_PATH=$$HOME/.debros/bootstrap/ipfs/repo ipfs config --json Addresses.Swarm '["/ip4/0.0.0.0/tcp/4001","/ip6/::/tcp/4001"]' 2>&1 | grep -v "generating" || true; \ + fi; \ + echo " Initializing IPFS Cluster..."; \ + mkdir -p $$HOME/.debros/bootstrap/ipfs-cluster; \ + env IPFS_CLUSTER_PATH=$$HOME/.debros/bootstrap/ipfs-cluster ipfs-cluster-service init --force >/dev/null 2>&1 || true; \ + jq '.cluster.peername = "bootstrap" | .cluster.secret = "'$$SECRET'" | .cluster.listen_multiaddress = ["/ip4/0.0.0.0/tcp/9096"] | .consensus.crdt.cluster_name = "debros-cluster" | .consensus.crdt.trusted_peers = ["*"] | .api.restapi.http_listen_multiaddress = "/ip4/0.0.0.0/tcp/9094" | .api.ipfsproxy.listen_multiaddress = "/ip4/127.0.0.1/tcp/9095" | .api.pinsvcapi.http_listen_multiaddress = "/ip4/127.0.0.1/tcp/9097" | .ipfs_connector.ipfshttp.node_multiaddress = "/ip4/127.0.0.1/tcp/5001"' $$HOME/.debros/bootstrap/ipfs-cluster/service.json > $$HOME/.debros/bootstrap/ipfs-cluster/service.json.tmp && mv $$HOME/.debros/bootstrap/ipfs-cluster/service.json.tmp $$HOME/.debros/bootstrap/ipfs-cluster/service.json; \ + echo " Setting up node2 (IPFS: 5002, Cluster: 9104)..."; \ + if [ ! -d $$HOME/.debros/node2/ipfs/repo ]; then \ + echo " Initializing IPFS..."; \ + mkdir -p $$HOME/.debros/node2/ipfs; \ + IPFS_PATH=$$HOME/.debros/node2/ipfs/repo ipfs init --profile=server 2>&1 | grep -v "generating" | grep -v "peer identity" || true; \ + IPFS_PATH=$$HOME/.debros/node2/ipfs/repo ipfs config --json Addresses.API '["/ip4/127.0.0.1/tcp/5002"]' 2>&1 | grep -v "generating" || true; \ + IPFS_PATH=$$HOME/.debros/node2/ipfs/repo ipfs config --json Addresses.Gateway '["/ip4/127.0.0.1/tcp/8081"]' 2>&1 | grep -v "generating" || true; \ + IPFS_PATH=$$HOME/.debros/node2/ipfs/repo ipfs config --json Addresses.Swarm '["/ip4/0.0.0.0/tcp/4002","/ip6/::/tcp/4002"]' 2>&1 | grep -v "generating" || true; \ + fi; \ + echo " Initializing IPFS Cluster..."; \ + mkdir -p $$HOME/.debros/node2/ipfs-cluster; \ + env IPFS_CLUSTER_PATH=$$HOME/.debros/node2/ipfs-cluster ipfs-cluster-service init --force >/dev/null 2>&1 || true; \ + jq '.cluster.peername = "node2" | .cluster.secret = "'$$SECRET'" | .cluster.listen_multiaddress = ["/ip4/0.0.0.0/tcp/9106"] | .consensus.crdt.cluster_name = "debros-cluster" | .consensus.crdt.trusted_peers = ["*"] | .api.restapi.http_listen_multiaddress = "/ip4/0.0.0.0/tcp/9104" | .api.ipfsproxy.listen_multiaddress = "/ip4/127.0.0.1/tcp/9105" | .api.pinsvcapi.http_listen_multiaddress = "/ip4/127.0.0.1/tcp/9107" | .ipfs_connector.ipfshttp.node_multiaddress = "/ip4/127.0.0.1/tcp/5002"' $$HOME/.debros/node2/ipfs-cluster/service.json > $$HOME/.debros/node2/ipfs-cluster/service.json.tmp && mv $$HOME/.debros/node2/ipfs-cluster/service.json.tmp $$HOME/.debros/node2/ipfs-cluster/service.json; \ + echo " Setting up node3 (IPFS: 5003, Cluster: 9114)..."; \ + if [ ! -d $$HOME/.debros/node3/ipfs/repo ]; then \ + echo " Initializing IPFS..."; \ + mkdir -p $$HOME/.debros/node3/ipfs; \ + IPFS_PATH=$$HOME/.debros/node3/ipfs/repo ipfs init --profile=server 2>&1 | grep -v "generating" | grep -v "peer identity" || true; \ + IPFS_PATH=$$HOME/.debros/node3/ipfs/repo ipfs config --json Addresses.API '["/ip4/127.0.0.1/tcp/5003"]' 2>&1 | grep -v "generating" || true; \ + IPFS_PATH=$$HOME/.debros/node3/ipfs/repo ipfs config --json Addresses.Gateway '["/ip4/127.0.0.1/tcp/8082"]' 2>&1 | grep -v "generating" || true; \ + IPFS_PATH=$$HOME/.debros/node3/ipfs/repo ipfs config --json Addresses.Swarm '["/ip4/0.0.0.0/tcp/4003","/ip6/::/tcp/4003"]' 2>&1 | grep -v "generating" || true; \ + fi; \ + echo " Initializing IPFS Cluster..."; \ + mkdir -p $$HOME/.debros/node3/ipfs-cluster; \ + env IPFS_CLUSTER_PATH=$$HOME/.debros/node3/ipfs-cluster ipfs-cluster-service init --force >/dev/null 2>&1 || true; \ + jq '.cluster.peername = "node3" | .cluster.secret = "'$$SECRET'" | .cluster.listen_multiaddress = ["/ip4/0.0.0.0/tcp/9116"] | .consensus.crdt.cluster_name = "debros-cluster" | .consensus.crdt.trusted_peers = ["*"] | .api.restapi.http_listen_multiaddress = "/ip4/0.0.0.0/tcp/9114" | .api.ipfsproxy.listen_multiaddress = "/ip4/127.0.0.1/tcp/9115" | .api.pinsvcapi.http_listen_multiaddress = "/ip4/127.0.0.1/tcp/9117" | .ipfs_connector.ipfshttp.node_multiaddress = "/ip4/127.0.0.1/tcp/5003"' $$HOME/.debros/node3/ipfs-cluster/service.json > $$HOME/.debros/node3/ipfs-cluster/service.json.tmp && mv $$HOME/.debros/node3/ipfs-cluster/service.json.tmp $$HOME/.debros/node3/ipfs-cluster/service.json; \ + echo "Starting IPFS daemons..."; \ + if [ ! -f .dev/pids/ipfs-bootstrap.pid ] || ! kill -0 $$(cat .dev/pids/ipfs-bootstrap.pid) 2>/dev/null; then \ + IPFS_PATH=$$HOME/.debros/bootstrap/ipfs/repo nohup ipfs daemon --enable-pubsub-experiment > $$HOME/.debros/logs/ipfs-bootstrap.log 2>&1 & echo $$! > .dev/pids/ipfs-bootstrap.pid; \ + echo " Bootstrap IPFS started (PID: $$(cat .dev/pids/ipfs-bootstrap.pid), API: 5001)"; \ + sleep 3; \ + else \ + echo " ✓ Bootstrap IPFS already running"; \ + fi; \ + if [ ! -f .dev/pids/ipfs-node2.pid ] || ! kill -0 $$(cat .dev/pids/ipfs-node2.pid) 2>/dev/null; then \ + IPFS_PATH=$$HOME/.debros/node2/ipfs/repo nohup ipfs daemon --enable-pubsub-experiment > $$HOME/.debros/logs/ipfs-node2.log 2>&1 & echo $$! > .dev/pids/ipfs-node2.pid; \ + echo " Node2 IPFS started (PID: $$(cat .dev/pids/ipfs-node2.pid), API: 5002)"; \ + sleep 3; \ + else \ + echo " ✓ Node2 IPFS already running"; \ + fi; \ + if [ ! -f .dev/pids/ipfs-node3.pid ] || ! kill -0 $$(cat .dev/pids/ipfs-node3.pid) 2>/dev/null; then \ + IPFS_PATH=$$HOME/.debros/node3/ipfs/repo nohup ipfs daemon --enable-pubsub-experiment > $$HOME/.debros/logs/ipfs-node3.log 2>&1 & echo $$! > .dev/pids/ipfs-node3.pid; \ + echo " Node3 IPFS started (PID: $$(cat .dev/pids/ipfs-node3.pid), API: 5003)"; \ + sleep 3; \ + else \ + echo " ✓ Node3 IPFS already running"; \ + fi; \ + \ + echo "Starting IPFS Cluster peers..."; \ + if [ ! -f .dev/pids/ipfs-cluster-bootstrap.pid ] || ! kill -0 $$(cat .dev/pids/ipfs-cluster-bootstrap.pid) 2>/dev/null; then \ + env IPFS_CLUSTER_PATH=$$HOME/.debros/bootstrap/ipfs-cluster nohup ipfs-cluster-service daemon > $$HOME/.debros/logs/ipfs-cluster-bootstrap.log 2>&1 & echo $$! > .dev/pids/ipfs-cluster-bootstrap.pid; \ + echo " Bootstrap Cluster started (PID: $$(cat .dev/pids/ipfs-cluster-bootstrap.pid), API: 9094)"; \ + sleep 3; \ + else \ + echo " ✓ Bootstrap Cluster already running"; \ + fi; \ + if [ ! -f .dev/pids/ipfs-cluster-node2.pid ] || ! kill -0 $$(cat .dev/pids/ipfs-cluster-node2.pid) 2>/dev/null; then \ + env IPFS_CLUSTER_PATH=$$HOME/.debros/node2/ipfs-cluster nohup ipfs-cluster-service daemon > $$HOME/.debros/logs/ipfs-cluster-node2.log 2>&1 & echo $$! > .dev/pids/ipfs-cluster-node2.pid; \ + echo " Node2 Cluster started (PID: $$(cat .dev/pids/ipfs-cluster-node2.pid), API: 9104)"; \ + sleep 3; \ + else \ + echo " ✓ Node2 Cluster already running"; \ + fi; \ + if [ ! -f .dev/pids/ipfs-cluster-node3.pid ] || ! kill -0 $$(cat .dev/pids/ipfs-cluster-node3.pid) 2>/dev/null; then \ + env IPFS_CLUSTER_PATH=$$HOME/.debros/node3/ipfs-cluster nohup ipfs-cluster-service daemon > $$HOME/.debros/logs/ipfs-cluster-node3.log 2>&1 & echo $$! > .dev/pids/ipfs-cluster-node3.pid; \ + echo " Node3 Cluster started (PID: $$(cat .dev/pids/ipfs-cluster-node3.pid), API: 9114)"; \ + sleep 3; \ + else \ + echo " ✓ Node3 Cluster already running"; \ + fi; \ + else \ + echo " ⚠️ ipfs or ipfs-cluster-service not found - skipping IPFS setup"; \ + echo " Install with: https://docs.ipfs.tech/install/ and https://ipfscluster.io/documentation/guides/install/"; \ + fi @sleep 2 @echo "Starting bootstrap node..." @nohup ./bin/node --config bootstrap.yaml > $$HOME/.debros/logs/bootstrap.log 2>&1 & echo $$! > .dev/pids/bootstrap.pid @@ -119,40 +215,6 @@ dev: build @echo "Starting node3..." @nohup ./bin/node --config node3.yaml > $$HOME/.debros/logs/node3.log 2>&1 & echo $$! > .dev/pids/node3.pid @sleep 1 - @echo "Starting IPFS daemon..." - @if command -v ipfs >/dev/null 2>&1; then \ - if [ ! -d $$HOME/.debros/ipfs ]; then \ - echo " Initializing IPFS repository..."; \ - IPFS_PATH=$$HOME/.debros/ipfs ipfs init 2>&1 | grep -v "generating" | grep -v "peer identity" || true; \ - fi; \ - if ! pgrep -f "ipfs daemon" >/dev/null 2>&1; then \ - IPFS_PATH=$$HOME/.debros/ipfs nohup ipfs daemon > $$HOME/.debros/logs/ipfs.log 2>&1 & echo $$! > .dev/pids/ipfs.pid; \ - echo " IPFS daemon started (PID: $$(cat .dev/pids/ipfs.pid))"; \ - sleep 5; \ - else \ - echo " ✓ IPFS daemon already running"; \ - fi; \ - else \ - echo " ⚠️ ipfs command not found - skipping IPFS (storage endpoints will be disabled)"; \ - echo " Install with: https://docs.ipfs.tech/install/"; \ - fi - @echo "Starting IPFS Cluster daemon..." - @if command -v ipfs-cluster-service >/dev/null 2>&1; then \ - if [ ! -d $$HOME/.debros/ipfs-cluster ]; then \ - echo " Initializing IPFS Cluster..."; \ - CLUSTER_PATH=$$HOME/.debros/ipfs-cluster ipfs-cluster-service init --force 2>&1 | grep -v "peer identity" || true; \ - fi; \ - if ! pgrep -f "ipfs-cluster-service" >/dev/null 2>&1; then \ - CLUSTER_PATH=$$HOME/.debros/ipfs-cluster nohup ipfs-cluster-service daemon > $$HOME/.debros/logs/ipfs-cluster.log 2>&1 & echo $$! > .dev/pids/ipfs-cluster.pid; \ - echo " IPFS Cluster daemon started (PID: $$(cat .dev/pids/ipfs-cluster.pid))"; \ - sleep 5; \ - else \ - echo " ✓ IPFS Cluster daemon already running"; \ - fi; \ - else \ - echo " ⚠️ ipfs-cluster-service command not found - skipping IPFS Cluster (storage endpoints will be disabled)"; \ - echo " Install with: https://ipfscluster.io/documentation/guides/install/"; \ - fi @echo "Starting Olric cache server..." @if command -v olric-server >/dev/null 2>&1; then \ if [ ! -f $$HOME/.debros/olric-config.yaml ]; then \ @@ -182,11 +244,23 @@ dev: build @if [ -f .dev/pids/anon.pid ]; then \ echo " Anon: PID=$$(cat .dev/pids/anon.pid) (SOCKS: 9050)"; \ fi - @if [ -f .dev/pids/ipfs.pid ]; then \ - echo " IPFS: PID=$$(cat .dev/pids/ipfs.pid) (API: 5001)"; \ + @if [ -f .dev/pids/ipfs-bootstrap.pid ]; then \ + echo " Bootstrap IPFS: PID=$$(cat .dev/pids/ipfs-bootstrap.pid) (API: 5001)"; \ fi - @if [ -f .dev/pids/ipfs-cluster.pid ]; then \ - echo " IPFS Cluster: PID=$$(cat .dev/pids/ipfs-cluster.pid) (API: 9094)"; \ + @if [ -f .dev/pids/ipfs-node2.pid ]; then \ + echo " Node2 IPFS: PID=$$(cat .dev/pids/ipfs-node2.pid) (API: 5002)"; \ + fi + @if [ -f .dev/pids/ipfs-node3.pid ]; then \ + echo " Node3 IPFS: PID=$$(cat .dev/pids/ipfs-node3.pid) (API: 5003)"; \ + fi + @if [ -f .dev/pids/ipfs-cluster-bootstrap.pid ]; then \ + echo " Bootstrap Cluster: PID=$$(cat .dev/pids/ipfs-cluster-bootstrap.pid) (API: 9094)"; \ + fi + @if [ -f .dev/pids/ipfs-cluster-node2.pid ]; then \ + echo " Node2 Cluster: PID=$$(cat .dev/pids/ipfs-cluster-node2.pid) (API: 9104)"; \ + fi + @if [ -f .dev/pids/ipfs-cluster-node3.pid ]; then \ + echo " Node3 Cluster: PID=$$(cat .dev/pids/ipfs-cluster-node3.pid) (API: 9114)"; \ fi @if [ -f .dev/pids/olric.pid ]; then \ echo " Olric: PID=$$(cat .dev/pids/olric.pid) (API: 3320)"; \ @@ -198,9 +272,13 @@ dev: build @echo "" @echo "Ports:" @echo " Anon SOCKS: 9050 (proxy endpoint: POST /v1/proxy/anon)" - @if [ -f .dev/pids/ipfs.pid ]; then \ - echo " IPFS API: 5001 (content retrieval)"; \ - echo " IPFS Cluster: 9094 (pin management)"; \ + @if [ -f .dev/pids/ipfs-bootstrap.pid ]; then \ + echo " Bootstrap IPFS API: 5001"; \ + echo " Node2 IPFS API: 5002"; \ + echo " Node3 IPFS API: 5003"; \ + echo " Bootstrap Cluster: 9094 (pin management)"; \ + echo " Node2 Cluster: 9104 (pin management)"; \ + echo " Node3 Cluster: 9114 (pin management)"; \ fi @if [ -f .dev/pids/olric.pid ]; then \ echo " Olric: 3320 (cache API)"; \ @@ -217,15 +295,85 @@ dev: build if [ -f .dev/pids/anon.pid ]; then \ LOGS="$$LOGS $$HOME/.debros/logs/anon.log"; \ fi; \ - if [ -f .dev/pids/ipfs.pid ]; then \ - LOGS="$$LOGS $$HOME/.debros/logs/ipfs.log"; \ + if [ -f .dev/pids/ipfs-bootstrap.pid ]; then \ + LOGS="$$LOGS $$HOME/.debros/logs/ipfs-bootstrap.log $$HOME/.debros/logs/ipfs-node2.log $$HOME/.debros/logs/ipfs-node3.log"; \ fi; \ - if [ -f .dev/pids/ipfs-cluster.pid ]; then \ - LOGS="$$LOGS $$HOME/.debros/logs/ipfs-cluster.log"; \ + if [ -f .dev/pids/ipfs-cluster-bootstrap.pid ]; then \ + LOGS="$$LOGS $$HOME/.debros/logs/ipfs-cluster-bootstrap.log $$HOME/.debros/logs/ipfs-cluster-node2.log $$HOME/.debros/logs/ipfs-cluster-node3.log"; \ + fi; \ + if [ -f .dev/pids/olric.pid ]; then \ + LOGS="$$LOGS $$HOME/.debros/logs/olric.log"; \ fi; \ trap 'echo "Stopping all processes..."; kill $$(cat .dev/pids/*.pid) 2>/dev/null; rm -f .dev/pids/*.pid; exit 0' INT; \ tail -f $$LOGS +# Kill all processes +kill: + @echo "🛑 Stopping all DeBros network services..." + @echo "" + @echo "Stopping DeBros nodes and gateway..." + @if [ -f .dev/pids/gateway.pid ]; then \ + kill -TERM $$(cat .dev/pids/gateway.pid) 2>/dev/null && echo " ✓ Gateway stopped" || echo " ✗ Gateway not running"; \ + rm -f .dev/pids/gateway.pid; \ + fi + @if [ -f .dev/pids/bootstrap.pid ]; then \ + kill -TERM $$(cat .dev/pids/bootstrap.pid) 2>/dev/null && echo " ✓ Bootstrap node stopped" || echo " ✗ Bootstrap not running"; \ + rm -f .dev/pids/bootstrap.pid; \ + fi + @if [ -f .dev/pids/node2.pid ]; then \ + kill -TERM $$(cat .dev/pids/node2.pid) 2>/dev/null && echo " ✓ Node2 stopped" || echo " ✗ Node2 not running"; \ + rm -f .dev/pids/node2.pid; \ + fi + @if [ -f .dev/pids/node3.pid ]; then \ + kill -TERM $$(cat .dev/pids/node3.pid) 2>/dev/null && echo " ✓ Node3 stopped" || echo " ✗ Node3 not running"; \ + rm -f .dev/pids/node3.pid; \ + fi + @echo "" + @echo "Stopping IPFS Cluster peers..." + @if [ -f .dev/pids/ipfs-cluster-bootstrap.pid ]; then \ + kill -TERM $$(cat .dev/pids/ipfs-cluster-bootstrap.pid) 2>/dev/null && echo " ✓ Bootstrap Cluster stopped" || echo " ✗ Bootstrap Cluster not running"; \ + rm -f .dev/pids/ipfs-cluster-bootstrap.pid; \ + fi + @if [ -f .dev/pids/ipfs-cluster-node2.pid ]; then \ + kill -TERM $$(cat .dev/pids/ipfs-cluster-node2.pid) 2>/dev/null && echo " ✓ Node2 Cluster stopped" || echo " ✗ Node2 Cluster not running"; \ + rm -f .dev/pids/ipfs-cluster-node2.pid; \ + fi + @if [ -f .dev/pids/ipfs-cluster-node3.pid ]; then \ + kill -TERM $$(cat .dev/pids/ipfs-cluster-node3.pid) 2>/dev/null && echo " ✓ Node3 Cluster stopped" || echo " ✗ Node3 Cluster not running"; \ + rm -f .dev/pids/ipfs-cluster-node3.pid; \ + fi + @echo "" + @echo "Stopping IPFS daemons..." + @if [ -f .dev/pids/ipfs-bootstrap.pid ]; then \ + kill -TERM $$(cat .dev/pids/ipfs-bootstrap.pid) 2>/dev/null && echo " ✓ Bootstrap IPFS stopped" || echo " ✗ Bootstrap IPFS not running"; \ + rm -f .dev/pids/ipfs-bootstrap.pid; \ + fi + @if [ -f .dev/pids/ipfs-node2.pid ]; then \ + kill -TERM $$(cat .dev/pids/ipfs-node2.pid) 2>/dev/null && echo " ✓ Node2 IPFS stopped" || echo " ✗ Node2 IPFS not running"; \ + rm -f .dev/pids/ipfs-node2.pid; \ + fi + @if [ -f .dev/pids/ipfs-node3.pid ]; then \ + kill -TERM $$(cat .dev/pids/ipfs-node3.pid) 2>/dev/null && echo " ✓ Node3 IPFS stopped" || echo " ✗ Node3 IPFS not running"; \ + rm -f .dev/pids/ipfs-node3.pid; \ + fi + @echo "" + @echo "Stopping Olric cache..." + @if [ -f .dev/pids/olric.pid ]; then \ + kill -TERM $$(cat .dev/pids/olric.pid) 2>/dev/null && echo " ✓ Olric stopped" || echo " ✗ Olric not running"; \ + rm -f .dev/pids/olric.pid; \ + fi + @echo "" + @echo "Stopping Anon proxy..." + @if [ -f .dev/pids/anyone.pid ]; then \ + kill -TERM $$(cat .dev/pids/anyone.pid) 2>/dev/null && echo " ✓ Anon proxy stopped" || echo " ✗ Anon proxy not running"; \ + rm -f .dev/pids/anyone.pid; \ + fi + @echo "" + @echo "Cleaning up any remaining processes on ports..." + @lsof -ti:7001,7002,7003,5001,5002,5003,6001,4001,4002,4003,9050,3320,3322,9094,9095,9096,9097,9104,9105,9106,9107,9114,9115,9116,9117,8080,8081,8082 2>/dev/null | xargs kill -9 2>/dev/null && echo " ✓ Cleaned up remaining port bindings" || echo " ✓ No lingering processes found" + @echo "" + @echo "✅ All services stopped!" + # Help help: @echo "Available targets:" @@ -277,6 +425,7 @@ help: @echo " vet - Vet code" @echo " lint - Lint code (fmt + vet)" @echo " clear-ports - Clear common dev ports" + @echo " kill - Stop all running services (nodes, IPFS, cluster, gateway, olric)" @echo " dev-setup - Setup development environment" @echo " dev-cluster - Show cluster startup commands" @echo " dev - Full development workflow" diff --git a/docs/ipfs-cluster-setup.md b/docs/ipfs-cluster-setup.md new file mode 100644 index 0000000..fa70343 --- /dev/null +++ b/docs/ipfs-cluster-setup.md @@ -0,0 +1,171 @@ +# IPFS Cluster Setup Guide + +This guide explains how IPFS Cluster is configured to run on every DeBros Network node. + +## Overview + +Each DeBros Network node runs its own IPFS Cluster peer, enabling distributed pinning and replication across the network. The cluster uses CRDT consensus for automatic peer discovery. + +## Architecture + +- **IPFS (Kubo)**: Runs on each node, handles content storage and retrieval +- **IPFS Cluster**: Runs on each node, manages pinning and replication +- **Cluster Consensus**: Uses CRDT (instead of Raft) for simpler multi-node setup + +## Automatic Setup + +When you run `network-cli setup`, the following happens automatically: + +1. IPFS (Kubo) and IPFS Cluster are installed +2. IPFS repository is initialized for each node +3. IPFS Cluster service.json config is generated +4. Systemd services are created and started: + - `debros-ipfs` - IPFS daemon + - `debros-ipfs-cluster` - IPFS Cluster service + - `debros-node` - DeBros Network node (depends on cluster) + - `debros-gateway` - HTTP Gateway (depends on node) + +## Configuration + +### Node Configs + +Each node config (`~/.debros/bootstrap.yaml`, `~/.debros/node.yaml`, etc.) includes: + +```yaml +database: + ipfs: + cluster_api_url: "http://localhost:9094" # Local cluster API + api_url: "http://localhost:5001" # Local IPFS API + replication_factor: 3 # Desired replication +``` + +### Cluster Service Config + +Cluster service configs are stored at: + +- Bootstrap: `~/.debros/bootstrap/ipfs-cluster/service.json` +- Nodes: `~/.debros/node/ipfs-cluster/service.json` + +Key settings: + +- **Consensus**: CRDT (automatic peer discovery) +- **API Listen**: `0.0.0.0:9094` (REST API) +- **Cluster Listen**: `0.0.0.0:9096` (peer-to-peer) +- **Secret**: Shared cluster secret stored at `~/.debros/cluster-secret` + +## Verification + +### Check Cluster Peers + +From any node, verify all cluster peers are connected: + +```bash +sudo -u debros ipfs-cluster-ctl --host http://localhost:9094 peers ls +``` + +You should see all cluster peers listed (bootstrap, node1, node2, etc.). + +### Check IPFS Daemon + +Verify IPFS is running: + +```bash +sudo -u debros ipfs daemon --repo-dir=~/.debros/bootstrap/ipfs/repo +# Or for regular nodes: +sudo -u debros ipfs daemon --repo-dir=~/.debros/node/ipfs/repo +``` + +### Check Service Status + +```bash +network-cli service status all +``` + +Should show: + +- `debros-ipfs` - running +- `debros-ipfs-cluster` - running +- `debros-node` - running +- `debros-gateway` - running + +## Troubleshooting + +### Cluster Peers Not Connecting + +If peers aren't discovering each other: + +1. **Check firewall**: Ensure ports 9096 (cluster swarm) and 9094 (cluster API) are open +2. **Verify secret**: All nodes must use the same cluster secret from `~/.debros/cluster-secret` +3. **Check logs**: `journalctl -u debros-ipfs-cluster -f` + +### Not Enough Peers Error + +If you see "not enough peers to allocate CID" errors: + +- The cluster needs at least `replication_factor` peers running +- Check that all nodes have `debros-ipfs-cluster` service running +- Verify with `ipfs-cluster-ctl peers ls` + +### IPFS Not Starting + +If IPFS daemon fails to start: + +1. Check IPFS repo exists: `ls -la ~/.debros/bootstrap/ipfs/repo/` +2. Check permissions: `chown -R debros:debros ~/.debros/bootstrap/ipfs/` +3. Check logs: `journalctl -u debros-ipfs -f` + +## Manual Setup (If Needed) + +If automatic setup didn't work, you can manually initialize: + +### 1. Initialize IPFS + +```bash +sudo -u debros ipfs init --profile=server --repo-dir=~/.debros/bootstrap/ipfs/repo +sudo -u debros ipfs config --json Addresses.API '["/ip4/127.0.0.1/tcp/5001"]' --repo-dir=~/.debros/bootstrap/ipfs/repo +``` + +### 2. Initialize Cluster + +```bash +# Generate or get cluster secret +CLUSTER_SECRET=$(cat ~/.debros/cluster-secret) + +# Initialize cluster (will create service.json) +sudo -u debros ipfs-cluster-service init --consensus crdt +``` + +### 3. Start Services + +```bash +systemctl start debros-ipfs +systemctl start debros-ipfs-cluster +systemctl start debros-node +systemctl start debros-gateway +``` + +## Ports + +- **4001**: IPFS swarm (LibP2P) +- **5001**: IPFS HTTP API +- **8080**: IPFS Gateway (optional) +- **9094**: IPFS Cluster REST API +- **9096**: IPFS Cluster swarm (LibP2P) + +## Replication Factor + +The default replication factor is 3, meaning content is pinned to 3 cluster peers. This requires at least 3 nodes running cluster peers. + +To change replication factor, edit node configs: + +```yaml +database: + ipfs: + replication_factor: 1 # For single-node development +``` + +## Security Notes + +- Cluster secret is stored at `~/.debros/cluster-secret` (mode 0600) +- Cluster API (port 9094) should be firewalled in production +- IPFS API (port 5001) should only be accessible locally diff --git a/pkg/cli/setup.go b/pkg/cli/setup.go index c681554..f9e8634 100644 --- a/pkg/cli/setup.go +++ b/pkg/cli/setup.go @@ -2,6 +2,9 @@ package cli import ( "bufio" + "crypto/rand" + "encoding/hex" + "encoding/json" "fmt" "net" "os" @@ -63,11 +66,12 @@ func HandleSetupCommand(args []string) { fmt.Printf(" 4. Install RQLite database\n") fmt.Printf(" 5. Install Anyone Relay (Anon) for anonymous networking\n") fmt.Printf(" 6. Install Olric cache server\n") - fmt.Printf(" 7. Create directories (/home/debros/bin, /home/debros/src)\n") - fmt.Printf(" 8. Clone and build DeBros Network\n") - fmt.Printf(" 9. Generate configuration files\n") - fmt.Printf(" 10. Create systemd services (debros-node, debros-gateway, debros-olric)\n") - fmt.Printf(" 11. Start and enable services\n") + fmt.Printf(" 7. Install IPFS (Kubo) and IPFS Cluster\n") + fmt.Printf(" 8. Create directories (/home/debros/bin, /home/debros/src)\n") + fmt.Printf(" 9. Clone and build DeBros Network\n") + fmt.Printf(" 10. Generate configuration files\n") + fmt.Printf(" 11. Create systemd services (debros-ipfs, debros-ipfs-cluster, debros-node, debros-gateway, debros-olric)\n") + fmt.Printf(" 12. Start and enable services\n") fmt.Printf(strings.Repeat("=", 70) + "\n\n") fmt.Printf("Ready to begin setup? (yes/no): ") @@ -96,6 +100,9 @@ func HandleSetupCommand(args []string) { // Step 4.6: Install Olric cache server installOlric() + // Step 4.7: Install IPFS and IPFS Cluster + installIPFS() + // Step 5: Setup directories setupDirectories() @@ -123,6 +130,14 @@ func HandleSetupCommand(args []string) { fmt.Printf("🆔 Node Peer ID: %s\n\n", peerID) } + // Display IPFS Cluster information + fmt.Printf("IPFS Cluster Setup:\n") + fmt.Printf(" Each node runs its own IPFS Cluster peer\n") + fmt.Printf(" Cluster peers use CRDT consensus for automatic discovery\n") + fmt.Printf(" To verify cluster is working:\n") + fmt.Printf(" sudo -u debros ipfs-cluster-ctl --host http://localhost:9094 peers ls\n") + fmt.Printf(" You should see all cluster peers listed\n\n") + fmt.Printf("Service Management:\n") fmt.Printf(" network-cli service status all\n") fmt.Printf(" network-cli service logs node --follow\n") @@ -1156,6 +1171,92 @@ func configureFirewallForOlric() { fmt.Printf(" No active firewall detected for Olric\n") } +func installIPFS() { + fmt.Printf("🌐 Installing IPFS (Kubo) and IPFS Cluster...\n") + + // Check if IPFS is already installed + if _, err := exec.LookPath("ipfs"); err == nil { + fmt.Printf(" ✓ IPFS (Kubo) already installed\n") + } else { + fmt.Printf(" Installing IPFS (Kubo)...\n") + // Install IPFS via official installation script + cmd := exec.Command("bash", "-c", "curl -fsSL https://dist.ipfs.tech/kubo/v0.27.0/install.sh | bash") + if err := cmd.Run(); err != nil { + fmt.Fprintf(os.Stderr, "⚠️ Failed to install IPFS: %v\n", err) + fmt.Fprintf(os.Stderr, " You may need to install IPFS manually: https://docs.ipfs.tech/install/command-line/\n") + return + } + // Make sure ipfs is in PATH + exec.Command("ln", "-sf", "/usr/local/bin/ipfs", "/usr/bin/ipfs").Run() + fmt.Printf(" ✓ IPFS (Kubo) installed\n") + } + + // Check if IPFS Cluster is already installed + if _, err := exec.LookPath("ipfs-cluster-service"); err == nil { + fmt.Printf(" ✓ IPFS Cluster already installed\n") + } else { + fmt.Printf(" Installing IPFS Cluster...\n") + // Install IPFS Cluster via go install + if _, err := exec.LookPath("go"); err != nil { + fmt.Fprintf(os.Stderr, "⚠️ Go not found - cannot install IPFS Cluster. Please install Go first.\n") + return + } + cmd := exec.Command("go", "install", "github.com/ipfs-cluster/ipfs-cluster/cmd/ipfs-cluster-service@latest") + cmd.Env = append(os.Environ(), "GOBIN=/usr/local/bin") + if output, err := cmd.CombinedOutput(); err != nil { + fmt.Fprintf(os.Stderr, "⚠️ Failed to install IPFS Cluster: %v\n", err) + if len(output) > 0 { + fmt.Fprintf(os.Stderr, " Output: %s\n", string(output)) + } + fmt.Fprintf(os.Stderr, " You can manually install with: go install github.com/ipfs-cluster/ipfs-cluster/cmd/ipfs-cluster-service@latest\n") + return + } + // Also install ipfs-cluster-ctl for management + exec.Command("go", "install", "github.com/ipfs-cluster/ipfs-cluster/cmd/ipfs-cluster-ctl@latest").Run() + fmt.Printf(" ✓ IPFS Cluster installed\n") + } + + // Configure firewall for IPFS and Cluster + configureFirewallForIPFS() + + fmt.Printf(" ✓ IPFS and IPFS Cluster setup complete\n") +} + +func configureFirewallForIPFS() { + fmt.Printf(" Checking firewall configuration for IPFS...\n") + + // Check for UFW + if _, err := exec.LookPath("ufw"); err == nil { + output, _ := exec.Command("ufw", "status").CombinedOutput() + if strings.Contains(string(output), "Status: active") { + fmt.Printf(" Adding UFW rules for IPFS and Cluster...\n") + exec.Command("ufw", "allow", "4001/tcp", "comment", "IPFS Swarm").Run() + exec.Command("ufw", "allow", "5001/tcp", "comment", "IPFS API").Run() + exec.Command("ufw", "allow", "9094/tcp", "comment", "IPFS Cluster API").Run() + exec.Command("ufw", "allow", "9096/tcp", "comment", "IPFS Cluster Swarm").Run() + fmt.Printf(" ✓ UFW rules added for IPFS\n") + return + } + } + + // Check for firewalld + if _, err := exec.LookPath("firewall-cmd"); err == nil { + output, _ := exec.Command("firewall-cmd", "--state").CombinedOutput() + if strings.Contains(string(output), "running") { + fmt.Printf(" Adding firewalld rules for IPFS...\n") + exec.Command("firewall-cmd", "--permanent", "--add-port=4001/tcp").Run() + exec.Command("firewall-cmd", "--permanent", "--add-port=5001/tcp").Run() + exec.Command("firewall-cmd", "--permanent", "--add-port=9094/tcp").Run() + exec.Command("firewall-cmd", "--permanent", "--add-port=9096/tcp").Run() + exec.Command("firewall-cmd", "--reload").Run() + fmt.Printf(" ✓ firewalld rules added for IPFS\n") + return + } + } + + fmt.Printf(" No active firewall detected for IPFS\n") +} + func setupDirectories() { fmt.Printf("📁 Creating directories...\n") @@ -1405,6 +1506,18 @@ func generateConfigsInteractive(force bool) { exec.Command("chown", "debros:debros", nodeConfigPath).Run() fmt.Printf(" ✓ Node config created: %s\n", nodeConfigPath) + // Initialize IPFS and Cluster for this node + var nodeID string + if isBootstrap { + nodeID = "bootstrap" + } else { + nodeID = "node" + } + if err := initializeIPFSForNode(nodeID, vpsIP, isBootstrap); err != nil { + fmt.Fprintf(os.Stderr, "⚠️ Failed to initialize IPFS/Cluster: %v\n", err) + fmt.Fprintf(os.Stderr, " You may need to initialize IPFS and Cluster manually\n") + } + // Generate Olric config file for this node (uses multicast discovery) var olricConfigPath string if isBootstrap { @@ -1730,14 +1843,309 @@ func generateOlricConfig(configPath, bindIP string, httpPort, memberlistPort int return nil } +// getOrGenerateClusterSecret gets or generates a shared cluster secret +func getOrGenerateClusterSecret() (string, error) { + secretPath := "/home/debros/.debros/cluster-secret" + + // Try to read existing secret + if data, err := os.ReadFile(secretPath); err == nil { + secret := strings.TrimSpace(string(data)) + if len(secret) == 64 { + return secret, nil + } + } + + // Generate new secret (64 hex characters = 32 bytes) + bytes := make([]byte, 32) + if _, err := rand.Read(bytes); err != nil { + return "", fmt.Errorf("failed to generate cluster secret: %w", err) + } + secret := hex.EncodeToString(bytes) + + // Save secret + if err := os.WriteFile(secretPath, []byte(secret), 0600); err != nil { + return "", fmt.Errorf("failed to save cluster secret: %w", err) + } + exec.Command("chown", "debros:debros", secretPath).Run() + + return secret, nil +} + +// initializeIPFSForNode initializes IPFS and IPFS Cluster for a node +func initializeIPFSForNode(nodeID, vpsIP string, isBootstrap bool) error { + fmt.Printf(" Initializing IPFS and Cluster for node %s...\n", nodeID) + + // Get or generate cluster secret + secret, err := getOrGenerateClusterSecret() + if err != nil { + return fmt.Errorf("failed to get cluster secret: %w", err) + } + + // Determine data directories + var ipfsDataDir, clusterDataDir string + if nodeID == "bootstrap" { + ipfsDataDir = "/home/debros/.debros/bootstrap/ipfs" + clusterDataDir = "/home/debros/.debros/bootstrap/ipfs-cluster" + } else { + ipfsDataDir = "/home/debros/.debros/node/ipfs" + clusterDataDir = "/home/debros/.debros/node/ipfs-cluster" + } + + // Create directories + os.MkdirAll(ipfsDataDir, 0755) + os.MkdirAll(clusterDataDir, 0755) + exec.Command("chown", "-R", "debros:debros", ipfsDataDir).Run() + exec.Command("chown", "-R", "debros:debros", clusterDataDir).Run() + + // Initialize IPFS if not already initialized + ipfsRepoPath := filepath.Join(ipfsDataDir, "repo") + if _, err := os.Stat(filepath.Join(ipfsRepoPath, "config")); os.IsNotExist(err) { + fmt.Printf(" Initializing IPFS repository...\n") + cmd := exec.Command("sudo", "-u", "debros", "ipfs", "init", "--profile=server", "--repo-dir="+ipfsRepoPath) + if output, err := cmd.CombinedOutput(); err != nil { + return fmt.Errorf("failed to initialize IPFS: %v\n%s", err, string(output)) + } + + // Configure IPFS API and Gateway addresses + exec.Command("sudo", "-u", "debros", "ipfs", "config", "--json", "Addresses.API", `["/ip4/127.0.0.1/tcp/5001"]`, "--repo-dir="+ipfsRepoPath).Run() + exec.Command("sudo", "-u", "debros", "ipfs", "config", "--json", "Addresses.Gateway", `["/ip4/127.0.0.1/tcp/8080"]`, "--repo-dir="+ipfsRepoPath).Run() + exec.Command("sudo", "-u", "debros", "ipfs", "config", "--json", "Addresses.Swarm", `["/ip4/0.0.0.0/tcp/4001","/ip6/::/tcp/4001"]`, "--repo-dir="+ipfsRepoPath).Run() + fmt.Printf(" ✓ IPFS initialized\n") + } + + // Initialize IPFS Cluster if not already initialized + clusterConfigPath := filepath.Join(clusterDataDir, "service.json") + if _, err := os.Stat(clusterConfigPath); os.IsNotExist(err) { + fmt.Printf(" Initializing IPFS Cluster...\n") + + // Generate cluster config + clusterConfig := generateClusterServiceConfig(nodeID, vpsIP, secret, isBootstrap) + + // Write config + configJSON, err := json.MarshalIndent(clusterConfig, "", " ") + if err != nil { + return fmt.Errorf("failed to marshal cluster config: %w", err) + } + + if err := os.WriteFile(clusterConfigPath, configJSON, 0644); err != nil { + return fmt.Errorf("failed to write cluster config: %w", err) + } + exec.Command("chown", "debros:debros", clusterConfigPath).Run() + + fmt.Printf(" ✓ IPFS Cluster initialized\n") + } + + return nil +} + +// getClusterPeerID gets the cluster peer ID from a running cluster service +func getClusterPeerID(clusterAPIURL string) (string, error) { + cmd := exec.Command("ipfs-cluster-ctl", "--host", clusterAPIURL, "id") + output, err := cmd.CombinedOutput() + if err != nil { + return "", fmt.Errorf("failed to get cluster peer ID: %v\n%s", err, string(output)) + } + + // Parse output to extract peer ID + // Output format: "12D3KooW..." + lines := strings.Split(string(output), "\n") + for _, line := range lines { + line = strings.TrimSpace(line) + if strings.HasPrefix(line, "12D3Koo") { + return line, nil + } + } + + return "", fmt.Errorf("could not parse cluster peer ID from output: %s", string(output)) +} + +// getClusterPeerMultiaddr constructs the cluster peer multiaddr +func getClusterPeerMultiaddr(vpsIP, peerID string) string { + return fmt.Sprintf("/ip4/%s/tcp/9096/p2p/%s", vpsIP, peerID) +} + +// clusterServiceConfig represents IPFS Cluster service.json structure +type clusterServiceConfig struct { + Cluster clusterConfig `json:"cluster"` + Consensus consensusConfig `json:"consensus"` + API apiConfig `json:"api"` + IPFSConnector ipfsConnectorConfig `json:"ipfs_connector"` + Datastore datastoreConfig `json:"datastore"` +} + +type clusterConfig struct { + ID string `json:"id"` + PrivateKey string `json:"private_key"` + Secret string `json:"secret"` + Peername string `json:"peername"` + Bootstrap []string `json:"bootstrap"` + LeaveOnShutdown bool `json:"leave_on_shutdown"` + ListenMultiaddr string `json:"listen_multiaddress"` + ConnectionManager connectionManagerConfig `json:"connection_manager"` +} + +type connectionManagerConfig struct { + LowWater int `json:"low_water"` + HighWater int `json:"high_water"` + GracePeriod string `json:"grace_period"` +} + +type consensusConfig struct { + CRDT crdtConfig `json:"crdt"` +} + +type crdtConfig struct { + ClusterName string `json:"cluster_name"` + TrustedPeers []string `json:"trusted_peers"` +} + +type apiConfig struct { + RestAPI restAPIConfig `json:"restapi"` +} + +type restAPIConfig struct { + HTTPListenMultiaddress string `json:"http_listen_multiaddress"` + ID string `json:"id"` + BasicAuthCredentials interface{} `json:"basic_auth_credentials"` +} + +type ipfsConnectorConfig struct { + IPFSHTTP ipfsHTTPConfig `json:"ipfshttp"` +} + +type ipfsHTTPConfig struct { + NodeMultiaddress string `json:"node_multiaddress"` +} + +type datastoreConfig struct { + Type string `json:"type"` + Path string `json:"path"` +} + +// generateClusterServiceConfig generates IPFS Cluster service.json config +func generateClusterServiceConfig(nodeID, vpsIP, secret string, isBootstrap bool) clusterServiceConfig { + clusterListenAddr := "/ip4/0.0.0.0/tcp/9096" + restAPIListenAddr := "/ip4/0.0.0.0/tcp/9094" + + // For bootstrap node, use empty bootstrap list + // For other nodes, bootstrap list will be set when starting the service + bootstrap := []string{} + + return clusterServiceConfig{ + Cluster: clusterConfig{ + Peername: nodeID, + Secret: secret, + Bootstrap: bootstrap, + LeaveOnShutdown: false, + ListenMultiaddr: clusterListenAddr, + ConnectionManager: connectionManagerConfig{ + LowWater: 50, + HighWater: 200, + GracePeriod: "20s", + }, + }, + Consensus: consensusConfig{ + CRDT: crdtConfig{ + ClusterName: "debros-cluster", + TrustedPeers: []string{"*"}, // Trust all peers + }, + }, + API: apiConfig{ + RestAPI: restAPIConfig{ + HTTPListenMultiaddress: restAPIListenAddr, + ID: "", + BasicAuthCredentials: nil, + }, + }, + IPFSConnector: ipfsConnectorConfig{ + IPFSHTTP: ipfsHTTPConfig{ + NodeMultiaddress: "/ip4/127.0.0.1/tcp/5001", + }, + }, + Datastore: datastoreConfig{ + Type: "badger", + Path: fmt.Sprintf("/home/debros/.debros/%s/ipfs-cluster/badger", nodeID), + }, + } +} + func createSystemdServices() { fmt.Printf("🔧 Creating systemd services...\n") + // IPFS service (runs on all nodes) + ipfsService := `[Unit] +Description=IPFS Daemon +After=network-online.target +Wants=network-online.target + +[Service] +Type=simple +User=debros +Group=debros +Environment=HOME=/home/debros +ExecStartPre=/bin/bash -c 'if [ -f /home/debros/.debros/node.yaml ]; then export IPFS_PATH=/home/debros/.debros/node/ipfs/repo; elif [ -f /home/debros/.debros/bootstrap.yaml ]; then export IPFS_PATH=/home/debros/.debros/bootstrap/ipfs/repo; else export IPFS_PATH=/home/debros/.debros/bootstrap/ipfs/repo; fi' +ExecStart=/usr/bin/ipfs daemon --enable-pubsub-experiment --repo-dir=${IPFS_PATH} +Restart=always +RestartSec=5 +StandardOutput=journal +StandardError=journal +SyslogIdentifier=ipfs + +NoNewPrivileges=yes +PrivateTmp=yes +ProtectSystem=strict +ReadWritePaths=/home/debros + +[Install] +WantedBy=multi-user.target +` + + if err := os.WriteFile("/etc/systemd/system/debros-ipfs.service", []byte(ipfsService), 0644); err != nil { + fmt.Fprintf(os.Stderr, "❌ Failed to create IPFS service: %v\n", err) + os.Exit(1) + } + + // IPFS Cluster service (runs on all nodes) + clusterService := `[Unit] +Description=IPFS Cluster Service +After=debros-ipfs.service +Wants=debros-ipfs.service +Requires=debros-ipfs.service + +[Service] +Type=simple +User=debros +Group=debros +WorkingDirectory=/home/debros +Environment=HOME=/home/debros +ExecStartPre=/bin/bash -c 'if [ -f /home/debros/.debros/node.yaml ]; then export CLUSTER_PATH=/home/debros/.debros/node/ipfs-cluster; elif [ -f /home/debros/.debros/bootstrap.yaml ]; then export CLUSTER_PATH=/home/debros/.debros/bootstrap/ipfs-cluster; else export CLUSTER_PATH=/home/debros/.debros/bootstrap/ipfs-cluster; fi' +ExecStart=/usr/local/bin/ipfs-cluster-service daemon --config ${CLUSTER_PATH}/service.json +Restart=always +RestartSec=5 +StandardOutput=journal +StandardError=journal +SyslogIdentifier=ipfs-cluster + +NoNewPrivileges=yes +PrivateTmp=yes +ProtectSystem=strict +ReadWritePaths=/home/debros + +[Install] +WantedBy=multi-user.target +` + + if err := os.WriteFile("/etc/systemd/system/debros-ipfs-cluster.service", []byte(clusterService), 0644); err != nil { + fmt.Fprintf(os.Stderr, "❌ Failed to create IPFS Cluster service: %v\n", err) + os.Exit(1) + } + // Node service nodeService := `[Unit] Description=DeBros Network Node -After=network-online.target -Wants=network-online.target +After=network-online.target debros-ipfs-cluster.service +Wants=network-online.target debros-ipfs-cluster.service +Requires=debros-ipfs-cluster.service [Service] Type=simple @@ -1807,6 +2215,8 @@ WantedBy=multi-user.target // Reload systemd exec.Command("systemctl", "daemon-reload").Run() + exec.Command("systemctl", "enable", "debros-ipfs").Run() + exec.Command("systemctl", "enable", "debros-ipfs-cluster").Run() exec.Command("systemctl", "enable", "debros-node").Run() exec.Command("systemctl", "enable", "debros-gateway").Run() @@ -1841,6 +2251,18 @@ func startServices() { } } + // Start IPFS first (required by Cluster) + startOrRestartService("debros-ipfs") + + // Wait a bit for IPFS to start + time.Sleep(2 * time.Second) + + // Start IPFS Cluster (required by Node) + startOrRestartService("debros-ipfs-cluster") + + // Wait a bit for Cluster to start + time.Sleep(2 * time.Second) + // Start or restart node service startOrRestartService("debros-node") diff --git a/pkg/gateway/gateway.go b/pkg/gateway/gateway.go index fc2dce1..d1d1545 100644 --- a/pkg/gateway/gateway.go +++ b/pkg/gateway/gateway.go @@ -254,6 +254,25 @@ func New(logger *logging.ColoredLogger, cfg *Config) (*Gateway, error) { logger.ComponentWarn(logging.ComponentGeneral, "failed to initialize IPFS Cluster client; storage endpoints disabled", zap.Error(ipfsErr)) } else { gw.ipfsClient = ipfsClient + + // Check peer count and warn if insufficient (use background context to avoid blocking) + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + if peerCount, err := ipfsClient.GetPeerCount(ctx); err == nil { + if peerCount < ipfsReplicationFactor { + logger.ComponentWarn(logging.ComponentGeneral, "insufficient cluster peers for replication factor", + zap.Int("peer_count", peerCount), + zap.Int("replication_factor", ipfsReplicationFactor), + zap.String("message", "Some pin operations may fail until more peers join the cluster")) + } else { + logger.ComponentInfo(logging.ComponentGeneral, "IPFS Cluster peer count sufficient", + zap.Int("peer_count", peerCount), + zap.Int("replication_factor", ipfsReplicationFactor)) + } + } else { + logger.ComponentWarn(logging.ComponentGeneral, "failed to get cluster peer count", zap.Error(err)) + } + logger.ComponentInfo(logging.ComponentGeneral, "IPFS Cluster client ready", zap.String("cluster_api_url", ipfsCfg.ClusterAPIURL), zap.String("ipfs_api_url", ipfsAPIURL), diff --git a/pkg/gateway/storage_handlers.go b/pkg/gateway/storage_handlers.go index 13269e1..16706b3 100644 --- a/pkg/gateway/storage_handlers.go +++ b/pkg/gateway/storage_handlers.go @@ -275,7 +275,12 @@ func (g *Gateway) storageGetHandler(w http.ResponseWriter, r *http.Request) { reader, err := g.ipfsClient.Get(ctx, path, ipfsAPIURL) if err != nil { g.logger.ComponentError(logging.ComponentGeneral, "failed to get content from IPFS", zap.Error(err), zap.String("cid", path)) - writeError(w, http.StatusInternalServerError, fmt.Sprintf("failed to get content: %v", err)) + // Check if error indicates content not found (404) + if strings.Contains(err.Error(), "not found") || strings.Contains(err.Error(), "status 404") { + writeError(w, http.StatusNotFound, fmt.Sprintf("content not found: %s", path)) + } else { + writeError(w, http.StatusInternalServerError, fmt.Sprintf("failed to get content: %v", err)) + } return } defer reader.Close() diff --git a/pkg/ipfs/client.go b/pkg/ipfs/client.go index b415fd0..83dbb5d 100644 --- a/pkg/ipfs/client.go +++ b/pkg/ipfs/client.go @@ -8,6 +8,7 @@ import ( "io" "mime/multipart" "net/http" + "net/url" "time" "go.uber.org/zap" @@ -21,6 +22,7 @@ type IPFSClient interface { Get(ctx context.Context, cid string, ipfsAPIURL string) (io.ReadCloser, error) Unpin(ctx context.Context, cid string) error Health(ctx context.Context) error + GetPeerCount(ctx context.Context) (int, error) Close(ctx context.Context) error } @@ -110,6 +112,33 @@ func (c *Client) Health(ctx context.Context) error { return nil } +// GetPeerCount returns the number of cluster peers +func (c *Client) GetPeerCount(ctx context.Context) (int, error) { + req, err := http.NewRequestWithContext(ctx, "GET", c.apiURL+"/peers", nil) + if err != nil { + return 0, fmt.Errorf("failed to create peers request: %w", err) + } + + resp, err := c.httpClient.Do(req) + if err != nil { + return 0, fmt.Errorf("peers request failed: %w", err) + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + return 0, fmt.Errorf("peers request failed with status: %d", resp.StatusCode) + } + + var peers []struct { + ID string `json:"id"` + } + if err := json.NewDecoder(resp.Body).Decode(&peers); err != nil { + return 0, fmt.Errorf("failed to decode peers response: %w", err) + } + + return len(peers), nil +} + // Add adds content to IPFS and returns the CID func (c *Client) Add(ctx context.Context, reader io.Reader, name string) (*AddResponse, error) { // Create multipart form request for IPFS Cluster API @@ -157,28 +186,25 @@ func (c *Client) Add(ctx context.Context, reader io.Reader, name string) (*AddRe } // Pin pins a CID with specified replication factor +// IPFS Cluster expects pin options (including name) as query parameters, not in JSON body func (c *Client) Pin(ctx context.Context, cid string, name string, replicationFactor int) (*PinResponse, error) { - reqBody := map[string]interface{}{ - "cid": cid, - "replication_factor_min": replicationFactor, - "replication_factor_max": replicationFactor, - } + // Build URL with query parameters + reqURL := c.apiURL + "/pins/" + cid + values := url.Values{} + values.Set("replication-min", fmt.Sprintf("%d", replicationFactor)) + values.Set("replication-max", fmt.Sprintf("%d", replicationFactor)) if name != "" { - reqBody["name"] = name + values.Set("name", name) + } + if len(values) > 0 { + reqURL += "?" + values.Encode() } - jsonBody, err := json.Marshal(reqBody) - if err != nil { - return nil, fmt.Errorf("failed to marshal pin request: %w", err) - } - - req, err := http.NewRequestWithContext(ctx, "POST", c.apiURL+"/pins/"+cid, bytes.NewReader(jsonBody)) + req, err := http.NewRequestWithContext(ctx, "POST", reqURL, nil) if err != nil { return nil, fmt.Errorf("failed to create pin request: %w", err) } - req.Header.Set("Content-Type", "application/json") - resp, err := c.httpClient.Do(req) if err != nil { return nil, fmt.Errorf("pin request failed: %w", err) @@ -242,6 +268,9 @@ func (c *Client) PinStatus(ctx context.Context, cid string) (*PinStatus, error) return nil, fmt.Errorf("failed to decode pin status response: %w", err) } + // Use name from GlobalPinInfo + name := gpi.Name + // Extract status from peer map (use first peer's status, or aggregate) status := "unknown" peers := make([]string, 0, len(gpi.PeerMap)) @@ -274,7 +303,7 @@ func (c *Client) PinStatus(ctx context.Context, cid string) (*PinStatus, error) result := &PinStatus{ Cid: gpi.Cid, - Name: gpi.Name, + Name: name, Status: status, ReplicationMin: 0, // Not available in GlobalPinInfo ReplicationMax: 0, // Not available in GlobalPinInfo @@ -331,8 +360,12 @@ func (c *Client) Get(ctx context.Context, cid string, ipfsAPIURL string) (io.Rea } if resp.StatusCode != http.StatusOK { + body, _ := io.ReadAll(resp.Body) resp.Body.Close() - return nil, fmt.Errorf("get failed with status %d", resp.StatusCode) + if resp.StatusCode == http.StatusNotFound { + return nil, fmt.Errorf("content not found (CID: %s). The content may not be available on the IPFS node, or the IPFS API may not be accessible at %s", cid, ipfsAPIURL) + } + return nil, fmt.Errorf("get failed with status %d: %s", resp.StatusCode, string(body)) } return resp.Body, nil diff --git a/scripts/update_changelog.sh b/scripts/update_changelog.sh index 1f10e1d..72f70c2 100755 --- a/scripts/update_changelog.sh +++ b/scripts/update_changelog.sh @@ -67,6 +67,15 @@ if ! command -v curl > /dev/null 2>&1; then exit 1 fi +# Check for skip flag +# To skip changelog generation, set SKIP_CHANGELOG=1 before committing: +# SKIP_CHANGELOG=1 git commit -m "your message" +# SKIP_CHANGELOG=1 git commit +if [ "$SKIP_CHANGELOG" = "1" ] || [ "$SKIP_CHANGELOG" = "true" ]; then + log "Skipping changelog update (SKIP_CHANGELOG is set)" + exit 0 +fi + # Check if we're in a git repo if ! git rev-parse --git-dir > /dev/null 2>&1; then error "Not in a git repository"