diff --git a/Makefile b/Makefile index 5effa04..7e4029d 100644 --- a/Makefile +++ b/Makefile @@ -103,6 +103,66 @@ build: deps go build -ldflags "$(LDFLAGS) -X 'github.com/DeBrosOfficial/network/pkg/gateway.BuildVersion=$(VERSION)' -X 'github.com/DeBrosOfficial/network/pkg/gateway.BuildCommit=$(COMMIT)' -X 'github.com/DeBrosOfficial/network/pkg/gateway.BuildTime=$(DATE)'" -o bin/gateway ./cmd/gateway @echo "Build complete! Run ./bin/orama version" +# Cross-compile all binaries for Linux (used with --pre-built flag on VPS) +# Builds: DeBros binaries + Olric + CoreDNS (with rqlite plugin) + Caddy (with orama DNS module) +build-linux: deps + @echo "Cross-compiling all binaries for linux/amd64 (version=$(VERSION))..." + @mkdir -p bin-linux + GOOS=linux GOARCH=amd64 go build -ldflags "$(LDFLAGS)" -o bin-linux/identity ./cmd/identity + GOOS=linux GOARCH=amd64 go build -ldflags "$(LDFLAGS)" -o bin-linux/orama-node ./cmd/node + GOOS=linux GOARCH=amd64 go build -ldflags "$(LDFLAGS)" -o bin-linux/orama cmd/cli/main.go + GOOS=linux GOARCH=amd64 go build -ldflags "$(LDFLAGS)" -o bin-linux/rqlite-mcp ./cmd/rqlite-mcp + GOOS=linux GOARCH=amd64 go build -ldflags "$(LDFLAGS) -X 'github.com/DeBrosOfficial/network/pkg/gateway.BuildVersion=$(VERSION)' -X 'github.com/DeBrosOfficial/network/pkg/gateway.BuildCommit=$(COMMIT)' -X 'github.com/DeBrosOfficial/network/pkg/gateway.BuildTime=$(DATE)'" -o bin-linux/gateway ./cmd/gateway + GOOS=linux GOARCH=amd64 go build -ldflags "$(LDFLAGS)" -o bin-linux/orama-cli ./cmd/cli + @echo "Installing Olric for linux/amd64..." + GOOS=linux GOARCH=amd64 GOBIN=$(CURDIR)/bin-linux go install github.com/olric-data/olric/cmd/olric-server@v0.7.0 + @echo "✓ All Linux binaries built in bin-linux/" + @echo "" + @echo "Next steps:" + @echo " 1. Build CoreDNS: make build-linux-coredns" + @echo " 2. Build Caddy: make build-linux-caddy" + @echo " 3. Or build all: make build-linux-all" + +# Build CoreDNS with rqlite plugin for Linux +build-linux-coredns: + @echo "Building CoreDNS with rqlite plugin for linux/amd64..." + @rm -rf /tmp/coredns-build-linux + @mkdir -p /tmp/coredns-build-linux + git clone --depth 1 --branch v1.12.0 https://github.com/coredns/coredns.git /tmp/coredns-build-linux + @mkdir -p /tmp/coredns-build-linux/plugin/rqlite + @cp pkg/coredns/rqlite/*.go /tmp/coredns-build-linux/plugin/rqlite/ + @echo 'metadata:metadata\ncancel:cancel\ntls:tls\nreload:reload\nnsid:nsid\nbufsize:bufsize\nroot:root\nbind:bind\ndebug:debug\ntrace:trace\nready:ready\nhealth:health\npprof:pprof\nprometheus:metrics\nerrors:errors\nlog:log\ndnstap:dnstap\nlocal:local\ndns64:dns64\nacl:acl\nany:any\nchaos:chaos\nloadbalance:loadbalance\ncache:cache\nrewrite:rewrite\nheader:header\ndnssec:dnssec\nautopath:autopath\nminimal:minimal\ntemplate:template\ntransfer:transfer\nhosts:hosts\nfile:file\nauto:auto\nsecondary:secondary\nloop:loop\nforward:forward\ngrpc:grpc\nerratic:erratic\nwhoami:whoami\non:github.com/coredns/caddy/onevent\nsign:sign\nview:view\nrqlite:rqlite' > /tmp/coredns-build-linux/plugin.cfg + cd /tmp/coredns-build-linux && go get github.com/miekg/dns@latest && go get go.uber.org/zap@latest && go mod tidy && go generate && GOOS=linux GOARCH=amd64 CGO_ENABLED=0 go build -o coredns + @cp /tmp/coredns-build-linux/coredns bin-linux/coredns + @rm -rf /tmp/coredns-build-linux + @echo "✓ CoreDNS built: bin-linux/coredns" + +# Build Caddy with orama DNS module for Linux +build-linux-caddy: + @echo "Building Caddy with orama DNS module for linux/amd64..." + @which xcaddy > /dev/null 2>&1 || (echo "Installing xcaddy..." && go install github.com/caddyserver/xcaddy/cmd/xcaddy@latest) + @rm -rf /tmp/caddy-build-linux + @mkdir -p /tmp/caddy-build-linux/caddy-dns-orama + @# Generate the module source + @echo 'module github.com/DeBrosOfficial/caddy-dns-orama\n\ngo 1.22\n\nrequire (\n\tgithub.com/caddyserver/caddy/v2 v2.10.2\n\tgithub.com/libdns/libdns v1.1.0\n)' > /tmp/caddy-build-linux/caddy-dns-orama/go.mod + @cp pkg/caddy/dns/orama/provider.go /tmp/caddy-build-linux/caddy-dns-orama/ 2>/dev/null || true + cd /tmp/caddy-build-linux/caddy-dns-orama && go mod tidy + GOOS=linux GOARCH=amd64 xcaddy build v2.10.2 --with github.com/DeBrosOfficial/caddy-dns-orama=/tmp/caddy-build-linux/caddy-dns-orama --output bin-linux/caddy + @rm -rf /tmp/caddy-build-linux + @echo "✓ Caddy built: bin-linux/caddy" + +# Build everything for Linux (all binaries + CoreDNS + Caddy) +build-linux-all: build-linux build-linux-coredns build-linux-caddy + @echo "" + @echo "✅ All Linux binaries ready in bin-linux/:" + @ls -la bin-linux/ + @echo "" + @echo "Deploy to VPS:" + @echo " scp bin-linux/* ubuntu@:/home/debros/bin/" + @echo " scp bin-linux/coredns ubuntu@:/usr/local/bin/coredns" + @echo " scp bin-linux/caddy ubuntu@:/usr/bin/caddy" + @echo " sudo orama install --pre-built --no-pull ..." + # Install git hooks install-hooks: @echo "Installing git hooks..." diff --git a/docs/DEV_DEPLOY.md b/docs/DEV_DEPLOY.md index e0f2555..5b5980c 100644 --- a/docs/DEV_DEPLOY.md +++ b/docs/DEV_DEPLOY.md @@ -70,6 +70,45 @@ sudo mv /tmp/orama /usr/local/bin/orama && sudo chmod +x /usr/local/bin/orama sudo orama upgrade --no-pull --restart ``` +### Development Deployment with Pre-Built Binaries (Fastest) + +Cross-compile everything locally and skip all Go compilation on the VPS. This is significantly faster because your local machine compiles much faster than the VPS. + +```bash +# 1. Cross-compile all binaries for Linux (DeBros + Olric + CoreDNS + Caddy) +make build-linux-all +# Outputs everything to bin-linux/ + +# 2. Generate a source archive (still needed for configs, templates, etc.) +./scripts/generate-source-archive.sh + +# 3. Copy everything to the VPS +sshpass -p '' scp -o StrictHostKeyChecking=no bin-linux/orama ubuntu@:/tmp/orama +sshpass -p '' scp -o StrictHostKeyChecking=no /tmp/network-source.tar.gz ubuntu@:/tmp/ + +# 4. On the VPS: extract source, install CLI, and copy pre-built binaries +ssh ubuntu@ +sudo rm -rf /home/debros/src && sudo mkdir -p /home/debros/src +sudo tar xzf /tmp/network-source.tar.gz -C /home/debros/src +sudo chown -R debros:debros /home/debros/src +sudo mv /tmp/orama /usr/local/bin/orama && sudo chmod +x /usr/local/bin/orama + +# 5. Copy pre-built binaries to their expected locations +# (from your local machine) +sshpass -p '' scp -o StrictHostKeyChecking=no bin-linux/orama-node bin-linux/gateway bin-linux/identity bin-linux/rqlite-mcp bin-linux/olric-server ubuntu@:/home/debros/bin/ +sshpass -p '' scp -o StrictHostKeyChecking=no bin-linux/coredns ubuntu@:/usr/local/bin/coredns +sshpass -p '' scp -o StrictHostKeyChecking=no bin-linux/caddy ubuntu@:/usr/bin/caddy + +# 6. Install/upgrade with --pre-built (skips ALL Go compilation on VPS) +sudo orama install --no-pull --pre-built --vps-ip ... +# or +sudo orama upgrade --no-pull --pre-built --restart +``` + +**What `--pre-built` skips:** Go installation, `make build`, Olric `go install`, CoreDNS build, Caddy/xcaddy build. + +**What `--pre-built` still runs:** apt dependencies, RQLite/IPFS/IPFS Cluster downloads (pre-built binary downloads, fast), Anyone relay setup, config generation, systemd service creation. + ### Production Deployment (Via Git) For production releases — pulls source from GitHub on the VPS. @@ -107,6 +146,7 @@ To deploy to all nodes, repeat steps 3-5 (dev) or 3-4 (production) for each VPS | `--token ` | Invite token for joining (from `orama invite` on existing node) | | `--branch ` | Git branch to use (default: main) | | `--no-pull` | Skip git clone/pull, use existing `/home/debros/src` | +| `--pre-built` | Skip all Go compilation, use pre-built binaries already on disk (see above) | | `--force` | Force reconfiguration even if already installed | | `--skip-firewall` | Skip UFW firewall setup | | `--skip-checks` | Skip minimum resource checks (RAM/CPU) | @@ -137,6 +177,7 @@ To deploy to all nodes, repeat steps 3-5 (dev) or 3-4 (production) for each VPS |------|-------------| | `--branch ` | Git branch to pull from | | `--no-pull` | Skip git pull, use existing source | +| `--pre-built` | Skip all Go compilation, use pre-built binaries already on disk | | `--restart` | Restart all services after upgrade | ### Node Join Flow diff --git a/pkg/cli/production/install/flags.go b/pkg/cli/production/install/flags.go index 42b344a..8b02127 100644 --- a/pkg/cli/production/install/flags.go +++ b/pkg/cli/production/install/flags.go @@ -16,6 +16,7 @@ type Flags struct { Force bool DryRun bool SkipChecks bool + PreBuilt bool // Skip building binaries, use pre-built binaries already on disk Nameserver bool // Make this node a nameserver (runs CoreDNS + Caddy) JoinAddress string // HTTPS URL of existing node (e.g., https://node1.dbrs.space) Token string // Invite token for joining (from orama invite) @@ -58,6 +59,7 @@ func ParseFlags(args []string) (*Flags, error) { fs.BoolVar(&flags.Force, "force", false, "Force reconfiguration even if already installed") fs.BoolVar(&flags.DryRun, "dry-run", false, "Show what would be done without making changes") fs.BoolVar(&flags.SkipChecks, "skip-checks", false, "Skip minimum resource checks (RAM/CPU)") + fs.BoolVar(&flags.PreBuilt, "pre-built", false, "Skip building binaries on VPS, use pre-built binaries already in /home/debros/bin and /usr/local/bin") fs.BoolVar(&flags.Nameserver, "nameserver", false, "Make this node a nameserver (runs CoreDNS + Caddy)") // Cluster join flags diff --git a/pkg/cli/production/install/orchestrator.go b/pkg/cli/production/install/orchestrator.go index 392c247..3e19ad9 100644 --- a/pkg/cli/production/install/orchestrator.go +++ b/pkg/cli/production/install/orchestrator.go @@ -44,7 +44,7 @@ func NewOrchestrator(flags *Flags) (*Orchestrator, error) { return nil, fmt.Errorf("invalid peers: %w", err) } - setup := production.NewProductionSetup(oramaHome, os.Stdout, flags.Force, flags.Branch, flags.NoPull, flags.SkipChecks) + setup := production.NewProductionSetup(oramaHome, os.Stdout, flags.Force, flags.Branch, flags.NoPull, flags.SkipChecks, flags.PreBuilt) setup.SetNameserver(flags.Nameserver) // Configure Anyone relay if enabled @@ -83,6 +83,12 @@ func (o *Orchestrator) Execute() error { fmt.Printf(" Using existing repository at /home/debros/src\n") } + // Inform user if using pre-built binaries + if o.flags.PreBuilt { + fmt.Printf(" ⚠️ --pre-built flag enabled: Skipping all Go compilation\n") + fmt.Printf(" Using pre-built binaries from /home/debros/bin and /usr/local/bin\n") + } + // Validate DNS if domain is provided o.validator.ValidateDNS() diff --git a/pkg/cli/production/upgrade/flags.go b/pkg/cli/production/upgrade/flags.go index 2d6b276..e35f814 100644 --- a/pkg/cli/production/upgrade/flags.go +++ b/pkg/cli/production/upgrade/flags.go @@ -11,6 +11,7 @@ type Flags struct { Force bool RestartServices bool NoPull bool + PreBuilt bool Branch string Nameserver *bool // Pointer so we can detect if explicitly set vs default @@ -35,6 +36,7 @@ func ParseFlags(args []string) (*Flags, error) { fs.BoolVar(&flags.Force, "force", false, "Reconfigure all settings") fs.BoolVar(&flags.RestartServices, "restart", false, "Automatically restart services after upgrade") fs.BoolVar(&flags.NoPull, "no-pull", false, "Skip source download, use existing /home/debros/src") + fs.BoolVar(&flags.PreBuilt, "pre-built", false, "Skip building binaries on VPS, use pre-built binaries already in /home/debros/bin and /usr/local/bin") fs.StringVar(&flags.Branch, "branch", "", "Git branch to use (uses saved preference if not specified)") // Nameserver flag - use pointer to detect if explicitly set diff --git a/pkg/cli/production/upgrade/orchestrator.go b/pkg/cli/production/upgrade/orchestrator.go index 09d3c50..b700a22 100644 --- a/pkg/cli/production/upgrade/orchestrator.go +++ b/pkg/cli/production/upgrade/orchestrator.go @@ -41,7 +41,7 @@ func NewOrchestrator(flags *Flags) *Orchestrator { isNameserver = *flags.Nameserver } - setup := production.NewProductionSetup(oramaHome, os.Stdout, flags.Force, branch, flags.NoPull, false) + setup := production.NewProductionSetup(oramaHome, os.Stdout, flags.Force, branch, flags.NoPull, false, flags.PreBuilt) setup.SetNameserver(isNameserver) // Configure Anyone relay if enabled @@ -78,6 +78,12 @@ func (o *Orchestrator) Execute() error { fmt.Printf(" Using existing repository at %s/src\n", o.oramaHome) } + // Log if --pre-built is enabled + if o.flags.PreBuilt { + fmt.Printf(" ⚠️ --pre-built flag enabled: Skipping all Go compilation\n") + fmt.Printf(" Using pre-built binaries from %s/bin and /usr/local/bin\n", o.oramaHome) + } + // Handle branch preferences if err := o.handleBranchPreferences(); err != nil { return err diff --git a/pkg/environments/production/orchestrator.go b/pkg/environments/production/orchestrator.go index f091feb..767f2ae 100644 --- a/pkg/environments/production/orchestrator.go +++ b/pkg/environments/production/orchestrator.go @@ -51,6 +51,7 @@ type ProductionSetup struct { binaryInstaller *BinaryInstaller branch string skipRepoUpdate bool + skipBuild bool // Skip all Go compilation (use pre-built binaries) NodePeerID string // Captured during Phase3 for later display } @@ -82,7 +83,7 @@ func SaveBranchPreference(oramaDir, branch string) error { } // NewProductionSetup creates a new production setup orchestrator -func NewProductionSetup(oramaHome string, logWriter io.Writer, forceReconfigure bool, branch string, skipRepoUpdate bool, skipResourceChecks bool) *ProductionSetup { +func NewProductionSetup(oramaHome string, logWriter io.Writer, forceReconfigure bool, branch string, skipRepoUpdate bool, skipResourceChecks bool, skipBuild bool) *ProductionSetup { oramaDir := filepath.Join(oramaHome, ".orama") arch, _ := (&ArchitectureDetector{}).Detect() @@ -99,6 +100,7 @@ func NewProductionSetup(oramaHome string, logWriter io.Writer, forceReconfigure arch: arch, branch: branch, skipRepoUpdate: skipRepoUpdate, + skipBuild: skipBuild, skipResourceChecks: skipResourceChecks, privChecker: &PrivilegeChecker{}, osDetector: &OSDetector{}, @@ -280,17 +282,94 @@ func (ps *ProductionSetup) Phase2ProvisionEnvironment() error { func (ps *ProductionSetup) Phase2bInstallBinaries() error { ps.logf("Phase 2b: Installing binaries...") - // Install system dependencies + // Install system dependencies (always needed for runtime libs) if err := ps.binaryInstaller.InstallSystemDependencies(); err != nil { ps.logf(" ⚠️ System dependencies warning: %v", err) } - // Install Go if not present - if err := ps.binaryInstaller.InstallGo(); err != nil { - return fmt.Errorf("failed to install Go: %w", err) + if ps.skipBuild { + // --pre-built mode: skip all Go compilation, verify binaries exist + ps.logf(" ℹ️ --pre-built mode: skipping Go installation and all compilation") + + // Verify required DeBros binaries exist + binDir := filepath.Join(ps.oramaHome, "bin") + requiredBins := []string{"orama-node", "gateway", "orama", "identity"} + for _, bin := range requiredBins { + binPath := filepath.Join(binDir, bin) + if _, err := os.Stat(binPath); os.IsNotExist(err) { + return fmt.Errorf("--pre-built: required binary not found at %s (run 'make build-linux' locally and copy to VPS)", binPath) + } + ps.logf(" ✓ Found %s", binPath) + } + + // Grant CAP_NET_BIND_SERVICE to orama-node + nodeBinary := filepath.Join(binDir, "orama-node") + if err := exec.Command("setcap", "cap_net_bind_service=+ep", nodeBinary).Run(); err != nil { + ps.logf(" ⚠️ Warning: failed to setcap on orama-node: %v", err) + } + + // Verify Olric + if _, err := exec.LookPath("olric-server"); err != nil { + // Check if it's in the bin dir + olricPath := filepath.Join(binDir, "olric-server") + if _, err := os.Stat(olricPath); os.IsNotExist(err) { + return fmt.Errorf("--pre-built: olric-server not found in PATH or %s", binDir) + } + // Copy to /usr/local/bin + if data, err := os.ReadFile(olricPath); err == nil { + os.WriteFile("/usr/local/bin/olric-server", data, 0755) + } + ps.logf(" ✓ Found %s", olricPath) + } else { + ps.logf(" ✓ olric-server already in PATH") + } + + // Verify CoreDNS and Caddy if nameserver + if ps.isNameserver { + if _, err := os.Stat("/usr/local/bin/coredns"); os.IsNotExist(err) { + return fmt.Errorf("--pre-built: coredns not found at /usr/local/bin/coredns") + } + ps.logf(" ✓ Found /usr/local/bin/coredns") + + if _, err := os.Stat("/usr/bin/caddy"); os.IsNotExist(err) { + return fmt.Errorf("--pre-built: caddy not found at /usr/bin/caddy") + } + ps.logf(" ✓ Found /usr/bin/caddy") + + // Grant CAP_NET_BIND_SERVICE to caddy + if err := exec.Command("setcap", "cap_net_bind_service=+ep", "/usr/bin/caddy").Run(); err != nil { + ps.logf(" ⚠️ Warning: failed to setcap on caddy: %v", err) + } + } + } else { + // Normal mode: install Go and build everything + if err := ps.binaryInstaller.InstallGo(); err != nil { + return fmt.Errorf("failed to install Go: %w", err) + } + + if err := ps.binaryInstaller.InstallOlric(); err != nil { + ps.logf(" ⚠️ Olric install warning: %v", err) + } + + // Install DeBros binaries (must be done before CoreDNS since we need the RQLite plugin source) + if err := ps.binaryInstaller.InstallDeBrosBinaries(ps.branch, ps.oramaHome, ps.skipRepoUpdate); err != nil { + return fmt.Errorf("failed to install DeBros binaries: %w", err) + } + + // Install CoreDNS and Caddy only if this is a nameserver node + if ps.isNameserver { + if err := ps.binaryInstaller.InstallCoreDNS(); err != nil { + ps.logf(" ⚠️ CoreDNS install warning: %v", err) + } + if err := ps.binaryInstaller.InstallCaddy(); err != nil { + ps.logf(" ⚠️ Caddy install warning: %v", err) + } + } else { + ps.logf(" ℹ️ Skipping CoreDNS/Caddy (not a nameserver node)") + } } - // Install binaries + // These are pre-built binary downloads (not Go compilation), always run them if err := ps.binaryInstaller.InstallRQLite(); err != nil { ps.logf(" ⚠️ RQLite install warning: %v", err) } @@ -303,11 +382,7 @@ func (ps *ProductionSetup) Phase2bInstallBinaries() error { ps.logf(" ⚠️ IPFS Cluster install warning: %v", err) } - if err := ps.binaryInstaller.InstallOlric(); err != nil { - ps.logf(" ⚠️ Olric install warning: %v", err) - } - - // Install Anyone (client or relay based on configuration) + // Install Anyone (client or relay based on configuration) — apt-based, not Go if ps.IsAnyoneRelay() { ps.logf(" Installing Anyone relay (operator mode)...") relayConfig := installers.AnyoneRelayConfig{ @@ -345,26 +420,6 @@ func (ps *ProductionSetup) Phase2bInstallBinaries() error { } } - // Install DeBros binaries (must be done before CoreDNS since we need the RQLite plugin source) - if err := ps.binaryInstaller.InstallDeBrosBinaries(ps.branch, ps.oramaHome, ps.skipRepoUpdate); err != nil { - return fmt.Errorf("failed to install DeBros binaries: %w", err) - } - - // Install CoreDNS and Caddy only if this is a nameserver node - if ps.isNameserver { - // Install CoreDNS with RQLite plugin (for dynamic DNS records and ACME challenges) - if err := ps.binaryInstaller.InstallCoreDNS(); err != nil { - ps.logf(" ⚠️ CoreDNS install warning: %v", err) - } - - // Install Caddy with orama DNS module (for SSL certificate management) - if err := ps.binaryInstaller.InstallCaddy(); err != nil { - ps.logf(" ⚠️ Caddy install warning: %v", err) - } - } else { - ps.logf(" ℹ️ Skipping CoreDNS/Caddy (not a nameserver node)") - } - ps.logf(" ✓ All binaries installed") return nil }