Made building faster

This commit is contained in:
anonpenguin23 2026-02-01 16:15:37 +02:00
parent c401fdcd74
commit 683ce50106
7 changed files with 205 additions and 33 deletions

View File

@ -103,6 +103,66 @@ build: deps
go build -ldflags "$(LDFLAGS) -X 'github.com/DeBrosOfficial/network/pkg/gateway.BuildVersion=$(VERSION)' -X 'github.com/DeBrosOfficial/network/pkg/gateway.BuildCommit=$(COMMIT)' -X 'github.com/DeBrosOfficial/network/pkg/gateway.BuildTime=$(DATE)'" -o bin/gateway ./cmd/gateway
@echo "Build complete! Run ./bin/orama version"
# Cross-compile all binaries for Linux (used with --pre-built flag on VPS)
# Builds: DeBros binaries + Olric + CoreDNS (with rqlite plugin) + Caddy (with orama DNS module)
build-linux: deps
@echo "Cross-compiling all binaries for linux/amd64 (version=$(VERSION))..."
@mkdir -p bin-linux
GOOS=linux GOARCH=amd64 go build -ldflags "$(LDFLAGS)" -o bin-linux/identity ./cmd/identity
GOOS=linux GOARCH=amd64 go build -ldflags "$(LDFLAGS)" -o bin-linux/orama-node ./cmd/node
GOOS=linux GOARCH=amd64 go build -ldflags "$(LDFLAGS)" -o bin-linux/orama cmd/cli/main.go
GOOS=linux GOARCH=amd64 go build -ldflags "$(LDFLAGS)" -o bin-linux/rqlite-mcp ./cmd/rqlite-mcp
GOOS=linux GOARCH=amd64 go build -ldflags "$(LDFLAGS) -X 'github.com/DeBrosOfficial/network/pkg/gateway.BuildVersion=$(VERSION)' -X 'github.com/DeBrosOfficial/network/pkg/gateway.BuildCommit=$(COMMIT)' -X 'github.com/DeBrosOfficial/network/pkg/gateway.BuildTime=$(DATE)'" -o bin-linux/gateway ./cmd/gateway
GOOS=linux GOARCH=amd64 go build -ldflags "$(LDFLAGS)" -o bin-linux/orama-cli ./cmd/cli
@echo "Installing Olric for linux/amd64..."
GOOS=linux GOARCH=amd64 GOBIN=$(CURDIR)/bin-linux go install github.com/olric-data/olric/cmd/olric-server@v0.7.0
@echo "✓ All Linux binaries built in bin-linux/"
@echo ""
@echo "Next steps:"
@echo " 1. Build CoreDNS: make build-linux-coredns"
@echo " 2. Build Caddy: make build-linux-caddy"
@echo " 3. Or build all: make build-linux-all"
# Build CoreDNS with rqlite plugin for Linux
build-linux-coredns:
@echo "Building CoreDNS with rqlite plugin for linux/amd64..."
@rm -rf /tmp/coredns-build-linux
@mkdir -p /tmp/coredns-build-linux
git clone --depth 1 --branch v1.12.0 https://github.com/coredns/coredns.git /tmp/coredns-build-linux
@mkdir -p /tmp/coredns-build-linux/plugin/rqlite
@cp pkg/coredns/rqlite/*.go /tmp/coredns-build-linux/plugin/rqlite/
@echo 'metadata:metadata\ncancel:cancel\ntls:tls\nreload:reload\nnsid:nsid\nbufsize:bufsize\nroot:root\nbind:bind\ndebug:debug\ntrace:trace\nready:ready\nhealth:health\npprof:pprof\nprometheus:metrics\nerrors:errors\nlog:log\ndnstap:dnstap\nlocal:local\ndns64:dns64\nacl:acl\nany:any\nchaos:chaos\nloadbalance:loadbalance\ncache:cache\nrewrite:rewrite\nheader:header\ndnssec:dnssec\nautopath:autopath\nminimal:minimal\ntemplate:template\ntransfer:transfer\nhosts:hosts\nfile:file\nauto:auto\nsecondary:secondary\nloop:loop\nforward:forward\ngrpc:grpc\nerratic:erratic\nwhoami:whoami\non:github.com/coredns/caddy/onevent\nsign:sign\nview:view\nrqlite:rqlite' > /tmp/coredns-build-linux/plugin.cfg
cd /tmp/coredns-build-linux && go get github.com/miekg/dns@latest && go get go.uber.org/zap@latest && go mod tidy && go generate && GOOS=linux GOARCH=amd64 CGO_ENABLED=0 go build -o coredns
@cp /tmp/coredns-build-linux/coredns bin-linux/coredns
@rm -rf /tmp/coredns-build-linux
@echo "✓ CoreDNS built: bin-linux/coredns"
# Build Caddy with orama DNS module for Linux
build-linux-caddy:
@echo "Building Caddy with orama DNS module for linux/amd64..."
@which xcaddy > /dev/null 2>&1 || (echo "Installing xcaddy..." && go install github.com/caddyserver/xcaddy/cmd/xcaddy@latest)
@rm -rf /tmp/caddy-build-linux
@mkdir -p /tmp/caddy-build-linux/caddy-dns-orama
@# Generate the module source
@echo 'module github.com/DeBrosOfficial/caddy-dns-orama\n\ngo 1.22\n\nrequire (\n\tgithub.com/caddyserver/caddy/v2 v2.10.2\n\tgithub.com/libdns/libdns v1.1.0\n)' > /tmp/caddy-build-linux/caddy-dns-orama/go.mod
@cp pkg/caddy/dns/orama/provider.go /tmp/caddy-build-linux/caddy-dns-orama/ 2>/dev/null || true
cd /tmp/caddy-build-linux/caddy-dns-orama && go mod tidy
GOOS=linux GOARCH=amd64 xcaddy build v2.10.2 --with github.com/DeBrosOfficial/caddy-dns-orama=/tmp/caddy-build-linux/caddy-dns-orama --output bin-linux/caddy
@rm -rf /tmp/caddy-build-linux
@echo "✓ Caddy built: bin-linux/caddy"
# Build everything for Linux (all binaries + CoreDNS + Caddy)
build-linux-all: build-linux build-linux-coredns build-linux-caddy
@echo ""
@echo "✅ All Linux binaries ready in bin-linux/:"
@ls -la bin-linux/
@echo ""
@echo "Deploy to VPS:"
@echo " scp bin-linux/* ubuntu@<ip>:/home/debros/bin/"
@echo " scp bin-linux/coredns ubuntu@<ip>:/usr/local/bin/coredns"
@echo " scp bin-linux/caddy ubuntu@<ip>:/usr/bin/caddy"
@echo " sudo orama install --pre-built --no-pull ..."
# Install git hooks
install-hooks:
@echo "Installing git hooks..."

View File

@ -70,6 +70,45 @@ sudo mv /tmp/orama /usr/local/bin/orama && sudo chmod +x /usr/local/bin/orama
sudo orama upgrade --no-pull --restart
```
### Development Deployment with Pre-Built Binaries (Fastest)
Cross-compile everything locally and skip all Go compilation on the VPS. This is significantly faster because your local machine compiles much faster than the VPS.
```bash
# 1. Cross-compile all binaries for Linux (DeBros + Olric + CoreDNS + Caddy)
make build-linux-all
# Outputs everything to bin-linux/
# 2. Generate a source archive (still needed for configs, templates, etc.)
./scripts/generate-source-archive.sh
# 3. Copy everything to the VPS
sshpass -p '<password>' scp -o StrictHostKeyChecking=no bin-linux/orama ubuntu@<ip>:/tmp/orama
sshpass -p '<password>' scp -o StrictHostKeyChecking=no /tmp/network-source.tar.gz ubuntu@<ip>:/tmp/
# 4. On the VPS: extract source, install CLI, and copy pre-built binaries
ssh ubuntu@<ip>
sudo rm -rf /home/debros/src && sudo mkdir -p /home/debros/src
sudo tar xzf /tmp/network-source.tar.gz -C /home/debros/src
sudo chown -R debros:debros /home/debros/src
sudo mv /tmp/orama /usr/local/bin/orama && sudo chmod +x /usr/local/bin/orama
# 5. Copy pre-built binaries to their expected locations
# (from your local machine)
sshpass -p '<password>' scp -o StrictHostKeyChecking=no bin-linux/orama-node bin-linux/gateway bin-linux/identity bin-linux/rqlite-mcp bin-linux/olric-server ubuntu@<ip>:/home/debros/bin/
sshpass -p '<password>' scp -o StrictHostKeyChecking=no bin-linux/coredns ubuntu@<ip>:/usr/local/bin/coredns
sshpass -p '<password>' scp -o StrictHostKeyChecking=no bin-linux/caddy ubuntu@<ip>:/usr/bin/caddy
# 6. Install/upgrade with --pre-built (skips ALL Go compilation on VPS)
sudo orama install --no-pull --pre-built --vps-ip <ip> ...
# or
sudo orama upgrade --no-pull --pre-built --restart
```
**What `--pre-built` skips:** Go installation, `make build`, Olric `go install`, CoreDNS build, Caddy/xcaddy build.
**What `--pre-built` still runs:** apt dependencies, RQLite/IPFS/IPFS Cluster downloads (pre-built binary downloads, fast), Anyone relay setup, config generation, systemd service creation.
### Production Deployment (Via Git)
For production releases — pulls source from GitHub on the VPS.
@ -107,6 +146,7 @@ To deploy to all nodes, repeat steps 3-5 (dev) or 3-4 (production) for each VPS
| `--token <token>` | Invite token for joining (from `orama invite` on existing node) |
| `--branch <branch>` | Git branch to use (default: main) |
| `--no-pull` | Skip git clone/pull, use existing `/home/debros/src` |
| `--pre-built` | Skip all Go compilation, use pre-built binaries already on disk (see above) |
| `--force` | Force reconfiguration even if already installed |
| `--skip-firewall` | Skip UFW firewall setup |
| `--skip-checks` | Skip minimum resource checks (RAM/CPU) |
@ -137,6 +177,7 @@ To deploy to all nodes, repeat steps 3-5 (dev) or 3-4 (production) for each VPS
|------|-------------|
| `--branch <branch>` | Git branch to pull from |
| `--no-pull` | Skip git pull, use existing source |
| `--pre-built` | Skip all Go compilation, use pre-built binaries already on disk |
| `--restart` | Restart all services after upgrade |
### Node Join Flow

View File

@ -16,6 +16,7 @@ type Flags struct {
Force bool
DryRun bool
SkipChecks bool
PreBuilt bool // Skip building binaries, use pre-built binaries already on disk
Nameserver bool // Make this node a nameserver (runs CoreDNS + Caddy)
JoinAddress string // HTTPS URL of existing node (e.g., https://node1.dbrs.space)
Token string // Invite token for joining (from orama invite)
@ -58,6 +59,7 @@ func ParseFlags(args []string) (*Flags, error) {
fs.BoolVar(&flags.Force, "force", false, "Force reconfiguration even if already installed")
fs.BoolVar(&flags.DryRun, "dry-run", false, "Show what would be done without making changes")
fs.BoolVar(&flags.SkipChecks, "skip-checks", false, "Skip minimum resource checks (RAM/CPU)")
fs.BoolVar(&flags.PreBuilt, "pre-built", false, "Skip building binaries on VPS, use pre-built binaries already in /home/debros/bin and /usr/local/bin")
fs.BoolVar(&flags.Nameserver, "nameserver", false, "Make this node a nameserver (runs CoreDNS + Caddy)")
// Cluster join flags

View File

@ -44,7 +44,7 @@ func NewOrchestrator(flags *Flags) (*Orchestrator, error) {
return nil, fmt.Errorf("invalid peers: %w", err)
}
setup := production.NewProductionSetup(oramaHome, os.Stdout, flags.Force, flags.Branch, flags.NoPull, flags.SkipChecks)
setup := production.NewProductionSetup(oramaHome, os.Stdout, flags.Force, flags.Branch, flags.NoPull, flags.SkipChecks, flags.PreBuilt)
setup.SetNameserver(flags.Nameserver)
// Configure Anyone relay if enabled
@ -83,6 +83,12 @@ func (o *Orchestrator) Execute() error {
fmt.Printf(" Using existing repository at /home/debros/src\n")
}
// Inform user if using pre-built binaries
if o.flags.PreBuilt {
fmt.Printf(" ⚠️ --pre-built flag enabled: Skipping all Go compilation\n")
fmt.Printf(" Using pre-built binaries from /home/debros/bin and /usr/local/bin\n")
}
// Validate DNS if domain is provided
o.validator.ValidateDNS()

View File

@ -11,6 +11,7 @@ type Flags struct {
Force bool
RestartServices bool
NoPull bool
PreBuilt bool
Branch string
Nameserver *bool // Pointer so we can detect if explicitly set vs default
@ -35,6 +36,7 @@ func ParseFlags(args []string) (*Flags, error) {
fs.BoolVar(&flags.Force, "force", false, "Reconfigure all settings")
fs.BoolVar(&flags.RestartServices, "restart", false, "Automatically restart services after upgrade")
fs.BoolVar(&flags.NoPull, "no-pull", false, "Skip source download, use existing /home/debros/src")
fs.BoolVar(&flags.PreBuilt, "pre-built", false, "Skip building binaries on VPS, use pre-built binaries already in /home/debros/bin and /usr/local/bin")
fs.StringVar(&flags.Branch, "branch", "", "Git branch to use (uses saved preference if not specified)")
// Nameserver flag - use pointer to detect if explicitly set

View File

@ -41,7 +41,7 @@ func NewOrchestrator(flags *Flags) *Orchestrator {
isNameserver = *flags.Nameserver
}
setup := production.NewProductionSetup(oramaHome, os.Stdout, flags.Force, branch, flags.NoPull, false)
setup := production.NewProductionSetup(oramaHome, os.Stdout, flags.Force, branch, flags.NoPull, false, flags.PreBuilt)
setup.SetNameserver(isNameserver)
// Configure Anyone relay if enabled
@ -78,6 +78,12 @@ func (o *Orchestrator) Execute() error {
fmt.Printf(" Using existing repository at %s/src\n", o.oramaHome)
}
// Log if --pre-built is enabled
if o.flags.PreBuilt {
fmt.Printf(" ⚠️ --pre-built flag enabled: Skipping all Go compilation\n")
fmt.Printf(" Using pre-built binaries from %s/bin and /usr/local/bin\n", o.oramaHome)
}
// Handle branch preferences
if err := o.handleBranchPreferences(); err != nil {
return err

View File

@ -51,6 +51,7 @@ type ProductionSetup struct {
binaryInstaller *BinaryInstaller
branch string
skipRepoUpdate bool
skipBuild bool // Skip all Go compilation (use pre-built binaries)
NodePeerID string // Captured during Phase3 for later display
}
@ -82,7 +83,7 @@ func SaveBranchPreference(oramaDir, branch string) error {
}
// NewProductionSetup creates a new production setup orchestrator
func NewProductionSetup(oramaHome string, logWriter io.Writer, forceReconfigure bool, branch string, skipRepoUpdate bool, skipResourceChecks bool) *ProductionSetup {
func NewProductionSetup(oramaHome string, logWriter io.Writer, forceReconfigure bool, branch string, skipRepoUpdate bool, skipResourceChecks bool, skipBuild bool) *ProductionSetup {
oramaDir := filepath.Join(oramaHome, ".orama")
arch, _ := (&ArchitectureDetector{}).Detect()
@ -99,6 +100,7 @@ func NewProductionSetup(oramaHome string, logWriter io.Writer, forceReconfigure
arch: arch,
branch: branch,
skipRepoUpdate: skipRepoUpdate,
skipBuild: skipBuild,
skipResourceChecks: skipResourceChecks,
privChecker: &PrivilegeChecker{},
osDetector: &OSDetector{},
@ -280,17 +282,94 @@ func (ps *ProductionSetup) Phase2ProvisionEnvironment() error {
func (ps *ProductionSetup) Phase2bInstallBinaries() error {
ps.logf("Phase 2b: Installing binaries...")
// Install system dependencies
// Install system dependencies (always needed for runtime libs)
if err := ps.binaryInstaller.InstallSystemDependencies(); err != nil {
ps.logf(" ⚠️ System dependencies warning: %v", err)
}
// Install Go if not present
if err := ps.binaryInstaller.InstallGo(); err != nil {
return fmt.Errorf("failed to install Go: %w", err)
if ps.skipBuild {
// --pre-built mode: skip all Go compilation, verify binaries exist
ps.logf(" --pre-built mode: skipping Go installation and all compilation")
// Verify required DeBros binaries exist
binDir := filepath.Join(ps.oramaHome, "bin")
requiredBins := []string{"orama-node", "gateway", "orama", "identity"}
for _, bin := range requiredBins {
binPath := filepath.Join(binDir, bin)
if _, err := os.Stat(binPath); os.IsNotExist(err) {
return fmt.Errorf("--pre-built: required binary not found at %s (run 'make build-linux' locally and copy to VPS)", binPath)
}
ps.logf(" ✓ Found %s", binPath)
}
// Grant CAP_NET_BIND_SERVICE to orama-node
nodeBinary := filepath.Join(binDir, "orama-node")
if err := exec.Command("setcap", "cap_net_bind_service=+ep", nodeBinary).Run(); err != nil {
ps.logf(" ⚠️ Warning: failed to setcap on orama-node: %v", err)
}
// Verify Olric
if _, err := exec.LookPath("olric-server"); err != nil {
// Check if it's in the bin dir
olricPath := filepath.Join(binDir, "olric-server")
if _, err := os.Stat(olricPath); os.IsNotExist(err) {
return fmt.Errorf("--pre-built: olric-server not found in PATH or %s", binDir)
}
// Copy to /usr/local/bin
if data, err := os.ReadFile(olricPath); err == nil {
os.WriteFile("/usr/local/bin/olric-server", data, 0755)
}
ps.logf(" ✓ Found %s", olricPath)
} else {
ps.logf(" ✓ olric-server already in PATH")
}
// Verify CoreDNS and Caddy if nameserver
if ps.isNameserver {
if _, err := os.Stat("/usr/local/bin/coredns"); os.IsNotExist(err) {
return fmt.Errorf("--pre-built: coredns not found at /usr/local/bin/coredns")
}
ps.logf(" ✓ Found /usr/local/bin/coredns")
if _, err := os.Stat("/usr/bin/caddy"); os.IsNotExist(err) {
return fmt.Errorf("--pre-built: caddy not found at /usr/bin/caddy")
}
ps.logf(" ✓ Found /usr/bin/caddy")
// Grant CAP_NET_BIND_SERVICE to caddy
if err := exec.Command("setcap", "cap_net_bind_service=+ep", "/usr/bin/caddy").Run(); err != nil {
ps.logf(" ⚠️ Warning: failed to setcap on caddy: %v", err)
}
}
} else {
// Normal mode: install Go and build everything
if err := ps.binaryInstaller.InstallGo(); err != nil {
return fmt.Errorf("failed to install Go: %w", err)
}
if err := ps.binaryInstaller.InstallOlric(); err != nil {
ps.logf(" ⚠️ Olric install warning: %v", err)
}
// Install DeBros binaries (must be done before CoreDNS since we need the RQLite plugin source)
if err := ps.binaryInstaller.InstallDeBrosBinaries(ps.branch, ps.oramaHome, ps.skipRepoUpdate); err != nil {
return fmt.Errorf("failed to install DeBros binaries: %w", err)
}
// Install CoreDNS and Caddy only if this is a nameserver node
if ps.isNameserver {
if err := ps.binaryInstaller.InstallCoreDNS(); err != nil {
ps.logf(" ⚠️ CoreDNS install warning: %v", err)
}
if err := ps.binaryInstaller.InstallCaddy(); err != nil {
ps.logf(" ⚠️ Caddy install warning: %v", err)
}
} else {
ps.logf(" Skipping CoreDNS/Caddy (not a nameserver node)")
}
}
// Install binaries
// These are pre-built binary downloads (not Go compilation), always run them
if err := ps.binaryInstaller.InstallRQLite(); err != nil {
ps.logf(" ⚠️ RQLite install warning: %v", err)
}
@ -303,11 +382,7 @@ func (ps *ProductionSetup) Phase2bInstallBinaries() error {
ps.logf(" ⚠️ IPFS Cluster install warning: %v", err)
}
if err := ps.binaryInstaller.InstallOlric(); err != nil {
ps.logf(" ⚠️ Olric install warning: %v", err)
}
// Install Anyone (client or relay based on configuration)
// Install Anyone (client or relay based on configuration) — apt-based, not Go
if ps.IsAnyoneRelay() {
ps.logf(" Installing Anyone relay (operator mode)...")
relayConfig := installers.AnyoneRelayConfig{
@ -345,26 +420,6 @@ func (ps *ProductionSetup) Phase2bInstallBinaries() error {
}
}
// Install DeBros binaries (must be done before CoreDNS since we need the RQLite plugin source)
if err := ps.binaryInstaller.InstallDeBrosBinaries(ps.branch, ps.oramaHome, ps.skipRepoUpdate); err != nil {
return fmt.Errorf("failed to install DeBros binaries: %w", err)
}
// Install CoreDNS and Caddy only if this is a nameserver node
if ps.isNameserver {
// Install CoreDNS with RQLite plugin (for dynamic DNS records and ACME challenges)
if err := ps.binaryInstaller.InstallCoreDNS(); err != nil {
ps.logf(" ⚠️ CoreDNS install warning: %v", err)
}
// Install Caddy with orama DNS module (for SSL certificate management)
if err := ps.binaryInstaller.InstallCaddy(); err != nil {
ps.logf(" ⚠️ Caddy install warning: %v", err)
}
} else {
ps.logf(" Skipping CoreDNS/Caddy (not a nameserver node)")
}
ps.logf(" ✓ All binaries installed")
return nil
}