diff --git a/.gitignore b/.gitignore index 471a2b8..f25c04c 100644 --- a/.gitignore +++ b/.gitignore @@ -83,4 +83,6 @@ configs/ .cursor/ # Remote node credentials -scripts/remote-nodes.conf \ No newline at end of file +scripts/remote-nodes.conf + +orama-cli-linux \ No newline at end of file diff --git a/cli b/cli new file mode 100755 index 0000000..c6c8605 Binary files /dev/null and b/cli differ diff --git a/pkg/cli/production/install/flags.go b/pkg/cli/production/install/flags.go index 76b0dfa..3372af3 100644 --- a/pkg/cli/production/install/flags.go +++ b/pkg/cli/production/install/flags.go @@ -15,6 +15,7 @@ type Flags struct { Force bool DryRun bool SkipChecks bool + Nameserver bool // Make this node a nameserver (runs CoreDNS + Caddy) JoinAddress string ClusterSecret string SwarmKey string @@ -41,6 +42,7 @@ func ParseFlags(args []string) (*Flags, error) { fs.BoolVar(&flags.Force, "force", false, "Force reconfiguration even if already installed") fs.BoolVar(&flags.DryRun, "dry-run", false, "Show what would be done without making changes") fs.BoolVar(&flags.SkipChecks, "skip-checks", false, "Skip minimum resource checks (RAM/CPU)") + fs.BoolVar(&flags.Nameserver, "nameserver", false, "Make this node a nameserver (runs CoreDNS + Caddy)") // Cluster join flags fs.StringVar(&flags.JoinAddress, "join", "", "Join an existing cluster (e.g. 1.2.3.4:7001)") diff --git a/pkg/cli/production/install/orchestrator.go b/pkg/cli/production/install/orchestrator.go index bedb719..c635dcb 100644 --- a/pkg/cli/production/install/orchestrator.go +++ b/pkg/cli/production/install/orchestrator.go @@ -32,6 +32,7 @@ func NewOrchestrator(flags *Flags) (*Orchestrator, error) { } setup := production.NewProductionSetup(oramaHome, os.Stdout, flags.Force, flags.Branch, flags.NoPull, flags.SkipChecks) + setup.SetNameserver(flags.Nameserver) validator := NewValidator(flags, oramaDir) return &Orchestrator{ @@ -68,9 +69,16 @@ func (o *Orchestrator) Execute() error { return err } - // Save branch preference for future upgrades - if err := production.SaveBranchPreference(o.oramaDir, o.flags.Branch); err != nil { - fmt.Fprintf(os.Stderr, "⚠️ Warning: Failed to save branch preference: %v\n", err) + // Save preferences for future upgrades (branch + nameserver) + prefs := &production.NodePreferences{ + Branch: o.flags.Branch, + Nameserver: o.flags.Nameserver, + } + if err := production.SavePreferences(o.oramaDir, prefs); err != nil { + fmt.Fprintf(os.Stderr, "⚠️ Warning: Failed to save preferences: %v\n", err) + } + if o.flags.Nameserver { + fmt.Printf(" ℹ️ This node will be a nameserver (CoreDNS + Caddy)\n") } // Phase 1: Check prerequisites diff --git a/pkg/cli/production/upgrade/flags.go b/pkg/cli/production/upgrade/flags.go index 6277267..892a134 100644 --- a/pkg/cli/production/upgrade/flags.go +++ b/pkg/cli/production/upgrade/flags.go @@ -12,6 +12,7 @@ type Flags struct { RestartServices bool NoPull bool Branch string + Nameserver *bool // Pointer so we can detect if explicitly set vs default } // ParseFlags parses upgrade command flags @@ -23,8 +24,11 @@ func ParseFlags(args []string) (*Flags, error) { fs.BoolVar(&flags.Force, "force", false, "Reconfigure all settings") fs.BoolVar(&flags.RestartServices, "restart", false, "Automatically restart services after upgrade") - fs.BoolVar(&flags.NoPull, "no-pull", false, "Skip git clone/pull, use existing /home/debros/src") - fs.StringVar(&flags.Branch, "branch", "", "Git branch to use (main or nightly, uses saved preference if not specified)") + fs.BoolVar(&flags.NoPull, "no-pull", false, "Skip source download, use existing /home/debros/src") + fs.StringVar(&flags.Branch, "branch", "", "Git branch to use (uses saved preference if not specified)") + + // Nameserver flag - use pointer to detect if explicitly set + nameserver := fs.Bool("nameserver", false, "Make this node a nameserver (uses saved preference if not specified)") // Support legacy flags for backwards compatibility nightly := fs.Bool("nightly", false, "Use nightly branch (deprecated, use --branch nightly)") @@ -45,9 +49,9 @@ func ParseFlags(args []string) (*Flags, error) { flags.Branch = "main" } - // Validate branch if provided - if flags.Branch != "" && flags.Branch != "main" && flags.Branch != "nightly" { - return nil, fmt.Errorf("invalid branch: %s (must be 'main' or 'nightly')", flags.Branch) + // Set nameserver if explicitly provided + if *nameserver { + flags.Nameserver = nameserver } return flags, nil diff --git a/pkg/cli/production/upgrade/orchestrator.go b/pkg/cli/production/upgrade/orchestrator.go index 2b3a042..d12c27e 100644 --- a/pkg/cli/production/upgrade/orchestrator.go +++ b/pkg/cli/production/upgrade/orchestrator.go @@ -25,7 +25,24 @@ type Orchestrator struct { func NewOrchestrator(flags *Flags) *Orchestrator { oramaHome := "/home/debros" oramaDir := oramaHome + "/.orama" - setup := production.NewProductionSetup(oramaHome, os.Stdout, flags.Force, flags.Branch, flags.NoPull, false) + + // Load existing preferences + prefs := production.LoadPreferences(oramaDir) + + // Use saved branch if not specified + branch := flags.Branch + if branch == "" { + branch = prefs.Branch + } + + // Use saved nameserver preference if not explicitly specified + isNameserver := prefs.Nameserver + if flags.Nameserver != nil { + isNameserver = *flags.Nameserver + } + + setup := production.NewProductionSetup(oramaHome, os.Stdout, flags.Force, branch, flags.NoPull, false) + setup.SetNameserver(isNameserver) return &Orchestrator{ oramaHome: oramaHome, @@ -132,31 +149,50 @@ func (o *Orchestrator) Execute() error { } func (o *Orchestrator) handleBranchPreferences() error { - // If branch was explicitly provided, save it for future upgrades + // Load current preferences + prefs := production.LoadPreferences(o.oramaDir) + prefsChanged := false + + // If branch was explicitly provided, update it if o.flags.Branch != "" { - if err := production.SaveBranchPreference(o.oramaDir, o.flags.Branch); err != nil { - fmt.Fprintf(os.Stderr, "⚠️ Warning: Failed to save branch preference: %v\n", err) - } else { - fmt.Printf(" Using branch: %s (saved for future upgrades)\n", o.flags.Branch) - } + prefs.Branch = o.flags.Branch + prefsChanged = true + fmt.Printf(" Using branch: %s (saved for future upgrades)\n", o.flags.Branch) } else { - // Show which branch is being used (read from saved preference) - currentBranch := production.ReadBranchPreference(o.oramaDir) - fmt.Printf(" Using branch: %s (from saved preference)\n", currentBranch) + fmt.Printf(" Using branch: %s (from saved preference)\n", prefs.Branch) + } + + // If nameserver was explicitly provided, update it + if o.flags.Nameserver != nil { + prefs.Nameserver = *o.flags.Nameserver + prefsChanged = true + } + if o.setup.IsNameserver() { + fmt.Printf(" Nameserver mode: enabled (CoreDNS + Caddy)\n") + } + + // Save preferences if anything changed + if prefsChanged { + if err := production.SavePreferences(o.oramaDir, prefs); err != nil { + fmt.Fprintf(os.Stderr, "⚠️ Warning: Failed to save preferences: %v\n", err) + } } return nil } func (o *Orchestrator) stopServices() error { - fmt.Printf("\n⏹️ Stopping services before upgrade...\n") + fmt.Printf("\n⏹️ Stopping all services before upgrade...\n") serviceController := production.NewSystemdController() + // Stop services in reverse dependency order services := []string{ - "debros-gateway.service", - "debros-node.service", - "debros-ipfs-cluster.service", - "debros-ipfs.service", - // Note: RQLite is managed by node process, not as separate service - "debros-olric.service", + "caddy.service", // Depends on node + "coredns.service", // Depends on node + "debros-gateway.service", // Legacy + "debros-node.service", // Depends on cluster, olric + "debros-ipfs-cluster.service", // Depends on IPFS + "debros-ipfs.service", // Base IPFS + "debros-olric.service", // Independent + "debros-anyone-client.service", // Independent } for _, svc := range services { unitPath := filepath.Join("/etc/systemd/system", svc) @@ -169,7 +205,7 @@ func (o *Orchestrator) stopServices() error { } } // Give services time to shut down gracefully - time.Sleep(2 * time.Second) + time.Sleep(3 * time.Second) return nil } diff --git a/pkg/environments/production/installers/gateway.go b/pkg/environments/production/installers/gateway.go index d5f57e8..6600de2 100644 --- a/pkg/environments/production/installers/gateway.go +++ b/pkg/environments/production/installers/gateway.go @@ -39,7 +39,77 @@ func (gi *GatewayInstaller) Configure() error { return nil } -// InstallDeBrosBinaries clones and builds DeBros binaries +// downloadSourceZIP downloads source code as ZIP from GitHub +// This is simpler and more reliable than git clone with shallow clones +func (gi *GatewayInstaller) downloadSourceZIP(branch string, srcDir string) error { + // GitHub archive URL format + zipURL := fmt.Sprintf("https://github.com/DeBrosOfficial/network/archive/refs/heads/%s.zip", branch) + zipPath := "/tmp/network-source.zip" + extractDir := "/tmp/network-extract" + + // Clean up any previous download artifacts + os.RemoveAll(zipPath) + os.RemoveAll(extractDir) + + // Download ZIP + fmt.Fprintf(gi.logWriter, " Downloading source (branch: %s)...\n", branch) + if err := DownloadFile(zipURL, zipPath); err != nil { + return fmt.Errorf("failed to download source from %s: %w", zipURL, err) + } + + // Create extraction directory + if err := os.MkdirAll(extractDir, 0755); err != nil { + return fmt.Errorf("failed to create extraction directory: %w", err) + } + + // Extract ZIP + fmt.Fprintf(gi.logWriter, " Extracting source...\n") + extractCmd := exec.Command("unzip", "-q", "-o", zipPath, "-d", extractDir) + if output, err := extractCmd.CombinedOutput(); err != nil { + return fmt.Errorf("failed to extract source: %w\n%s", err, string(output)) + } + + // GitHub extracts to network-{branch}/ directory + extractedDir := filepath.Join(extractDir, fmt.Sprintf("network-%s", branch)) + + // Verify extracted directory exists + if _, err := os.Stat(extractedDir); os.IsNotExist(err) { + // Try alternative naming (GitHub may sanitize branch names) + entries, _ := os.ReadDir(extractDir) + if len(entries) == 1 && entries[0].IsDir() { + extractedDir = filepath.Join(extractDir, entries[0].Name()) + } else { + return fmt.Errorf("extracted directory not found at %s", extractedDir) + } + } + + // Remove existing source directory + os.RemoveAll(srcDir) + + // Move extracted content to source directory + if err := os.Rename(extractedDir, srcDir); err != nil { + // Cross-filesystem fallback: copy instead of rename + fmt.Fprintf(gi.logWriter, " Moving source (cross-filesystem copy)...\n") + copyCmd := exec.Command("cp", "-r", extractedDir, srcDir) + if output, err := copyCmd.CombinedOutput(); err != nil { + return fmt.Errorf("failed to move source: %w\n%s", err, string(output)) + } + } + + // Cleanup temp files + os.RemoveAll(zipPath) + os.RemoveAll(extractDir) + + // Fix ownership + if err := exec.Command("chown", "-R", "debros:debros", srcDir).Run(); err != nil { + fmt.Fprintf(gi.logWriter, " ⚠️ Warning: failed to chown source directory: %v\n", err) + } + + fmt.Fprintf(gi.logWriter, " ✓ Source downloaded\n") + return nil +} + +// InstallDeBrosBinaries downloads and builds DeBros binaries func (gi *GatewayInstaller) InstallDeBrosBinaries(branch string, oramaHome string, skipRepoUpdate bool) error { fmt.Fprintf(gi.logWriter, " Building DeBros binaries...\n") @@ -54,45 +124,23 @@ func (gi *GatewayInstaller) InstallDeBrosBinaries(branch string, oramaHome strin return fmt.Errorf("failed to create bin directory %s: %w", binDir, err) } - // Check if source directory has content (either git repo or pre-existing source) + // Check if source directory has content hasSourceContent := false if entries, err := os.ReadDir(srcDir); err == nil && len(entries) > 0 { hasSourceContent = true } - // Check if git repository is already initialized - isGitRepo := false - if _, err := os.Stat(filepath.Join(srcDir, ".git")); err == nil { - isGitRepo = true - } - - // Handle repository update/clone based on skipRepoUpdate flag + // Handle repository update/download based on skipRepoUpdate flag if skipRepoUpdate { - fmt.Fprintf(gi.logWriter, " Skipping repo clone/pull (--no-pull flag)\n") + fmt.Fprintf(gi.logWriter, " Skipping source download (--no-pull flag)\n") if !hasSourceContent { - return fmt.Errorf("cannot skip pull: source directory is empty at %s (need to populate it first)", srcDir) + return fmt.Errorf("cannot skip download: source directory is empty at %s (need to populate it first)", srcDir) } - fmt.Fprintf(gi.logWriter, " Using existing source at %s (skipping git operations)\n", srcDir) - // Skip to build step - don't execute any git commands + fmt.Fprintf(gi.logWriter, " Using existing source at %s\n", srcDir) } else { - // Clone repository if not present, otherwise update it - if !isGitRepo { - fmt.Fprintf(gi.logWriter, " Cloning repository...\n") - cmd := exec.Command("git", "clone", "--branch", branch, "--depth", "1", "https://github.com/DeBrosOfficial/network.git", srcDir) - if err := cmd.Run(); err != nil { - return fmt.Errorf("failed to clone repository: %w", err) - } - } else { - fmt.Fprintf(gi.logWriter, " Updating repository to latest changes...\n") - if output, err := exec.Command("git", "-C", srcDir, "fetch", "origin", branch).CombinedOutput(); err != nil { - return fmt.Errorf("failed to fetch repository updates: %v\n%s", err, string(output)) - } - if output, err := exec.Command("git", "-C", srcDir, "reset", "--hard", "origin/"+branch).CombinedOutput(); err != nil { - return fmt.Errorf("failed to reset repository: %v\n%s", err, string(output)) - } - if output, err := exec.Command("git", "-C", srcDir, "clean", "-fd").CombinedOutput(); err != nil { - return fmt.Errorf("failed to clean repository: %v\n%s", err, string(output)) - } + // Download source as ZIP from GitHub (simpler than git, no shallow clone issues) + if err := gi.downloadSourceZIP(branch, srcDir); err != nil { + return err } } @@ -210,8 +258,8 @@ func (gi *GatewayInstaller) InstallSystemDependencies() error { fmt.Fprintf(gi.logWriter, " Warning: apt update failed\n") } - // Install dependencies including Node.js for anyone-client - cmd = exec.Command("apt-get", "install", "-y", "curl", "git", "make", "build-essential", "wget", "nodejs", "npm") + // Install dependencies including Node.js for anyone-client and unzip for source downloads + cmd = exec.Command("apt-get", "install", "-y", "curl", "make", "build-essential", "wget", "unzip", "nodejs", "npm") if err := cmd.Run(); err != nil { return fmt.Errorf("failed to install dependencies: %w", err) } diff --git a/pkg/environments/production/orchestrator.go b/pkg/environments/production/orchestrator.go index eafb0e5..96a7c55 100644 --- a/pkg/environments/production/orchestrator.go +++ b/pkg/environments/production/orchestrator.go @@ -20,6 +20,7 @@ type ProductionSetup struct { forceReconfigure bool skipOptionalDeps bool skipResourceChecks bool + isNameserver bool // Whether this node is a nameserver (runs CoreDNS + Caddy) privChecker *PrivilegeChecker osDetector *OSDetector archDetector *ArchitectureDetector @@ -112,6 +113,16 @@ func (ps *ProductionSetup) IsUpdate() bool { return ps.stateDetector.IsConfigured() || ps.stateDetector.HasIPFSData() } +// SetNameserver sets whether this node is a nameserver (runs CoreDNS + Caddy) +func (ps *ProductionSetup) SetNameserver(isNameserver bool) { + ps.isNameserver = isNameserver +} + +// IsNameserver returns whether this node is configured as a nameserver +func (ps *ProductionSetup) IsNameserver() bool { + return ps.isNameserver +} + // Phase1CheckPrerequisites performs initial environment validation func (ps *ProductionSetup) Phase1CheckPrerequisites() error { ps.logf("Phase 1: Checking prerequisites...") @@ -274,14 +285,19 @@ func (ps *ProductionSetup) Phase2bInstallBinaries() error { return fmt.Errorf("failed to install DeBros binaries: %w", err) } - // Install CoreDNS with RQLite plugin (for dynamic DNS records and ACME challenges) - if err := ps.binaryInstaller.InstallCoreDNS(); err != nil { - ps.logf(" ⚠️ CoreDNS install warning: %v", err) - } + // Install CoreDNS and Caddy only if this is a nameserver node + if ps.isNameserver { + // Install CoreDNS with RQLite plugin (for dynamic DNS records and ACME challenges) + if err := ps.binaryInstaller.InstallCoreDNS(); err != nil { + ps.logf(" ⚠️ CoreDNS install warning: %v", err) + } - // Install Caddy with orama DNS module (for SSL certificate management) - if err := ps.binaryInstaller.InstallCaddy(); err != nil { - ps.logf(" ⚠️ Caddy install warning: %v", err) + // Install Caddy with orama DNS module (for SSL certificate management) + if err := ps.binaryInstaller.InstallCaddy(); err != nil { + ps.logf(" ⚠️ Caddy install warning: %v", err) + } + } else { + ps.logf(" ℹ️ Skipping CoreDNS/Caddy (not a nameserver node)") } ps.logf(" ✓ All binaries installed") @@ -533,28 +549,31 @@ func (ps *ProductionSetup) Phase5CreateSystemdServices(enableHTTPS bool) error { } ps.logf(" ✓ Anyone Client service created") - // CoreDNS service (for dynamic DNS with RQLite) - if _, err := os.Stat("/usr/local/bin/coredns"); err == nil { - corednsUnit := ps.serviceGenerator.GenerateCoreDNSService() - if err := ps.serviceController.WriteServiceUnit("coredns.service", corednsUnit); err != nil { - ps.logf(" ⚠️ Failed to write CoreDNS service: %v", err) - } else { - ps.logf(" ✓ CoreDNS service created") + // CoreDNS and Caddy services (only for nameserver nodes) + if ps.isNameserver { + // CoreDNS service (for dynamic DNS with RQLite) + if _, err := os.Stat("/usr/local/bin/coredns"); err == nil { + corednsUnit := ps.serviceGenerator.GenerateCoreDNSService() + if err := ps.serviceController.WriteServiceUnit("coredns.service", corednsUnit); err != nil { + ps.logf(" ⚠️ Failed to write CoreDNS service: %v", err) + } else { + ps.logf(" ✓ CoreDNS service created") + } } - } - // Caddy service (for SSL/TLS with DNS-01 ACME challenges) - if _, err := os.Stat("/usr/bin/caddy"); err == nil { - // Create caddy user if it doesn't exist - exec.Command("useradd", "-r", "-s", "/sbin/nologin", "caddy").Run() - exec.Command("mkdir", "-p", "/var/lib/caddy").Run() - exec.Command("chown", "caddy:caddy", "/var/lib/caddy").Run() + // Caddy service (for SSL/TLS with DNS-01 ACME challenges) + if _, err := os.Stat("/usr/bin/caddy"); err == nil { + // Create caddy user if it doesn't exist + exec.Command("useradd", "-r", "-s", "/sbin/nologin", "caddy").Run() + exec.Command("mkdir", "-p", "/var/lib/caddy").Run() + exec.Command("chown", "caddy:caddy", "/var/lib/caddy").Run() - caddyUnit := ps.serviceGenerator.GenerateCaddyService() - if err := ps.serviceController.WriteServiceUnit("caddy.service", caddyUnit); err != nil { - ps.logf(" ⚠️ Failed to write Caddy service: %v", err) - } else { - ps.logf(" ✓ Caddy service created") + caddyUnit := ps.serviceGenerator.GenerateCaddyService() + if err := ps.serviceController.WriteServiceUnit("caddy.service", caddyUnit); err != nil { + ps.logf(" ⚠️ Failed to write Caddy service: %v", err) + } else { + ps.logf(" ✓ Caddy service created") + } } } @@ -569,12 +588,14 @@ func (ps *ProductionSetup) Phase5CreateSystemdServices(enableHTTPS bool) error { // Note: debros-rqlite.service is NOT created - RQLite is managed by each node internally services := []string{"debros-ipfs.service", "debros-ipfs-cluster.service", "debros-olric.service", "debros-node.service", "debros-anyone-client.service"} - // Add CoreDNS and Caddy if installed - if _, err := os.Stat("/usr/local/bin/coredns"); err == nil { - services = append(services, "coredns.service") - } - if _, err := os.Stat("/usr/bin/caddy"); err == nil { - services = append(services, "caddy.service") + // Add CoreDNS and Caddy only for nameserver nodes + if ps.isNameserver { + if _, err := os.Stat("/usr/local/bin/coredns"); err == nil { + services = append(services, "coredns.service") + } + if _, err := os.Stat("/usr/bin/caddy"); err == nil { + services = append(services, "caddy.service") + } } for _, svc := range services { if err := ps.serviceController.EnableService(svc); err != nil { diff --git a/pkg/environments/production/preferences.go b/pkg/environments/production/preferences.go new file mode 100644 index 0000000..8e40c9d --- /dev/null +++ b/pkg/environments/production/preferences.go @@ -0,0 +1,85 @@ +package production + +import ( + "os" + "path/filepath" + "strings" + + "gopkg.in/yaml.v3" +) + +// NodePreferences contains persistent node configuration that survives upgrades +type NodePreferences struct { + Branch string `yaml:"branch"` + Nameserver bool `yaml:"nameserver"` +} + +const ( + preferencesFile = "preferences.yaml" + legacyBranchFile = ".branch" +) + +// SavePreferences saves node preferences to disk +func SavePreferences(oramaDir string, prefs *NodePreferences) error { + // Ensure directory exists + if err := os.MkdirAll(oramaDir, 0755); err != nil { + return err + } + + // Save to YAML file + path := filepath.Join(oramaDir, preferencesFile) + data, err := yaml.Marshal(prefs) + if err != nil { + return err + } + + if err := os.WriteFile(path, data, 0644); err != nil { + return err + } + + // Also save branch to legacy .branch file for backward compatibility + legacyPath := filepath.Join(oramaDir, legacyBranchFile) + os.WriteFile(legacyPath, []byte(prefs.Branch), 0644) + + return nil +} + +// LoadPreferences loads node preferences from disk +// Falls back to reading legacy .branch file if preferences.yaml doesn't exist +func LoadPreferences(oramaDir string) *NodePreferences { + prefs := &NodePreferences{ + Branch: "main", + Nameserver: false, + } + + // Try to load from preferences.yaml first + path := filepath.Join(oramaDir, preferencesFile) + if data, err := os.ReadFile(path); err == nil { + if err := yaml.Unmarshal(data, prefs); err == nil { + return prefs + } + } + + // Fall back to legacy .branch file + legacyPath := filepath.Join(oramaDir, legacyBranchFile) + if data, err := os.ReadFile(legacyPath); err == nil { + branch := strings.TrimSpace(string(data)) + if branch != "" { + prefs.Branch = branch + } + } + + return prefs +} + +// SaveNameserverPreference updates just the nameserver preference +func SaveNameserverPreference(oramaDir string, isNameserver bool) error { + prefs := LoadPreferences(oramaDir) + prefs.Nameserver = isNameserver + return SavePreferences(oramaDir, prefs) +} + +// ReadNameserverPreference reads just the nameserver preference +func ReadNameserverPreference(oramaDir string) bool { + return LoadPreferences(oramaDir).Nameserver +}