enchanced e2e tests, fixed rqlite issue

.gitignore

@@ -45,6 +45,9 @@ Thumbs.db
 .env.local
 .env.*.local
 
+# E2E test config (contains production credentials)
+e2e/config.yaml
+
 # Temporary files
 tmp/
 temp/

e2e/config.go

@@ -0,0 +1,171 @@
+//go:build e2e
+
+package e2e
+
+import (
+	"os"
+	"path/filepath"
+	"testing"
+
+	"gopkg.in/yaml.v2"
+)
+
+// E2EConfig holds the configuration for E2E tests
+type E2EConfig struct {
+	// Mode can be "local" or "production"
+	Mode string `yaml:"mode"`
+
+	// BaseDomain is the domain used for deployment routing (e.g., "dbrs.space" or "orama.network")
+	BaseDomain string `yaml:"base_domain"`
+
+	// Servers is a list of production servers (only used when mode=production)
+	Servers []ServerConfig `yaml:"servers"`
+
+	// Nameservers is a list of nameserver hostnames (e.g., ["ns1.dbrs.space", "ns2.dbrs.space"])
+	Nameservers []string `yaml:"nameservers"`
+
+	// APIKey is the API key for production testing (auto-discovered if empty)
+	APIKey string `yaml:"api_key"`
+}
+
+// ServerConfig holds configuration for a single production server
+type ServerConfig struct {
+	Name         string `yaml:"name"`
+	IP           string `yaml:"ip"`
+	User         string `yaml:"user"`
+	Password     string `yaml:"password"`
+	IsNameserver bool   `yaml:"is_nameserver"`
+}
+
+// DefaultConfig returns the default configuration for local development
+func DefaultConfig() *E2EConfig {
+	return &E2EConfig{
+		Mode:        "local",
+		BaseDomain:  "orama.network",
+		Servers:     []ServerConfig{},
+		Nameservers: []string{},
+		APIKey:      "",
+	}
+}
+
+// LoadE2EConfig loads the E2E test configuration from e2e/config.yaml
+// Falls back to defaults if the file doesn't exist
+func LoadE2EConfig() (*E2EConfig, error) {
+	// Try multiple locations for the config file
+	configPaths := []string{
+		"config.yaml",       // Relative to e2e directory (when running from e2e/)
+		"e2e/config.yaml",   // Relative to project root
+		"../e2e/config.yaml", // From subdirectory within e2e/
+	}
+
+	// Also try absolute path based on working directory
+	if cwd, err := os.Getwd(); err == nil {
+		configPaths = append(configPaths, filepath.Join(cwd, "config.yaml"))
+		configPaths = append(configPaths, filepath.Join(cwd, "e2e", "config.yaml"))
+		// Go up one level if we're in a subdirectory
+		configPaths = append(configPaths, filepath.Join(cwd, "..", "config.yaml"))
+	}
+
+	var configData []byte
+	var readErr error
+
+	for _, path := range configPaths {
+		data, err := os.ReadFile(path)
+		if err == nil {
+			configData = data
+			break
+		}
+		readErr = err
+	}
+
+	// If no config file found, return defaults
+	if configData == nil {
+		// Check if running in production mode via environment variable
+		if os.Getenv("E2E_MODE") == "production" {
+			return nil, readErr // Config file required for production mode
+		}
+		return DefaultConfig(), nil
+	}
+
+	var cfg E2EConfig
+	if err := yaml.Unmarshal(configData, &cfg); err != nil {
+		return nil, err
+	}
+
+	// Apply defaults for empty values
+	if cfg.Mode == "" {
+		cfg.Mode = "local"
+	}
+	if cfg.BaseDomain == "" {
+		cfg.BaseDomain = "orama.network"
+	}
+
+	return &cfg, nil
+}
+
+// IsProductionMode returns true if running in production mode
+func IsProductionMode() bool {
+	// Check environment variable first
+	if os.Getenv("E2E_MODE") == "production" {
+		return true
+	}
+
+	cfg, err := LoadE2EConfig()
+	if err != nil {
+		return false
+	}
+	return cfg.Mode == "production"
+}
+
+// IsLocalMode returns true if running in local mode
+func IsLocalMode() bool {
+	return !IsProductionMode()
+}
+
+// SkipIfLocal skips the test if running in local mode
+// Use this for tests that require real production infrastructure
+func SkipIfLocal(t *testing.T) {
+	t.Helper()
+	if IsLocalMode() {
+		t.Skip("Skipping: requires production environment (set mode: production in e2e/config.yaml)")
+	}
+}
+
+// SkipIfProduction skips the test if running in production mode
+// Use this for tests that should only run locally
+func SkipIfProduction(t *testing.T) {
+	t.Helper()
+	if IsProductionMode() {
+		t.Skip("Skipping: local-only test")
+	}
+}
+
+// GetServerIPs returns a list of all server IP addresses from config
+func GetServerIPs(cfg *E2EConfig) []string {
+	if cfg == nil {
+		return nil
+	}
+
+	ips := make([]string, 0, len(cfg.Servers))
+	for _, server := range cfg.Servers {
+		if server.IP != "" {
+			ips = append(ips, server.IP)
+		}
+	}
+	return ips
+}
+
+// GetNameserverServers returns servers configured as nameservers
+func GetNameserverServers(cfg *E2EConfig) []ServerConfig {
+	if cfg == nil {
+		return nil
+	}
+
+	var nameservers []ServerConfig
+	for _, server := range cfg.Servers {
+		if server.IsNameserver {
+			nameservers = append(nameservers, server)
+		}
+	}
+	return nameservers
+}

e2e/config.yaml.example

@@ -0,0 +1,45 @@
+# E2E Test Configuration
+#
+# Copy this file to config.yaml and fill in your values.
+# config.yaml is git-ignored and should contain your actual credentials.
+#
+# Usage:
+#   cp config.yaml.example config.yaml
+#   # Edit config.yaml with your server credentials
+#   go test -v -tags e2e ./e2e/...
+
+# Test mode: "local" or "production"
+# - local: Tests run against `make dev` cluster on localhost
+# - production: Tests run against real VPS servers
+mode: local
+
+# Base domain for deployment routing
+# - Local: orama.network (default)
+# - Production: dbrs.space (or your custom domain)
+base_domain: orama.network
+
+# Production servers (only used when mode=production)
+# Add your VPS servers here with their credentials
+servers:
+  # Example:
+  # - name: vps-1
+  #   ip: 1.2.3.4
+  #   user: ubuntu
+  #   password: "your-password-here"
+  #   is_nameserver: true
+  # - name: vps-2
+  #   ip: 5.6.7.8
+  #   user: ubuntu
+  #   password: "another-password"
+  #   is_nameserver: false
+
+# Nameserver hostnames (for DNS tests in production)
+# These should match your NS records
+nameservers:
+  # Example:
+  # - ns1.yourdomain.com
+  # - ns2.yourdomain.com
+
+# API key for production testing
+# Leave empty to auto-discover from RQLite or create fresh key
+api_key: ""

e2e/domain_routing_test.go

@@ -38,8 +38,8 @@ func TestDomainRouting_BasicRouting(t *testing.T) {
 		deploymentID, deployment["content_cid"], deployment["name"], deployment["status"])
 
 	t.Run("Standard domain resolves", func(t *testing.T) {
-		// Domain format: {deploymentName}.orama.network
-		domain := fmt.Sprintf("%s.orama.network", deploymentName)
+		// Domain format: {deploymentName}.{baseDomain}
+		domain := env.BuildDeploymentDomain(deploymentName)
 
 		resp := TestDeploymentWithHostHeader(t, env, domain, "/")
 		defer resp.Body.Close()
@@ -69,7 +69,7 @@ func TestDomainRouting_BasicRouting(t *testing.T) {
 
 	t.Run("API paths bypass domain routing", func(t *testing.T) {
 		// /v1/* paths should bypass domain routing and use API key auth
-		domain := fmt.Sprintf("%s.orama.network", deploymentName)
+		domain := env.BuildDeploymentDomain(deploymentName)
 
 		req, _ := http.NewRequest("GET", env.GatewayURL+"/v1/deployments/list", nil)
 		req.Host = domain
@@ -94,7 +94,7 @@ func TestDomainRouting_BasicRouting(t *testing.T) {
 	})
 
 	t.Run("Well-known paths bypass domain routing", func(t *testing.T) {
-		domain := fmt.Sprintf("%s.orama.network", deploymentName)
+		domain := env.BuildDeploymentDomain(deploymentName)
 
 		// /.well-known/ paths should bypass (used for ACME challenges, etc.)
 		resp := TestDeploymentWithHostHeader(t, env, domain, "/.well-known/acme-challenge/test")
@@ -139,8 +139,8 @@ func TestDomainRouting_MultipleDeployments(t *testing.T) {
 	time.Sleep(2 * time.Second)
 
 	t.Run("Each deployment routes independently", func(t *testing.T) {
-		domain1 := fmt.Sprintf("%s.orama.network", deployment1Name)
-		domain2 := fmt.Sprintf("%s.orama.network", deployment2Name)
+		domain1 := env.BuildDeploymentDomain(deployment1Name)
+		domain2 := env.BuildDeploymentDomain(deployment2Name)
 
 		// Test deployment 1
 		resp1 := TestDeploymentWithHostHeader(t, env, domain1, "/")
@@ -161,7 +161,7 @@ func TestDomainRouting_MultipleDeployments(t *testing.T) {
 
 	t.Run("Wrong domain returns 404", func(t *testing.T) {
 		// Request with non-existent deployment subdomain
-		fakeDeploymentDomain := fmt.Sprintf("nonexistent-deployment-%d.orama.network", time.Now().Unix())
+		fakeDeploymentDomain := env.BuildDeploymentDomain(fmt.Sprintf("nonexistent-deployment-%d", time.Now().Unix()))
 
 		resp := TestDeploymentWithHostHeader(t, env, fakeDeploymentDomain, "/")
 		defer resp.Body.Close()
@@ -189,7 +189,7 @@ func TestDomainRouting_ContentTypes(t *testing.T) {
 
 	time.Sleep(2 * time.Second)
 
-	domain := fmt.Sprintf("%s.orama.network", deploymentName)
+	domain := env.BuildDeploymentDomain(deploymentName)
 
 	contentTypeTests := []struct {
 		path        string
@@ -234,7 +234,7 @@ func TestDomainRouting_SPAFallback(t *testing.T) {
 
 	time.Sleep(2 * time.Second)
 
-	domain := fmt.Sprintf("%s.orama.network", deploymentName)
+	domain := env.BuildDeploymentDomain(deploymentName)
 
 	t.Run("Unknown paths fall back to index.html", func(t *testing.T) {
 		unknownPaths := []string{
@@ -260,3 +260,85 @@ func TestDomainRouting_SPAFallback(t *testing.T) {
 		t.Logf("✓ SPA fallback routing verified for %d paths", len(unknownPaths))
 	})
 }
+
+// TestDeployment_DomainFormat verifies that deployment URLs use the correct format:
+// - CORRECT: {name}.{baseDomain} (e.g., "myapp.dbrs.space")
+// - WRONG: {name}.node-{shortID}.{baseDomain} (should NOT exist)
+func TestDeployment_DomainFormat(t *testing.T) {
+	env, err := LoadTestEnv()
+	require.NoError(t, err, "Failed to load test environment")
+
+	deploymentName := fmt.Sprintf("format-test-%d", time.Now().Unix())
+	tarballPath := filepath.Join("../testdata/tarballs/react-vite.tar.gz")
+
+	deploymentID := CreateTestDeployment(t, env, deploymentName, tarballPath)
+	defer func() {
+		if !env.SkipCleanup {
+			DeleteDeployment(t, env, deploymentID)
+		}
+	}()
+
+	// Wait for deployment
+	time.Sleep(2 * time.Second)
+
+	t.Run("Deployment URL has correct format", func(t *testing.T) {
+		deployment := GetDeployment(t, env, deploymentID)
+
+		// Get the deployment URLs
+		urls, ok := deployment["urls"].([]interface{})
+		if !ok || len(urls) == 0 {
+			// Fall back to single url field
+			if url, ok := deployment["url"].(string); ok && url != "" {
+				urls = []interface{}{url}
+			}
+		}
+
+		expectedDomain := env.BuildDeploymentDomain(deploymentName)
+		t.Logf("Expected domain format: %s", expectedDomain)
+		t.Logf("Deployment URLs: %v", urls)
+
+		foundCorrectFormat := false
+		for _, u := range urls {
+			urlStr, ok := u.(string)
+			if !ok {
+				continue
+			}
+
+			// URL should contain the simple format: {name}.{baseDomain}
+			if assert.Contains(t, urlStr, expectedDomain,
+				"URL should contain %s", expectedDomain) {
+				foundCorrectFormat = true
+			}
+
+			// URL should NOT contain node identifier pattern
+			assert.NotContains(t, urlStr, ".node-",
+				"URL should NOT have node identifier (got: %s)", urlStr)
+		}
+
+		if len(urls) > 0 {
+			assert.True(t, foundCorrectFormat, "Should find URL with correct domain format")
+		}
+
+		t.Logf("✓ Domain format verification passed")
+		t.Logf("   - Expected: %s", expectedDomain)
+	})
+
+	t.Run("Domain resolves via Host header", func(t *testing.T) {
+		// Test that the simple domain format works
+		domain := env.BuildDeploymentDomain(deploymentName)
+
+		resp := TestDeploymentWithHostHeader(t, env, domain, "/")
+		defer resp.Body.Close()
+
+		assert.Equal(t, http.StatusOK, resp.StatusCode,
+			"Domain %s should resolve successfully", domain)
+
+		body, err := io.ReadAll(resp.Body)
+		require.NoError(t, err)
+
+		assert.Contains(t, string(body), "<div id=\"root\">",
+			"Should serve deployment content")
+
+		t.Logf("✓ Domain %s resolves correctly", domain)
+	})
+}

e2e/env.go

@@ -973,23 +973,45 @@ func (p *WSPubSubClientPair) Close() {
 
 // E2ETestEnv holds the environment configuration for deployment E2E tests
 type E2ETestEnv struct {
-	GatewayURL   string
-	APIKey       string
-	Namespace    string
-	HTTPClient   *http.Client
-	SkipCleanup  bool
+	GatewayURL  string
+	APIKey      string
+	Namespace   string
+	BaseDomain  string       // Domain for deployment routing (e.g., "dbrs.space")
+	Config      *E2EConfig   // Full E2E configuration (for production tests)
+	HTTPClient  *http.Client
+	SkipCleanup bool
 }
 
-// LoadTestEnv loads the test environment from environment variables
+// BuildDeploymentDomain returns the full domain for a deployment name
+// Format: {name}.{baseDomain} (e.g., "myapp.dbrs.space")
+func (env *E2ETestEnv) BuildDeploymentDomain(deploymentName string) string {
+	return fmt.Sprintf("%s.%s", deploymentName, env.BaseDomain)
+}
+
+// LoadTestEnv loads the test environment from environment variables and config file
 // If ORAMA_API_KEY is not set, it creates a fresh API key for the default test namespace
 func LoadTestEnv() (*E2ETestEnv, error) {
+	// Load E2E config (for base_domain and production settings)
+	cfg, err := LoadE2EConfig()
+	if err != nil {
+		// If config loading fails in production mode, that's an error
+		if IsProductionMode() {
+			return nil, fmt.Errorf("failed to load e2e config: %w", err)
+		}
+		// For local mode, use defaults
+		cfg = DefaultConfig()
+	}
+
 	gatewayURL := os.Getenv("ORAMA_GATEWAY_URL")
 	if gatewayURL == "" {
 		gatewayURL = GetGatewayURL()
 	}
 
-	// Check if API key is provided via environment variable
+	// Check if API key is provided via environment variable or config
 	apiKey := os.Getenv("ORAMA_API_KEY")
+	if apiKey == "" && cfg.APIKey != "" {
+		apiKey = cfg.APIKey
+	}
 	namespace := os.Getenv("ORAMA_NAMESPACE")
 
 	// If no API key provided, create a fresh one for a default test namespace
@@ -1055,6 +1077,8 @@ func LoadTestEnv() (*E2ETestEnv, error) {
 		GatewayURL:  gatewayURL,
 		APIKey:      apiKey,
 		Namespace:   namespace,
+		BaseDomain:  cfg.BaseDomain,
+		Config:      cfg,
 		HTTPClient:  NewHTTPClient(30 * time.Second),
 		SkipCleanup: skipCleanup,
 	}, nil
@@ -1063,6 +1087,12 @@ func LoadTestEnv() (*E2ETestEnv, error) {
 // LoadTestEnvWithNamespace loads test environment with a specific namespace
 // It creates a new API key for the specified namespace to ensure proper isolation
 func LoadTestEnvWithNamespace(namespace string) (*E2ETestEnv, error) {
+	// Load E2E config (for base_domain and production settings)
+	cfg, err := LoadE2EConfig()
+	if err != nil {
+		cfg = DefaultConfig()
+	}
+
 	gatewayURL := os.Getenv("ORAMA_GATEWAY_URL")
 	if gatewayURL == "" {
 		gatewayURL = GetGatewayURL()
@@ -1122,6 +1152,8 @@ func LoadTestEnvWithNamespace(namespace string) (*E2ETestEnv, error) {
 		GatewayURL:  gatewayURL,
 		APIKey:      apiKey,
 		Namespace:   namespace,
+		BaseDomain:  cfg.BaseDomain,
+		Config:      cfg,
 		HTTPClient:  NewHTTPClient(30 * time.Second),
 		SkipCleanup: skipCleanup,
 	}, nil

e2e/fullstack_integration_test.go

@@ -129,7 +129,7 @@ func TestFullStack_GoAPI_SQLite(t *testing.T) {
 			return
 		}
 
-		backendDomain := fmt.Sprintf("%s.orama.network", backendName)
+		backendDomain := env.BuildDeploymentDomain(backendName)
 
 		// Test health endpoint
 		resp := TestDeploymentWithHostHeader(t, env, backendDomain, "/health")
@@ -262,7 +262,7 @@ func TestFullStack_StaticSite_SQLite(t *testing.T) {
 	})
 
 	t.Run("Test frontend serving and database interaction", func(t *testing.T) {
-		frontendDomain := fmt.Sprintf("%s.orama.network", frontendName)
+		frontendDomain := env.BuildDeploymentDomain(frontendName)
 
 		// Test frontend
 		resp := TestDeploymentWithHostHeader(t, env, frontendDomain, "/")

e2e/production/cross_node_proxy_test.go

@@ -0,0 +1,227 @@
+//go:build e2e
+
+package production
+
+import (
+	"fmt"
+	"io"
+	"net/http"
+	"path/filepath"
+	"testing"
+	"time"
+
+	"github.com/DeBrosOfficial/network/e2e"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+// TestCrossNode_ProxyRouting tests that requests can be made to any node
+// and get proxied to the correct home node for a deployment
+func TestCrossNode_ProxyRouting(t *testing.T) {
+	e2e.SkipIfLocal(t)
+
+	env, err := e2e.LoadTestEnv()
+	require.NoError(t, err, "Failed to load test environment")
+
+	if len(env.Config.Servers) < 2 {
+		t.Skip("Cross-node testing requires at least 2 servers in config")
+	}
+
+	deploymentName := fmt.Sprintf("proxy-test-%d", time.Now().Unix())
+	tarballPath := filepath.Join("../../testdata/tarballs/react-vite.tar.gz")
+
+	deploymentID := e2e.CreateTestDeployment(t, env, deploymentName, tarballPath)
+	defer func() {
+		if !env.SkipCleanup {
+			e2e.DeleteDeployment(t, env, deploymentID)
+		}
+	}()
+
+	// Wait for deployment to be active
+	time.Sleep(3 * time.Second)
+
+	domain := env.BuildDeploymentDomain(deploymentName)
+	t.Logf("Testing cross-node routing for: %s", domain)
+
+	t.Run("Request via each server succeeds", func(t *testing.T) {
+		for _, server := range env.Config.Servers {
+			t.Run("via_"+server.Name, func(t *testing.T) {
+				// Make request directly to this server's IP
+				gatewayURL := fmt.Sprintf("http://%s:6001", server.IP)
+
+				req, err := http.NewRequest("GET", gatewayURL+"/", nil)
+				require.NoError(t, err)
+
+				// Set Host header to the deployment domain
+				req.Host = domain
+
+				resp, err := env.HTTPClient.Do(req)
+				require.NoError(t, err, "Request to %s should succeed", server.Name)
+				defer resp.Body.Close()
+
+				body, _ := io.ReadAll(resp.Body)
+
+				assert.Equal(t, http.StatusOK, resp.StatusCode,
+					"Request via %s should return 200 (got %d: %s)",
+					server.Name, resp.StatusCode, string(body))
+
+				assert.Contains(t, string(body), "<div id=\"root\">",
+					"Should serve deployment content via %s", server.Name)
+
+				t.Logf("✓ Request via %s (%s) succeeded", server.Name, server.IP)
+			})
+		}
+	})
+}
+
+// TestCrossNode_APIConsistency tests that API responses are consistent across nodes
+func TestCrossNode_APIConsistency(t *testing.T) {
+	e2e.SkipIfLocal(t)
+
+	env, err := e2e.LoadTestEnv()
+	require.NoError(t, err, "Failed to load test environment")
+
+	if len(env.Config.Servers) < 2 {
+		t.Skip("Cross-node testing requires at least 2 servers in config")
+	}
+
+	deploymentName := fmt.Sprintf("consistency-test-%d", time.Now().Unix())
+	tarballPath := filepath.Join("../../testdata/tarballs/react-vite.tar.gz")
+
+	deploymentID := e2e.CreateTestDeployment(t, env, deploymentName, tarballPath)
+	defer func() {
+		if !env.SkipCleanup {
+			e2e.DeleteDeployment(t, env, deploymentID)
+		}
+	}()
+
+	// Wait for replication
+	time.Sleep(5 * time.Second)
+
+	t.Run("Deployment list is consistent across nodes", func(t *testing.T) {
+		var deploymentCounts []int
+
+		for _, server := range env.Config.Servers {
+			gatewayURL := fmt.Sprintf("http://%s:6001", server.IP)
+
+			req, err := http.NewRequest("GET", gatewayURL+"/v1/deployments/list", nil)
+			require.NoError(t, err)
+			req.Header.Set("Authorization", "Bearer "+env.APIKey)
+
+			resp, err := env.HTTPClient.Do(req)
+			if err != nil {
+				t.Logf("⚠ Could not reach %s: %v", server.Name, err)
+				continue
+			}
+			defer resp.Body.Close()
+
+			if resp.StatusCode != http.StatusOK {
+				t.Logf("⚠ %s returned status %d", server.Name, resp.StatusCode)
+				continue
+			}
+
+			var result map[string]interface{}
+			if err := e2e.DecodeJSON(mustReadAll(t, resp.Body), &result); err != nil {
+				t.Logf("⚠ Could not decode response from %s", server.Name)
+				continue
+			}
+
+			deployments, ok := result["deployments"].([]interface{})
+			if !ok {
+				t.Logf("⚠ Invalid response format from %s", server.Name)
+				continue
+			}
+
+			deploymentCounts = append(deploymentCounts, len(deployments))
+			t.Logf("%s reports %d deployments", server.Name, len(deployments))
+		}
+
+		// All nodes should report the same count (or close to it, allowing for replication delay)
+		if len(deploymentCounts) >= 2 {
+			for i := 1; i < len(deploymentCounts); i++ {
+				diff := deploymentCounts[i] - deploymentCounts[0]
+				if diff < 0 {
+					diff = -diff
+				}
+				assert.LessOrEqual(t, diff, 1,
+					"Deployment counts should be consistent across nodes (allowing for replication)")
+			}
+		}
+	})
+}
+
+// TestCrossNode_DeploymentGetConsistency tests that deployment details are consistent
+func TestCrossNode_DeploymentGetConsistency(t *testing.T) {
+	e2e.SkipIfLocal(t)
+
+	env, err := e2e.LoadTestEnv()
+	require.NoError(t, err, "Failed to load test environment")
+
+	if len(env.Config.Servers) < 2 {
+		t.Skip("Cross-node testing requires at least 2 servers in config")
+	}
+
+	deploymentName := fmt.Sprintf("get-consistency-%d", time.Now().Unix())
+	tarballPath := filepath.Join("../../testdata/tarballs/react-vite.tar.gz")
+
+	deploymentID := e2e.CreateTestDeployment(t, env, deploymentName, tarballPath)
+	defer func() {
+		if !env.SkipCleanup {
+			e2e.DeleteDeployment(t, env, deploymentID)
+		}
+	}()
+
+	// Wait for replication
+	time.Sleep(5 * time.Second)
+
+	t.Run("Deployment details match across nodes", func(t *testing.T) {
+		var cids []string
+
+		for _, server := range env.Config.Servers {
+			gatewayURL := fmt.Sprintf("http://%s:6001", server.IP)
+
+			req, err := http.NewRequest("GET", gatewayURL+"/v1/deployments/get?id="+deploymentID, nil)
+			require.NoError(t, err)
+			req.Header.Set("Authorization", "Bearer "+env.APIKey)
+
+			resp, err := env.HTTPClient.Do(req)
+			if err != nil {
+				t.Logf("⚠ Could not reach %s: %v", server.Name, err)
+				continue
+			}
+			defer resp.Body.Close()
+
+			if resp.StatusCode != http.StatusOK {
+				t.Logf("⚠ %s returned status %d", server.Name, resp.StatusCode)
+				continue
+			}
+
+			var deployment map[string]interface{}
+			if err := e2e.DecodeJSON(mustReadAll(t, resp.Body), &deployment); err != nil {
+				t.Logf("⚠ Could not decode response from %s", server.Name)
+				continue
+			}
+
+			cid, _ := deployment["content_cid"].(string)
+			cids = append(cids, cid)
+
+			t.Logf("%s: name=%s, cid=%s, status=%s",
+				server.Name, deployment["name"], cid, deployment["status"])
+		}
+
+		// All nodes should have the same CID
+		if len(cids) >= 2 {
+			for i := 1; i < len(cids); i++ {
+				assert.Equal(t, cids[0], cids[i],
+					"Content CID should be consistent across nodes")
+			}
+		}
+	})
+}
+
+func mustReadAll(t *testing.T, r io.Reader) []byte {
+	t.Helper()
+	data, err := io.ReadAll(r)
+	require.NoError(t, err)
+	return data
+}

e2e/production/dns_resolution_test.go

@@ -0,0 +1,121 @@
+//go:build e2e
+
+package production
+
+import (
+	"context"
+	"fmt"
+	"net"
+	"path/filepath"
+	"testing"
+	"time"
+
+	"github.com/DeBrosOfficial/network/e2e"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+// TestDNS_DeploymentResolution tests that deployed applications are resolvable via DNS
+// This test requires production mode as it performs real DNS lookups
+func TestDNS_DeploymentResolution(t *testing.T) {
+	e2e.SkipIfLocal(t)
+
+	env, err := e2e.LoadTestEnv()
+	require.NoError(t, err, "Failed to load test environment")
+
+	deploymentName := fmt.Sprintf("dns-test-%d", time.Now().Unix())
+	tarballPath := filepath.Join("../../testdata/tarballs/react-vite.tar.gz")
+
+	deploymentID := e2e.CreateTestDeployment(t, env, deploymentName, tarballPath)
+	defer func() {
+		if !env.SkipCleanup {
+			e2e.DeleteDeployment(t, env, deploymentID)
+		}
+	}()
+
+	// Wait for DNS propagation
+	domain := env.BuildDeploymentDomain(deploymentName)
+	t.Logf("Testing DNS resolution for: %s", domain)
+
+	t.Run("DNS resolves to valid server IP", func(t *testing.T) {
+		// Allow some time for DNS propagation
+		ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
+		defer cancel()
+
+		var ips []string
+		var err error
+
+		// Poll for DNS resolution
+		for {
+			select {
+			case <-ctx.Done():
+				t.Fatalf("DNS resolution timeout for %s", domain)
+			default:
+				ips, err = net.LookupHost(domain)
+				if err == nil && len(ips) > 0 {
+					goto resolved
+				}
+				time.Sleep(2 * time.Second)
+			}
+		}
+
+	resolved:
+		t.Logf("DNS resolved: %s -> %v", domain, ips)
+		assert.NotEmpty(t, ips, "Should have IP addresses")
+
+		// Verify resolved IP is one of our servers
+		validIPs := e2e.GetServerIPs(env.Config)
+		if len(validIPs) > 0 {
+			found := false
+			for _, ip := range ips {
+				for _, validIP := range validIPs {
+					if ip == validIP {
+						found = true
+						break
+					}
+				}
+			}
+			assert.True(t, found, "Resolved IP should be one of our servers: %v (valid: %v)", ips, validIPs)
+		}
+	})
+}
+
+// TestDNS_BaseDomainResolution tests that the base domain resolves correctly
+func TestDNS_BaseDomainResolution(t *testing.T) {
+	e2e.SkipIfLocal(t)
+
+	env, err := e2e.LoadTestEnv()
+	require.NoError(t, err, "Failed to load test environment")
+
+	t.Run("Base domain resolves", func(t *testing.T) {
+		ips, err := net.LookupHost(env.BaseDomain)
+		require.NoError(t, err, "Base domain %s should resolve", env.BaseDomain)
+		assert.NotEmpty(t, ips, "Should have IP addresses")
+
+		t.Logf("✓ Base domain %s resolves to: %v", env.BaseDomain, ips)
+	})
+}
+
+// TestDNS_WildcardResolution tests wildcard DNS for arbitrary subdomains
+func TestDNS_WildcardResolution(t *testing.T) {
+	e2e.SkipIfLocal(t)
+
+	env, err := e2e.LoadTestEnv()
+	require.NoError(t, err, "Failed to load test environment")
+
+	t.Run("Wildcard subdomain resolves", func(t *testing.T) {
+		// Test with a random subdomain that doesn't exist as a deployment
+		randomSubdomain := fmt.Sprintf("random-test-%d.%s", time.Now().UnixNano(), env.BaseDomain)
+
+		ips, err := net.LookupHost(randomSubdomain)
+		if err != nil {
+			// DNS may not support wildcard - that's OK for some setups
+			t.Logf("⚠ Wildcard DNS not configured (this may be expected): %v", err)
+			t.Skip("Wildcard DNS not configured")
+			return
+		}
+
+		assert.NotEmpty(t, ips, "Wildcard subdomain should resolve")
+		t.Logf("✓ Wildcard subdomain resolves: %s -> %v", randomSubdomain, ips)
+	})
+}

e2e/production/https_certificate_test.go

@@ -0,0 +1,191 @@
+//go:build e2e
+
+package production
+
+import (
+	"crypto/tls"
+	"fmt"
+	"io"
+	"net/http"
+	"path/filepath"
+	"testing"
+	"time"
+
+	"github.com/DeBrosOfficial/network/e2e"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+// TestHTTPS_CertificateValid tests that HTTPS works with a valid certificate
+func TestHTTPS_CertificateValid(t *testing.T) {
+	e2e.SkipIfLocal(t)
+
+	env, err := e2e.LoadTestEnv()
+	require.NoError(t, err, "Failed to load test environment")
+
+	deploymentName := fmt.Sprintf("https-test-%d", time.Now().Unix())
+	tarballPath := filepath.Join("../../testdata/tarballs/react-vite.tar.gz")
+
+	deploymentID := e2e.CreateTestDeployment(t, env, deploymentName, tarballPath)
+	defer func() {
+		if !env.SkipCleanup {
+			e2e.DeleteDeployment(t, env, deploymentID)
+		}
+	}()
+
+	// Wait for deployment and certificate provisioning
+	time.Sleep(5 * time.Second)
+
+	domain := env.BuildDeploymentDomain(deploymentName)
+	httpsURL := fmt.Sprintf("https://%s", domain)
+
+	t.Run("HTTPS connection with certificate verification", func(t *testing.T) {
+		// Create client that DOES verify certificates
+		client := &http.Client{
+			Timeout: 30 * time.Second,
+			Transport: &http.Transport{
+				TLSClientConfig: &tls.Config{
+					// Do NOT skip verification - we want to test real certs
+					InsecureSkipVerify: false,
+				},
+			},
+		}
+
+		req, err := http.NewRequest("GET", httpsURL+"/", nil)
+		require.NoError(t, err)
+
+		resp, err := client.Do(req)
+		if err != nil {
+			// Certificate might not be ready yet, or domain might not resolve
+			t.Logf("⚠ HTTPS request failed (this may be expected if certs are still provisioning): %v", err)
+			t.Skip("HTTPS not available or certificate not ready")
+			return
+		}
+		defer resp.Body.Close()
+
+		assert.Equal(t, http.StatusOK, resp.StatusCode, "HTTPS should return 200")
+
+		body, _ := io.ReadAll(resp.Body)
+		assert.Contains(t, string(body), "<div id=\"root\">", "Should serve deployment content over HTTPS")
+
+		// Check TLS connection state
+		if resp.TLS != nil {
+			t.Logf("✓ HTTPS works with valid certificate")
+			t.Logf("   - Domain: %s", domain)
+			t.Logf("   - TLS Version: %x", resp.TLS.Version)
+			t.Logf("   - Cipher Suite: %x", resp.TLS.CipherSuite)
+			if len(resp.TLS.PeerCertificates) > 0 {
+				cert := resp.TLS.PeerCertificates[0]
+				t.Logf("   - Certificate Subject: %s", cert.Subject)
+				t.Logf("   - Certificate Issuer: %s", cert.Issuer)
+				t.Logf("   - Valid Until: %s", cert.NotAfter)
+			}
+		}
+	})
+}
+
+// TestHTTPS_CertificateDetails tests certificate properties
+func TestHTTPS_CertificateDetails(t *testing.T) {
+	e2e.SkipIfLocal(t)
+
+	env, err := e2e.LoadTestEnv()
+	require.NoError(t, err, "Failed to load test environment")
+
+	t.Run("Base domain certificate", func(t *testing.T) {
+		httpsURL := fmt.Sprintf("https://%s", env.BaseDomain)
+
+		// Connect and get certificate info
+		conn, err := tls.Dial("tcp", env.BaseDomain+":443", &tls.Config{
+			InsecureSkipVerify: true, // We just want to inspect the cert
+		})
+		if err != nil {
+			t.Logf("⚠ Could not connect to %s:443: %v", env.BaseDomain, err)
+			t.Skip("HTTPS not available on base domain")
+			return
+		}
+		defer conn.Close()
+
+		certs := conn.ConnectionState().PeerCertificates
+		require.NotEmpty(t, certs, "Should have certificates")
+
+		cert := certs[0]
+		t.Logf("Certificate for %s:", env.BaseDomain)
+		t.Logf("  - Subject: %s", cert.Subject)
+		t.Logf("  - DNS Names: %v", cert.DNSNames)
+		t.Logf("  - Valid From: %s", cert.NotBefore)
+		t.Logf("  - Valid Until: %s", cert.NotAfter)
+		t.Logf("  - Issuer: %s", cert.Issuer)
+
+		// Check that certificate covers our domain
+		coversDomain := false
+		for _, name := range cert.DNSNames {
+			if name == env.BaseDomain || name == "*."+env.BaseDomain {
+				coversDomain = true
+				break
+			}
+		}
+		assert.True(t, coversDomain, "Certificate should cover %s", env.BaseDomain)
+
+		// Check certificate is not expired
+		assert.True(t, time.Now().Before(cert.NotAfter), "Certificate should not be expired")
+		assert.True(t, time.Now().After(cert.NotBefore), "Certificate should be valid now")
+
+		// Make actual HTTPS request to verify it works
+		client := &http.Client{
+			Timeout: 30 * time.Second,
+			Transport: &http.Transport{
+				TLSClientConfig: &tls.Config{
+					InsecureSkipVerify: false,
+				},
+			},
+		}
+
+		resp, err := client.Get(httpsURL)
+		if err != nil {
+			t.Logf("⚠ HTTPS request failed: %v", err)
+		} else {
+			resp.Body.Close()
+			t.Logf("✓ HTTPS request succeeded with status %d", resp.StatusCode)
+		}
+	})
+}
+
+// TestHTTPS_HTTPRedirect tests that HTTP requests are redirected to HTTPS
+func TestHTTPS_HTTPRedirect(t *testing.T) {
+	e2e.SkipIfLocal(t)
+
+	env, err := e2e.LoadTestEnv()
+	require.NoError(t, err, "Failed to load test environment")
+
+	t.Run("HTTP redirects to HTTPS", func(t *testing.T) {
+		// Create client that doesn't follow redirects
+		client := &http.Client{
+			Timeout: 30 * time.Second,
+			CheckRedirect: func(req *http.Request, via []*http.Request) error {
+				return http.ErrUseLastResponse
+			},
+		}
+
+		httpURL := fmt.Sprintf("http://%s", env.BaseDomain)
+
+		resp, err := client.Get(httpURL)
+		if err != nil {
+			t.Logf("⚠ HTTP request failed: %v", err)
+			t.Skip("HTTP not available or redirects not configured")
+			return
+		}
+		defer resp.Body.Close()
+
+		// Check for redirect
+		if resp.StatusCode >= 300 && resp.StatusCode < 400 {
+			location := resp.Header.Get("Location")
+			t.Logf("✓ HTTP redirects to: %s (status %d)", location, resp.StatusCode)
+			assert.Contains(t, location, "https://", "Should redirect to HTTPS")
+		} else if resp.StatusCode == http.StatusOK {
+			// HTTP might just serve content directly in some configurations
+			t.Logf("⚠ HTTP returned 200 instead of redirect (HTTPS redirect may not be configured)")
+		} else {
+			t.Logf("HTTP returned status %d", resp.StatusCode)
+		}
+	})
+}

e2e/production/nameserver_test.go

@@ -0,0 +1,181 @@
+//go:build e2e
+
+package production
+
+import (
+	"context"
+	"net"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/DeBrosOfficial/network/e2e"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+// TestNameserver_NSRecords tests that NS records are properly configured for the domain
+func TestNameserver_NSRecords(t *testing.T) {
+	e2e.SkipIfLocal(t)
+
+	env, err := e2e.LoadTestEnv()
+	require.NoError(t, err, "Failed to load test environment")
+
+	if len(env.Config.Nameservers) == 0 {
+		t.Skip("No nameservers configured in e2e/config.yaml")
+	}
+
+	t.Run("NS records exist for base domain", func(t *testing.T) {
+		nsRecords, err := net.LookupNS(env.BaseDomain)
+		require.NoError(t, err, "Should be able to look up NS records for %s", env.BaseDomain)
+		require.NotEmpty(t, nsRecords, "Should have NS records")
+
+		t.Logf("Found %d NS records for %s:", len(nsRecords), env.BaseDomain)
+		for _, ns := range nsRecords {
+			t.Logf("  - %s", ns.Host)
+		}
+
+		// Verify our nameservers are listed
+		for _, expected := range env.Config.Nameservers {
+			found := false
+			for _, ns := range nsRecords {
+				// Trim trailing dot for comparison
+				nsHost := strings.TrimSuffix(ns.Host, ".")
+				if nsHost == expected || nsHost == expected+"." {
+					found = true
+					break
+				}
+			}
+			assert.True(t, found, "NS records should include %s", expected)
+		}
+	})
+}
+
+// TestNameserver_GlueRecords tests that glue records point to correct IPs
+func TestNameserver_GlueRecords(t *testing.T) {
+	e2e.SkipIfLocal(t)
+
+	env, err := e2e.LoadTestEnv()
+	require.NoError(t, err, "Failed to load test environment")
+
+	if len(env.Config.Nameservers) == 0 {
+		t.Skip("No nameservers configured in e2e/config.yaml")
+	}
+
+	nameserverServers := e2e.GetNameserverServers(env.Config)
+	if len(nameserverServers) == 0 {
+		t.Skip("No servers marked as nameservers in config")
+	}
+
+	t.Run("Glue records resolve to correct IPs", func(t *testing.T) {
+		for i, ns := range env.Config.Nameservers {
+			ips, err := net.LookupHost(ns)
+			require.NoError(t, err, "Nameserver %s should resolve", ns)
+			require.NotEmpty(t, ips, "Nameserver %s should have IP addresses", ns)
+
+			t.Logf("Nameserver %s resolves to: %v", ns, ips)
+
+			// If we have the expected IP, verify it matches
+			if i < len(nameserverServers) {
+				expectedIP := nameserverServers[i].IP
+				found := false
+				for _, ip := range ips {
+					if ip == expectedIP {
+						found = true
+						break
+					}
+				}
+				assert.True(t, found, "Glue record for %s should point to %s (got %v)", ns, expectedIP, ips)
+			}
+		}
+	})
+}
+
+// TestNameserver_CoreDNSResponds tests that our CoreDNS servers respond to queries
+func TestNameserver_CoreDNSResponds(t *testing.T) {
+	e2e.SkipIfLocal(t)
+
+	env, err := e2e.LoadTestEnv()
+	require.NoError(t, err, "Failed to load test environment")
+
+	nameserverServers := e2e.GetNameserverServers(env.Config)
+	if len(nameserverServers) == 0 {
+		t.Skip("No servers marked as nameservers in config")
+	}
+
+	t.Run("CoreDNS servers respond to queries", func(t *testing.T) {
+		for _, server := range nameserverServers {
+			t.Run(server.Name, func(t *testing.T) {
+				// Create a custom resolver that queries this specific server
+				resolver := &net.Resolver{
+					PreferGo: true,
+					Dial: func(ctx context.Context, network, address string) (net.Conn, error) {
+						d := net.Dialer{
+							Timeout: 5 * time.Second,
+						}
+						return d.DialContext(ctx, "udp", server.IP+":53")
+					},
+				}
+
+				ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+				defer cancel()
+
+				// Query the base domain
+				ips, err := resolver.LookupHost(ctx, env.BaseDomain)
+				if err != nil {
+					// Log the error but don't fail - server might be configured differently
+					t.Logf("⚠ CoreDNS at %s (%s) query error: %v", server.Name, server.IP, err)
+					return
+				}
+
+				t.Logf("✓ CoreDNS at %s (%s) responded: %s -> %v", server.Name, server.IP, env.BaseDomain, ips)
+				assert.NotEmpty(t, ips, "CoreDNS should return IP addresses")
+			})
+		}
+	})
+}
+
+// TestNameserver_QueryLatency tests DNS query latency from our nameservers
+func TestNameserver_QueryLatency(t *testing.T) {
+	e2e.SkipIfLocal(t)
+
+	env, err := e2e.LoadTestEnv()
+	require.NoError(t, err, "Failed to load test environment")
+
+	nameserverServers := e2e.GetNameserverServers(env.Config)
+	if len(nameserverServers) == 0 {
+		t.Skip("No servers marked as nameservers in config")
+	}
+
+	t.Run("DNS query latency is acceptable", func(t *testing.T) {
+		for _, server := range nameserverServers {
+			resolver := &net.Resolver{
+				PreferGo: true,
+				Dial: func(ctx context.Context, network, address string) (net.Conn, error) {
+					d := net.Dialer{
+						Timeout: 5 * time.Second,
+					}
+					return d.DialContext(ctx, "udp", server.IP+":53")
+				},
+			}
+
+			ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+			defer cancel()
+
+			start := time.Now()
+			_, err := resolver.LookupHost(ctx, env.BaseDomain)
+			latency := time.Since(start)
+
+			if err != nil {
+				t.Logf("⚠ Query to %s failed: %v", server.Name, err)
+				continue
+			}
+
+			t.Logf("DNS latency from %s (%s): %v", server.Name, server.IP, latency)
+
+			// DNS queries should be fast (under 500ms is reasonable)
+			assert.Less(t, latency, 500*time.Millisecond,
+				"DNS query to %s should complete in under 500ms", server.Name)
+		}
+	})
+}

pkg/environments/production/config.go

@@ -134,9 +134,13 @@ func (cg *ConfigGenerator) GenerateNodeConfig(peerAddresses []string, vpsIP stri
 	var rqliteJoinAddr string
 	if joinAddress != "" {
 		// Use explicitly provided join address
-		// If it contains :7001 and HTTPS is enabled, update to :7002
+		// Adjust port based on HTTPS mode:
+		// - HTTPS enabled: use port 7002 (direct RQLite TLS, bypassing SNI gateway)
+		// - HTTPS disabled: use port 7001 (standard RQLite Raft port)
 		if enableHTTPS && strings.Contains(joinAddress, ":7001") {
 			rqliteJoinAddr = strings.Replace(joinAddress, ":7001", ":7002", 1)
+		} else if !enableHTTPS && strings.Contains(joinAddress, ":7002") {
+			rqliteJoinAddr = strings.Replace(joinAddress, ":7002", ":7001", 1)
 		} else {
 			rqliteJoinAddr = joinAddress
 		}

pkg/installer/installer.go

@@ -197,8 +197,9 @@ func (m *Model) handleEnter() (tea.Model, tea.Cmd) {
 		}
 		m.config.PeerIP = peerIP
 
-		// Auto-populate join address (direct RQLite TLS on port 7002) and bootstrap peers
-		m.config.JoinAddress = fmt.Sprintf("%s:7002", peerIP)
+		// Auto-populate join address using port 7001 (standard RQLite Raft port)
+		// config.go will adjust to 7002 if HTTPS/SNI is enabled
+		m.config.JoinAddress = fmt.Sprintf("%s:7001", peerIP)
 		m.config.Peers = []string{
 			fmt.Sprintf("/dns4/%s/tcp/4001/p2p/%s", peerDomain, disc.PeerID),
 		}

pkg/rqlite/process.go

@@ -46,6 +46,15 @@ func (r *RQLiteManager) launchProcess(ctx context.Context, rqliteDataDir string)
 	if r.config.RQLiteJoinAddress != "" {
 		r.logger.Info("Joining RQLite cluster", zap.String("join_address", r.config.RQLiteJoinAddress))
 
+		peersJSONPath := filepath.Join(rqliteDataDir, "raft", "peers.json")
+		if _, err := os.Stat(peersJSONPath); err == nil {
+			r.logger.Info("Removing existing peers.json before joining cluster",
+				zap.String("path", peersJSONPath))
+			if err := os.Remove(peersJSONPath); err != nil {
+				r.logger.Warn("Failed to remove peers.json", zap.Error(err))
+			}
+		}
+
 		joinArg := r.config.RQLiteJoinAddress
 		if strings.HasPrefix(joinArg, "http://") {
 			joinArg = strings.TrimPrefix(joinArg, "http://")
@@ -236,4 +245,3 @@ func (r *RQLiteManager) waitForJoinTarget(ctx context.Context, joinAddress strin
 
 	return lastErr
 }
-