package checks import ( "testing" "github.com/DeBrosOfficial/network/pkg/inspector" ) func TestCheckRQLite_Unresponsive(t *testing.T) { nd := makeNodeData("1.1.1.1", "node") nd.RQLite = &inspector.RQLiteData{Responsive: false} data := makeCluster(map[string]*inspector.NodeData{"1.1.1.1": nd}) results := CheckRQLite(data) expectStatus(t, results, "rqlite.responsive", inspector.StatusFail) // Should return early — no raft_state check if findCheck(results, "rqlite.raft_state") != nil { t.Error("should not check raft_state when unresponsive") } } func TestCheckRQLite_HealthyLeader(t *testing.T) { nd := makeNodeData("1.1.1.1", "node") nd.RQLite = &inspector.RQLiteData{ Responsive: true, StrongRead: true, Readyz: &inspector.RQLiteReadyz{Ready: true, Node: "ready", Leader: "ready", Store: "ready"}, Status: &inspector.RQLiteStatus{ RaftState: "Leader", LeaderNodeID: "node1", Voter: true, NumPeers: 2, Term: 5, CommitIndex: 1000, AppliedIndex: 1000, FsmPending: 0, LastLogTerm: 5, DBAppliedIndex: 1000, FsmIndex: 1000, LastSnapshot: 995, DBSizeFriendly: "1.2MB", Goroutines: 50, HeapAlloc: 100 * 1024 * 1024, // 100MB Version: "8.0.0", }, Nodes: map[string]*inspector.RQLiteNode{ "node1:5001": {Addr: "node1:5001", Reachable: true, Leader: true, Voter: true}, "node2:5001": {Addr: "node2:5001", Reachable: true, Leader: false, Voter: true}, "node3:5001": {Addr: "node3:5001", Reachable: true, Leader: false, Voter: true}, }, DebugVars: &inspector.RQLiteDebugVars{}, } data := makeCluster(map[string]*inspector.NodeData{"1.1.1.1": nd}) results := CheckRQLite(data) expectStatus(t, results, "rqlite.responsive", inspector.StatusPass) expectStatus(t, results, "rqlite.readyz", inspector.StatusPass) expectStatus(t, results, "rqlite.raft_state", inspector.StatusPass) expectStatus(t, results, "rqlite.leader_known", inspector.StatusPass) expectStatus(t, results, "rqlite.voter", inspector.StatusPass) expectStatus(t, results, "rqlite.commit_applied_gap", inspector.StatusPass) expectStatus(t, results, "rqlite.fsm_pending", inspector.StatusPass) expectStatus(t, results, "rqlite.db_fsm_sync", inspector.StatusPass) expectStatus(t, results, "rqlite.strong_read", inspector.StatusPass) expectStatus(t, results, "rqlite.all_reachable", inspector.StatusPass) expectStatus(t, results, "rqlite.goroutines", inspector.StatusPass) expectStatus(t, results, "rqlite.memory", inspector.StatusPass) expectStatus(t, results, "rqlite.query_errors", inspector.StatusPass) expectStatus(t, results, "rqlite.execute_errors", inspector.StatusPass) expectStatus(t, results, "rqlite.leader_not_found", inspector.StatusPass) expectStatus(t, results, "rqlite.snapshot_errors", inspector.StatusPass) expectStatus(t, results, "rqlite.client_health", inspector.StatusPass) } func TestCheckRQLite_RaftStates(t *testing.T) { tests := []struct { state string status inspector.Status }{ {"Leader", inspector.StatusPass}, {"Follower", inspector.StatusPass}, {"Candidate", inspector.StatusWarn}, {"Shutdown", inspector.StatusFail}, {"Unknown", inspector.StatusFail}, } for _, tt := range tests { t.Run(tt.state, func(t *testing.T) { nd := makeNodeData("1.1.1.1", "node") nd.RQLite = &inspector.RQLiteData{ Responsive: true, Status: &inspector.RQLiteStatus{ RaftState: tt.state, LeaderNodeID: "node1", Voter: true, }, } data := makeCluster(map[string]*inspector.NodeData{"1.1.1.1": nd}) results := CheckRQLite(data) expectStatus(t, results, "rqlite.raft_state", tt.status) }) } } func TestCheckRQLite_ReadyzFail(t *testing.T) { nd := makeNodeData("1.1.1.1", "node") nd.RQLite = &inspector.RQLiteData{ Responsive: true, Readyz: &inspector.RQLiteReadyz{Ready: false, Node: "ready", Leader: "not ready", Store: "ready"}, Status: &inspector.RQLiteStatus{RaftState: "Follower", LeaderNodeID: "n1", Voter: true}, } data := makeCluster(map[string]*inspector.NodeData{"1.1.1.1": nd}) results := CheckRQLite(data) expectStatus(t, results, "rqlite.readyz", inspector.StatusFail) } func TestCheckRQLite_CommitAppliedGap(t *testing.T) { tests := []struct { name string commit uint64 applied uint64 status inspector.Status }{ {"no gap", 1000, 1000, inspector.StatusPass}, {"small gap", 1002, 1000, inspector.StatusPass}, {"lagging", 1050, 1000, inspector.StatusWarn}, {"severely behind", 2000, 1000, inspector.StatusFail}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { nd := makeNodeData("1.1.1.1", "node") nd.RQLite = &inspector.RQLiteData{ Responsive: true, Status: &inspector.RQLiteStatus{ RaftState: "Follower", LeaderNodeID: "n1", Voter: true, CommitIndex: tt.commit, AppliedIndex: tt.applied, }, } data := makeCluster(map[string]*inspector.NodeData{"1.1.1.1": nd}) results := CheckRQLite(data) expectStatus(t, results, "rqlite.commit_applied_gap", tt.status) }) } } func TestCheckRQLite_FsmPending(t *testing.T) { tests := []struct { name string pending uint64 status inspector.Status }{ {"zero", 0, inspector.StatusPass}, {"small", 5, inspector.StatusWarn}, {"backlog", 100, inspector.StatusFail}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { nd := makeNodeData("1.1.1.1", "node") nd.RQLite = &inspector.RQLiteData{ Responsive: true, Status: &inspector.RQLiteStatus{ RaftState: "Follower", LeaderNodeID: "n1", Voter: true, FsmPending: tt.pending, }, } data := makeCluster(map[string]*inspector.NodeData{"1.1.1.1": nd}) results := CheckRQLite(data) expectStatus(t, results, "rqlite.fsm_pending", tt.status) }) } } func TestCheckRQLite_StrongReadFail(t *testing.T) { nd := makeNodeData("1.1.1.1", "node") nd.RQLite = &inspector.RQLiteData{ Responsive: true, StrongRead: false, Status: &inspector.RQLiteStatus{RaftState: "Follower", LeaderNodeID: "n1", Voter: true}, } data := makeCluster(map[string]*inspector.NodeData{"1.1.1.1": nd}) results := CheckRQLite(data) expectStatus(t, results, "rqlite.strong_read", inspector.StatusFail) } func TestCheckRQLite_DebugVarsErrors(t *testing.T) { nd := makeNodeData("1.1.1.1", "node") nd.RQLite = &inspector.RQLiteData{ Responsive: true, Status: &inspector.RQLiteStatus{RaftState: "Leader", LeaderNodeID: "n1", Voter: true}, DebugVars: &inspector.RQLiteDebugVars{ QueryErrors: 5, ExecuteErrors: 3, LeaderNotFound: 1, SnapshotErrors: 2, ClientRetries: 10, ClientTimeouts: 1, }, } data := makeCluster(map[string]*inspector.NodeData{"1.1.1.1": nd}) results := CheckRQLite(data) expectStatus(t, results, "rqlite.query_errors", inspector.StatusWarn) expectStatus(t, results, "rqlite.execute_errors", inspector.StatusWarn) expectStatus(t, results, "rqlite.leader_not_found", inspector.StatusFail) expectStatus(t, results, "rqlite.snapshot_errors", inspector.StatusFail) expectStatus(t, results, "rqlite.client_health", inspector.StatusWarn) } func TestCheckRQLite_Goroutines(t *testing.T) { tests := []struct { name string goroutines int status inspector.Status }{ {"healthy", 50, inspector.StatusPass}, {"elevated", 500, inspector.StatusWarn}, {"high", 2000, inspector.StatusFail}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { nd := makeNodeData("1.1.1.1", "node") nd.RQLite = &inspector.RQLiteData{ Responsive: true, Status: &inspector.RQLiteStatus{ RaftState: "Leader", LeaderNodeID: "n1", Voter: true, Goroutines: tt.goroutines, }, } data := makeCluster(map[string]*inspector.NodeData{"1.1.1.1": nd}) results := CheckRQLite(data) expectStatus(t, results, "rqlite.goroutines", tt.status) }) } } // --- Cross-node tests --- func makeRQLiteCluster(leaderHost string, states map[string]string, term uint64) *inspector.ClusterData { nodes := map[string]*inspector.NodeData{} rqliteNodes := map[string]*inspector.RQLiteNode{} for host := range states { rqliteNodes[host+":5001"] = &inspector.RQLiteNode{ Addr: host + ":5001", Reachable: true, Voter: true, Leader: states[host] == "Leader", } } for host, state := range states { nd := makeNodeData(host, "node") nd.RQLite = &inspector.RQLiteData{ Responsive: true, Status: &inspector.RQLiteStatus{ RaftState: state, LeaderNodeID: leaderHost, Voter: true, Term: term, AppliedIndex: 1000, CommitIndex: 1000, Version: "8.0.0", DBSize: 4096, }, Nodes: rqliteNodes, } nodes[host] = nd } return makeCluster(nodes) } func TestCheckRQLite_CrossNode_SingleLeader(t *testing.T) { data := makeRQLiteCluster("1.1.1.1", map[string]string{ "1.1.1.1": "Leader", "2.2.2.2": "Follower", "3.3.3.3": "Follower", }, 5) results := CheckRQLite(data) expectStatus(t, results, "rqlite.single_leader", inspector.StatusPass) expectStatus(t, results, "rqlite.term_consistent", inspector.StatusPass) expectStatus(t, results, "rqlite.leader_agreement", inspector.StatusPass) expectStatus(t, results, "rqlite.index_convergence", inspector.StatusPass) expectStatus(t, results, "rqlite.version_consistent", inspector.StatusPass) expectStatus(t, results, "rqlite.quorum", inspector.StatusPass) } func TestCheckRQLite_CrossNode_NoLeader(t *testing.T) { data := makeRQLiteCluster("", map[string]string{ "1.1.1.1": "Candidate", "2.2.2.2": "Candidate", "3.3.3.3": "Candidate", }, 5) results := CheckRQLite(data) expectStatus(t, results, "rqlite.single_leader", inspector.StatusFail) } func TestCheckRQLite_CrossNode_SplitBrain(t *testing.T) { nodes := map[string]*inspector.NodeData{} for _, host := range []string{"1.1.1.1", "2.2.2.2", "3.3.3.3"} { nd := makeNodeData(host, "node") state := "Follower" leaderID := "1.1.1.1" if host == "1.1.1.1" || host == "2.2.2.2" { state = "Leader" leaderID = host } nd.RQLite = &inspector.RQLiteData{ Responsive: true, Status: &inspector.RQLiteStatus{ RaftState: state, LeaderNodeID: leaderID, Voter: true, Term: 5, AppliedIndex: 1000, }, } nodes[host] = nd } data := makeCluster(nodes) results := CheckRQLite(data) expectStatus(t, results, "rqlite.single_leader", inspector.StatusFail) } func TestCheckRQLite_CrossNode_TermDivergence(t *testing.T) { nodes := map[string]*inspector.NodeData{} terms := map[string]uint64{"1.1.1.1": 5, "2.2.2.2": 5, "3.3.3.3": 6} for host, term := range terms { nd := makeNodeData(host, "node") nd.RQLite = &inspector.RQLiteData{ Responsive: true, Status: &inspector.RQLiteStatus{ RaftState: "Follower", LeaderNodeID: "1.1.1.1", Voter: true, Term: term, AppliedIndex: 1000, }, } nodes[host] = nd } data := makeCluster(nodes) results := CheckRQLite(data) expectStatus(t, results, "rqlite.term_consistent", inspector.StatusFail) } func TestCheckRQLite_CrossNode_IndexLagging(t *testing.T) { nodes := map[string]*inspector.NodeData{} applied := map[string]uint64{"1.1.1.1": 1000, "2.2.2.2": 1000, "3.3.3.3": 500} for host, idx := range applied { nd := makeNodeData(host, "node") state := "Follower" if host == "1.1.1.1" { state = "Leader" } nd.RQLite = &inspector.RQLiteData{ Responsive: true, Status: &inspector.RQLiteStatus{ RaftState: state, LeaderNodeID: "1.1.1.1", Voter: true, Term: 5, AppliedIndex: idx, CommitIndex: idx, }, } nodes[host] = nd } data := makeCluster(nodes) results := CheckRQLite(data) expectStatus(t, results, "rqlite.index_convergence", inspector.StatusWarn) } func TestCheckRQLite_CrossNode_SkipSingleNode(t *testing.T) { nd := makeNodeData("1.1.1.1", "node") nd.RQLite = &inspector.RQLiteData{ Responsive: true, Status: &inspector.RQLiteStatus{RaftState: "Leader", LeaderNodeID: "n1", Voter: true, Term: 5, AppliedIndex: 1000}, } data := makeCluster(map[string]*inspector.NodeData{"1.1.1.1": nd}) results := CheckRQLite(data) expectStatus(t, results, "rqlite.cross_node", inspector.StatusSkip) } func TestCheckRQLite_NilRQLiteData(t *testing.T) { nd := makeNodeData("1.1.1.1", "node") // nd.RQLite is nil — no per-node checks, but cross-node skip is expected data := makeCluster(map[string]*inspector.NodeData{"1.1.1.1": nd}) results := CheckRQLite(data) // Should only have the cross-node skip (not enough nodes) for _, r := range results { if r.Status != inspector.StatusSkip { t.Errorf("unexpected non-skip result: %s (status=%s)", r.ID, r.Status) } } }