NRG: do not count messages from removed peers towards quorum (#7589)

Ignore `appendEntryResponse` from removed peers.

Signed-off-by: Daniele Sciascia <daniele@nats.io>

Author: neilalexander

server/raft.go

@@ -3257,8 +3257,10 @@ func (n *raft) trackResponse(ar *appendEntryResponse) {
 		return
 	}
 
+	ps := n.peers[ar.peer]
+
 	// Update peer's last index.
-	if ps := n.peers[ar.peer]; ps != nil && ar.index > ps.li {
+	if ps != nil && ar.index > ps.li {
 		ps.li = ar.index
 	}
 
@@ -3273,6 +3275,12 @@ func (n *raft) trackResponse(ar *appendEntryResponse) {
 		return
 	}
 
+	// Not a peer, can't count this message towards quorum
+	if ps == nil {
+		n.Unlock()
+		return
+	}
+
 	// See if we have items to apply.
 	var sendHB bool
 

server/raft_helpers_test.go

@@ -57,6 +57,17 @@ func (sg smGroup) leader() stateMachine {
 	return nil
 }
 
+func (sg smGroup) followers() []stateMachine {
+	var f []stateMachine
+	for _, sm := range sg {
+		if sm.node().Leader() {
+			continue
+		}
+		f = append(f, sm)
+	}
+	return f
+}
+
 // Wait on a leader to be elected.
 func (sg smGroup) waitOnLeader() stateMachine {
 	expires := time.Now().Add(10 * time.Second)

server/raft_test.go

@@ -4256,3 +4256,36 @@ func TestNRGUncommittedMembershipChangeOnNewLeader(t *testing.T) {
 	err := n.ProposeRemovePeer(nats1)
 	require_Error(t, err, errMembershipChange)
 }
+
+func TestNRGProposeRemovePeerQuorum(t *testing.T) {
+	c := createJetStreamClusterExplicit(t, "R3S", 3)
+	defer c.shutdown()
+
+	rg := c.createMemRaftGroup("TEST", 3, newStateAdder)
+	rg.waitOnLeader()
+
+	leader := rg.leader().node()
+	followers := rg.followers()
+
+	require_True(t, len(followers) == 2)
+
+	// Block follower 0 and remove follower 1
+	followers[0].node().(*raft).Lock()
+	err := leader.ProposeRemovePeer(followers[1].node().ID())
+	require_NoError(t, err)
+
+	// Should not be able to make progress
+	time.Sleep(time.Second)
+	require_True(t, leader.MembershipChangeInProgress())
+
+	// Unlock the other follower and expect the membership
+	// change to eventually finish
+	followers[0].node().(*raft).Unlock()
+	checkFor(t, 10*time.Second, 200*time.Millisecond, func() error {
+		if leader.MembershipChangeInProgress() {
+			return errors.New("membership still in progress")
+		} else {
+			return nil
+		}
+	})
+}