raft: ForceNewCluster flag should only apply at startup

ForceNewCluster tells raft to use the existing state to create a new
cluster with a single member. This means that when first reading the
WAL/snapshot, the other existing managers are removed.

However, it doesn't make sense to do this more than once. For example,
if other managers are added to the cluster, and one of them takes over
as the leader, and sends the initial leader a snapshot, that snapshot
should be accepted without modifications.

Change the ForceNewCluster flag to only apply on startup.

Signed-off-by: Aaron Lehmann <aaron.lehmann@docker.com>

Author: Aaron Lehmann

manager/state/raft/raft.go

@@ -104,10 +104,6 @@ type Node struct {
 	// shutting down the node.
 	waitProp sync.WaitGroup
 
-	// forceNewCluster is a special flag used to recover from disaster
-	// scenario by pointing to an existing or backed up data directory.
-	forceNewCluster bool
-
 	confState     raftpb.ConfState
 	appliedIndex  uint64
 	snapshotIndex uint64
@@ -192,7 +188,6 @@ func NewNode(ctx context.Context, opts NewNodeOptions) *Node {
 			MaxInflightMsgs: cfg.MaxInflightMsgs,
 			Logger:          cfg.Logger,
 		},
-		forceNewCluster:     opts.ForceNewCluster,
 		stopCh:              make(chan struct{}),
 		doneCh:              make(chan struct{}),
 		removeRaftCh:        make(chan struct{}),
@@ -362,7 +357,7 @@ func (n *Node) Run(ctx context.Context) error {
 			// saveToStorage.
 			if !raft.IsEmptySnap(rd.Snapshot) {
 				// Load the snapshot data into the store
-				if err := n.restoreFromSnapshot(rd.Snapshot.Data, n.forceNewCluster); err != nil {
+				if err := n.restoreFromSnapshot(rd.Snapshot.Data, false); err != nil {
 					n.Config.Logger.Error(err)
 				}
 				n.appliedIndex = rd.Snapshot.Metadata.Index