etcd/raft: How does leader node discard data that is rejected by most followers.

[yongguangl]

Question

In the raftexample example, if the majority of followers reject the leader's request, how does the leader node discard this data?

I want to submit a write request to the leader node. When the follower node finds that the data does not meet its expectations, it will reject the leader's request. When the majority of nodes reject the request, the data will not be written and discarded.

I set the follower MsgAppResp Messages[i].Reject = true, however, it seems that the leader will keep sending retry requests and cannot discard this data.

How should I solve this problem, thank you！

// event loop on raft state machine updates
	for {
		select {
		case <-ticker.C:
			rc.node.Tick()

		// store raft entries to wal, then publish over commit channel
		case rd := <-rc.node.Ready():
			data, _ := json.Marshal(rd)
			if len(rd.CommittedEntries) > 0 {
				fmt.Println("CommittedEntries")
			}
			var reject bool
			for i, message := range rd.Messages {
				switch message.Type {
				case raftpb.MsgHeartbeat:
				case raftpb.MsgHeartbeatResp:
				case raftpb.MsgProp:
				case raftpb.MsgApp:
				case raftpb.MsgAppResp:
					fmt.Println(string(data))
					for _, entry := range rd.Entries {
						var dataKv kv
						dec := gob.NewDecoder(bytes.NewBuffer(entry.Data))
						if err := dec.Decode(&dataKv); err != nil {
							continue
						}

						// Simulate the follower node to reject the request
						if rc.node.Status().Lead != uint64(rc.id) {
							rd.Messages[i].Reject = true
							reject = true
						}

					}
				default:
				}
			}

			if reject {
				fmt.Println("reject")
				rc.transport.Send(rd.Messages)
				rc.node.Advance()
				continue
			}

			rc.wal.Save(rd.HardState, rd.Entries)
			if !raft.IsEmptySnap(rd.Snapshot) {
				rc.saveSnap(rd.Snapshot)
				rc.raftStorage.ApplySnapshot(rd.Snapshot)
				rc.publishSnapshot(rd.Snapshot)
			}
			rc.raftStorage.Append(rd.Entries)
			rc.transport.Send(rd.Messages)
			applyDoneC, ok := rc.publishEntries(rc.entriesToApply(rd.CommittedEntries))
			if !ok {
				rc.stop()
				return
			}
			rc.maybeTriggerSnapshot(applyDoneC)
			rc.node.Advance()

		case err := <-rc.transport.ErrorC:
			rc.writeError(err)
			return

		case <-rc.stopc:
			rc.stop()
			return
		}
	}

log

17:34:46 raftexample1 | {"term":0,"vote":0,"commit":0,"ReadStates":null,"Entries":[{"Term":2,"Index":5,"Type":0,"Data":"H38DAQECa3YB/4AAAQIBA0tleQEMAAEDVmFsAQwAAAAc/4ABBi90ZXN0MQEPMTE4NzYyMzJzZGFzZGFzAA=="}],"Snapshot":{"metadata":{"conf_state":{"auto_leave":false},"index":0,"term":0}},"CommittedEntries":null,"Messages":[{"type":4,"to":2,"from":1,"term":2,"logTerm":0,"index":5,"entries":null,"commit":0,"snapshot":{"metadata":{"conf_state":{"auto_leave":false},"index":0,"term":0}},"reject":false,"rejectHint":0}],"MustSync":true}
17:34:46 raftexample3 | {"term":0,"vote":0,"commit":0,"ReadStates":null,"Entries":[{"Term":2,"Index":5,"Type":0,"Data":"H38DAQECa3YB/4AAAQIBA0tleQEMAAEDVmFsAQwAAAAc/4ABBi90ZXN0MQEPMTE4NzYyMzJzZGFzZGFzAA=="}],"Snapshot":{"metadata":{"conf_state":{"auto_leave":false},"index":0,"term":0}},"CommittedEntries":null,"Messages":[{"type":4,"to":2,"from":3,"term":2,"logTerm":0,"index":5,"entries":null,"commit":0,"snapshot":{"metadata":{"conf_state":{"auto_leave":false},"index":0,"term":0}},"reject":false,"rejectHint":0}],"MustSync":true}
17:34:46 raftexample3 | reject
17:34:46 raftexample1 | reject
17:34:46 raftexample1 | raft2024/07/25 17:34:46 INFO: replace the unstable entries from index 5
17:34:46 raftexample3 | raft2024/07/25 17:34:46 INFO: replace the unstable entries from index 5
17:34:46 raftexample3 | {"term":0,"vote":0,"commit":0,"ReadStates":null,"Entries":[{"Term":2,"Index":5,"Type":0,"Data":"H38DAQECa3YB/4AAAQIBA0tleQEMAAEDVmFsAQwAAAAc/4ABBi90ZXN0MQEPMTE4NzYyMzJzZGFzZGFzAA=="}],"Snapshot":{"metadata":{"conf_state":{"auto_leave":false},"index":0,"term":0}},"CommittedEntries":null,"Messages":[{"type":4,"to":2,"from":3,"term":2,"logTerm":0,"index":5,"entries":null,"commit":0,"snapshot":{"metadata":{"conf_state":{"auto_leave":false},"index":0,"term":0}},"reject":false,"rejectHint":0}],"MustSync":true}
17:34:46 raftexample1 | {"term":0,"vote":0,"commit":0,"ReadStates":null,"Entries":[{"Term":2,"Index":5,"Type":0,"Data":"H38DAQECa3YB/4AAAQIBA0tleQEMAAEDVmFsAQwAAAAc/4ABBi90ZXN0MQEPMTE4NzYyMzJzZGFzZGFzAA=="}],"Snapshot":{"metadata":{"conf_state":{"auto_leave":false},"index":0,"term":0}},"CommittedEntries":null,"Messages":[{"type":4,"to":2,"from":1,"term":2,"logTerm":0,"index":5,"entries":null,"commit":0,"snapshot":{"metadata":{"conf_state":{"auto_leave":false},"index":0,"term":0}},"reject":false,"rejectHint":0}],"MustSync":true}
17:34:46 raftexample1 | reject
17:34:46 raftexample3 | reject
17:34:46 raftexample1 | raft2024/07/25 17:34:46 INFO: replace the unstable entries from index 5
17:34:46 raftexample1 | {"term":0,"vote":0,"commit":0,"ReadStates":null,"Entries":[{"Term":2,"Index":5,"Type":0,"Data":"H38DAQECa3YB/4AAAQIBA0tleQEMAAEDVmFsAQwAAAAc/4ABBi90ZXN0MQEPMTE4NzYyMzJzZGFzZGFzAA=="}],"Snapshot":{"metadata":{"conf_state":{"auto_leave":false},"index":0,"term":0}},"CommittedEntries":null,"Messages":[{"type":4,"to":2,"from":1,"term":2,"logTerm":0,"index":5,"entries":null,"commit":0,"snapshot":{"metadata":{"conf_state":{"auto_leave":false},"index":0,"term":0}},"reject":false,"rejectHint":0}],"MustSync":true}
17:34:46 raftexample3 | raft2024/07/25 17:34:46 INFO: replace the unstable entries from index 5
17:34:46 raftexample3 | {"term":0,"vote":0,"commit":0,"ReadStates":null,"Entries":[{"Term":2,"Index":5,"Type":0,"Data":"H38DAQECa3YB/4AAAQIBA0tleQEMAAEDVmFsAQwAAAAc/4ABBi90ZXN0MQEPMTE4NzYyMzJzZGFzZGFzAA=="}],"Snapshot":{"metadata":{"conf_state":{"auto_leave":false},"index":0,"term":0}},"CommittedEntries":null,"Messages":[{"type":4,"to":2,"from":3,"term":2,"logTerm":0,"index":5,"entries":null,"commit":0,"snapshot":{"metadata":{"conf_state":{"auto_leave":false},"index":0,"term":0}},"reject":false,"rejectHint":0}],"MustSync":true}
17:34:46 raftexample1 | reject
17:34:46 raftexample3 | reject
17:34:46 raftexample1 | raft2024/07/25 17:34:46 INFO: replace the unstable entries from index 5
17:34:46 raftexample1 | {"term":0,"vote":0,"commit":0,"ReadStates":null,"Entries":[{"Term":2,"Index":5,"Type":0,"Data":"H38DAQECa3YB/4AAAQIBA0tleQEMAAEDVmFsAQwAAAAc/4ABBi90ZXN0MQEPMTE4NzYyMzJzZGFzZGFzAA=="}],"Snapshot":{"metadata":{"conf_state":{"auto_leave":false},"index":0,"term":0}},"CommittedEntries":null,"Messages":[{"type":4,"to":2,"from":1,"term":2,"logTerm":0,"index":5,"entries":null,"commit":0,"snapshot":{"metadata":{"conf_state":{"auto_leave":false},"index":0,"term":0}},"reject":false,"rejectHint":0}],"MustSync":true}
17:34:46 raftexample1 | reject
17:34:46 raftexample3 | raft2024/07/25 17:34:46 INFO: replace the unstable entries from index 5
17:34:46 raftexample3 | {"term":0,"vote":0,"commit":0,"ReadStates":null,"Entries":[{"Term":2,"Index":5,"Type":0,"Data":"H38DAQECa3YB/4AAAQIBA0tleQEMAAEDVmFsAQwAAAAc/4ABBi90ZXN0MQEPMTE4NzYyMzJzZGFzZGFzAA=="}],"Snapshot":{"metadata":{"conf_state":{"auto_leave":false},"index":0,"term":0}},"CommittedEntries":null,"Messages":[{"type":4,"to":2,"from":3,"term":2,"logTerm":0,"index":5,"entries":null,"commit":0,"snapshot":{"metadata":{"conf_state":{"auto_leave":false},"index":0,"term":0}},"reject":false,"rejectHint":0}],"MustSync":true}
17:34:46 raftexample3 | reject
17:34:46 raftexample3 | raft2024/07/25 17:34:46 INFO: replace the unstable entries from index 5
17:34:46 raftexample3 | {"term":0,"vote":0,"commit":0,"ReadStates":null,"Entries":[{"Term":2,"Index":5,"Type":0,"Data":"H38DAQECa3YB/4AAAQIBA0tleQEMAAEDVmFsAQwAAAAc/4ABBi90ZXN0MQEPMTE4NzYyMzJzZGFzZGFzAA=="}],"Snapshot":{"metadata":{"conf_state":{"auto_leave":false},"index":0,"term":0}},"CommittedEntries":null,"Messages":[{"type":4,"to":2,"from":3,"term":2,"logTerm":0,"index":5,"entries":null,"commit":0,"snapshot":{"metadata":{"conf_state":{"auto_leave":false},"index":0,"term":0}},"reject":false,"rejectHint":0}],"MustSync":true}
17:34:46 raftexample3 | reject
17:34:46 raftexample1 | raft2024/07/25 17:34:46 INFO: replace the unstable entries from index 5
17:34:46 raftexample1 | {"term":0,"vote":0,"commit":0,"ReadStates":null,"Entries":[{"Term":2,"Index":5,"Type":0,"Data":"H38DAQECa3YB/4AAAQIBA0tleQEMAAEDVmFsAQwAAAAc/4ABBi90ZXN0MQEPMTE4NzYyMzJzZGFzZGFzAA=="}],"Snapshot":{"metadata":{"conf_state":{"auto_leave":false},"index":0,"term":0}},"CommittedEntries":null,"Messages":[{"type":4,"to":2,"from":1,"term":2,"logTerm":0,"index":5,"entries":null,"commit":0,"snapshot":{"metadata":{"conf_state":{"auto_leave":false},"index":0,"term":0}},"reject":false,"rejectHint":0}],"MustSync":true}
17:34:46 raftexample1 | reject
17:34:46 raftexample1 | raft2024/07/25 17:34:46 INFO: replace the unstable entries from index 5
17:34:46 raftexample3 | raft2024/07/25 17:34:46 INFO: replace the unstable entries from index 5
17:34:46 raftexample3 | {"term":0,"vote":0,"commit":0,"ReadStates":null,"Entries":[{"Term":2,"Index":5,"Type":0,"Data":"H38DAQECa3YB/4AAAQIBA0tleQEMAAEDVmFsAQwAAAAc/4ABBi90ZXN0MQEPMTE4NzYyMzJzZGFzZGFzAA=="}],"Snapshot":{"metadata":{"conf_state":{"auto_leave":false},"index":0,"term":0}},"CommittedEntries":null,"Messages":[{"type":4,"to":2,"from":3,"term":2,"logTerm":0,"index":5,"entries":null,"commit":0,"snapshot":{"metadata":{"conf_state":{"auto_leave":false},"index":0,"term":0}},"reject":false,"rejectHint":0}],"MustSync":true}
17:34:46 raftexample1 | {"term":0,"vote":0,"commit":0,"ReadStates":null,"Entries":[{"Term":2,"Index":5,"Type":0,"Data":"H38DAQECa3YB/4AAAQIBA0tleQEMAAEDVmFsAQwAAAAc/4ABBi90ZXN0MQEPMTE4NzYyMzJzZGFzZGFzAA=="}],"Snapshot":{"metadata":{"conf_state":{"auto_leave":false},"index":0,"term":0}},"CommittedEntries":null,"Messages":[{"type":4,"to":2,"from":1,"term":2,"logTerm":0,"index":5,"entries":null,"commit":0,"snapshot":{"metadata":{"conf_state":{"auto_leave":false},"index":0,"term":0}},"reject":false,"rejectHint":0}],"MustSync":true}
17:34:46 raftexample1 | reject
17:34:46 raftexample3 | reject
17:34:46 raftexample1 | raft2024/07/25 17:34:46 INFO: replace the unstable entries from index 5
17:34:46 raftexample1 | {"term":0,"vote":0,"commit":0,"ReadStates":null,"Entries":[{"Term":2,"Index":5,"Type":0,"Data":"H38DAQECa3YB/4AAAQIBA0tleQEMAAEDVmFsAQwAAAAc/4ABBi90ZXN0MQEPMTE4NzYyMzJzZGFzZGFzAA=="}],"Snapshot":{"metadata":{"conf_state":{"auto_leave":false},"index":0,"term":0}},"CommittedEntries":null,"Messages":[{"type":4,"to":2,"from":1,"term":2,"logTerm":0,"index":5,"entries":null,"commit":0,"snapshot":{"metadata":{"conf_state":{"auto_leave":false},"index":0,"term":0}},"reject":false,"rejectHint":0}],"MustSync":true}
17:34:46 raftexample3 | raft2024/07/25 17:34:46 INFO: replace the unstable entries from index 5
17:34:46 raftexample3 | {"term":0,"vote":0,"commit":0,"ReadStates":null,"Entries":[{"Term":2,"Index":5,"Type":0,"Data":"H38DAQECa3YB/4AAAQIBA0tleQEMAAEDVmFsAQwAAAAc/4ABBi90ZXN0MQEPMTE4NzYyMzJzZGFzZGFzAA=="}],"Snapshot":{"metadata":{"conf_state":{"auto_leave":false},"index":0,"term":0}},"CommittedEntries":null,"Messages":[{"type":4,"to":2,"from":3,"term":2,"logTerm":0,"index":5,"entries":null,"commit":0,"snapshot":{"metadata":{"conf_state":{"auto_leave":false},"index":0,"term":0}},"reject":false,"rejectHint":0}],"MustSync":true}
17:34:46 raftexample1 | reject
17:34:46 raftexample3 | reject
17:34:46 raftexample1 | raft2024/07/25 17:34:46 INFO: replace the unstable entries from index 5
17:34:46 raftexample1 | {"term":0,"vote":0,"commit":0,"ReadStates":null,"Entries":[{"Term":2,"Index":5,"Type":0,"Data":"H38DAQECa3YB/4AAAQIBA0tleQEMAAEDVmFsAQwAAAAc/4ABBi90ZXN0MQEPMTE4NzYyMzJzZGFzZGFzAA=="}],"Snapshot":{"metadata":{"conf_state":{"auto_leave":false},"index":0,"term":0}},"CommittedEntries":null,"Messages":[{"type":4,"to":2,"from":1,"term":2,"logTerm":0,"index":5,"entries":null,"commit":0,"snapshot":{"metadata":{"conf_state":{"auto_leave":false},"index":0,"term":0}},"reject":false,"rejectHint":0}],"MustSync":true}
17:34:46 raftexample3 | raft2024/07/25 17:34:46 INFO: replace the unstable entries from index 5
17:34:46 raftexample3 | {"term":0,"vote":0,"commit":0,"ReadStates":null,"Entries":[{"Term":2,"Index":5,"Type":0,"Data":"H38DAQECa3YB/4AAAQIBA0tleQEMAAEDVmFsAQwAAAAc/4ABBi90ZXN0MQEPMTE4NzYyMzJzZGFzZGFzAA=="}],"Snapshot":{"metadata":{"conf_state":{"auto_leave":false},"index":0,"term":0}},"CommittedEntries":null,"Messages":[{"type":4,"to":2,"from":3,"term":2,"logTerm":0,"index":5,"entries":null,"commit":0,"snapshot":{"metadata":{"conf_state":{"auto_leave":false},"index":0,"term":0}},"reject":false,"rejectHint":0}],"MustSync":true}
17:34:46 raftexample1 | reject
17:34:46 raftexample3 | reject
17:34:46 raftexample3 | raft2024/07/25 17:34:46 INFO: replace the unstable entries from index 5
17:34:46 raftexample3 | {"term":0,"vote":0,"commit":0,"ReadStates":null,"Entries":[{"Term":2,"Index":5,"Type":0,"Data":"H38DAQECa3YB/4AAAQIBA0tleQEMAAEDVmFsAQwAAAAc/4ABBi90ZXN0MQEPMTE4NzYyMzJzZGFzZGFzAA=="}],"Snapshot":{"metadata":{"conf_state":{"auto_leave":false},"index":0,"term":0}},"CommittedEntries":null,"Messages":[{"type":4,"to":2,"from":3,"term":2,"logTerm":0,"index":5,"entries":null,"commit":0,"snapshot":{"metadata":{"conf_state":{"auto_leave":false},"index":0,"term":0}},"reject":false,"rejectHint":0}],"MustSync":true}
17:34:46 raftexample1 | raft2024/07/25 17:34:46 INFO: replace the unstable entries from index 5
17:34:46 raftexample1 | {"term":0,"vote":0,"commit":0,"ReadStates":null,"Entries":[{"Term":2,"Index":5,"Type":0,"Data":"H38DAQECa3YB/4AAAQIBA0tleQEMAAEDVmFsAQwAAAAc/4ABBi90ZXN0MQEPMTE4NzYyMzJzZGFzZGFzAA=="}],"Snapshot":{"metadata":{"conf_state":{"auto_leave":false},"index":0,"term":0}},"CommittedEntries":null,"Messages":[{"type":4,"to":2,"from":1,"term":2,"logTerm":0,"index":5,"entries":null,"commit":0,"snapshot":{"metadata":{"conf_state":{"auto_leave":false},"index":0,"term":0}},"reject":false,"rejectHint":0}],"MustSync":true}
17:34:46 raftexample1 | reject
17:34:46 raftexample3 | reject
17:34:46 raftexample1 | raft2024/07/25 17:34:46 INFO: replace the unstable entries from index 5
17:34:46 raftexample3 | raft2024/07/25 17:34:46 INFO: replace the unstable entries from index 5
17:34:46 raftexample3 | {"term":0,"vote":0,"commit":0,"ReadStates":null,"Entries":[{"Term":2,"Index":5,"Type":0,"Data":"H38DAQECa3YB/4AAAQIBA0tleQEMAAEDVmFsAQwAAAAc/4ABBi90ZXN0MQEPMTE4NzYyMzJzZGFzZGFzAA=="}],"Snapshot":{"metadata":{"conf_state":{"auto_leave":false},"index":0,"term":0}},"CommittedEntries":null,"Messages":[{"type":4,"to":2,"from":3,"term":2,"logTerm":0,"index":5,"entries":null,"commit":0,"snapshot":{"metadata":{"conf_state":{"auto_leave":false},"index":0,"term":0}},"reject":false,"rejectHint":0}],"MustSync":true}
17:34:46 raftexample1 | {"term":0,"vote":0,"commit":0,"ReadStates":null,"Entries":[{"Term":2,"Index":5,"Type":0,"Data":"H38DAQECa3YB/4AAAQIBA0tleQEMAAEDVmFsAQwAAAAc/4ABBi90ZXN0MQEPMTE4NzYyMzJzZGFzZGFzAA=="}],"Snapshot":{"metadata":{"conf_state":{"auto_leave":false},"index":0,"term":0}},"CommittedEntries":null,"Messages":[{"type":4,"to":2,"from":1,"term":2,"logTerm":0,"index":5,"entries":null,"commit":0,"snapshot":{"metadata":{"conf_state":{"auto_leave":false},"index":0,"term":0}},"reject":false,"rejectHint":0}],"MustSync":true}
... ...

[ahrtr]

When a proposal/log isn't accepted by the quorum number of members (including the leader itself), then the log isn't committed yet. It will be eventually committed once it's replicated to majority members, or overwritten by a new leader.

Recommend to read the raft paper.