Skip to content

Commit

Permalink
fix: avoid duplicate log lines edge case
Browse files Browse the repository at this point in the history
  • Loading branch information
smlx committed Jul 20, 2023
1 parent 789be13 commit 6923ff1
Showing 1 changed file with 18 additions and 4 deletions.
22 changes: 18 additions & 4 deletions internal/k8s/logs.go
Original file line number Diff line number Diff line change
Expand Up @@ -101,6 +101,19 @@ func (c *Client) readLogs(ctx context.Context, egSend *errgroup.Group,
linewiseCopy(ctx, fmt.Sprintf("[pod/%s/%s]", p.Name, cName), logs,
logStream)
fmt.Println("done linewiseCopy")
// The k8s API sometimes sends an event showing a healthy pod even though
// the pod is terminating. This seems to occur sometimes on scale down of
// a deployment. When this happens there is a race that occurs where the
// existing log stream closes (linewiseCopy() returns), then the
// "healthy" event comes in and the container log streaming is started
// again, only to return immediately. This can result in duplicated log
// lines being returned.
// To hack around this behaviour, pause for a couple of seconds here
// before exiting. This means that the container ID is retained in
// c.logCIDs for a brief period after logs stop streaming which causes
// "healthy pod" events from the k8s API to be ignored for that period
// and thereby avoiding duplicate log lines being returned.
time.Sleep(2)
return nil
})
}
Expand All @@ -116,10 +129,11 @@ func (c *Client) podEventHandler(ctx context.Context,
// panic if obj is not a pod, since this is specifically a pod informer
pod := obj.(*corev1.Pod)
spew.Dump(pod.Status)
if !slices.ContainsFunc(pod.Status.Conditions, func(cond corev1.PodCondition) bool {
return (pod.Status.Phase == corev1.PodRunning) &&
(cond.Type == corev1.ContainersReady && cond.Status == corev1.ConditionTrue)
}) {
if !slices.ContainsFunc(pod.Status.Conditions,
func(cond corev1.PodCondition) bool {
return cond.Type == corev1.ContainersReady &&
cond.Status == corev1.ConditionTrue
}) {
fmt.Println("pod not ready")
return
}
Expand Down

0 comments on commit 6923ff1

Please sign in to comment.