fix(machine): correctly recreate machine in store if missing, improve context handling

This commit is contained in:
Pasha Sviderski
2026-06-23 09:29:04 +10:00
parent 8aa0a96c23
commit 5f07bc5832
2 changed files with 22 additions and 18 deletions
+15 -9
View File
@@ -523,12 +523,16 @@ func (cc *clusterController) runMachineSync(ctx context.Context) error {
for { for {
select { select {
case <-ctx.Done(): case <-ctx.Done():
return nil
case <-ticker.C: // Scheduled periodic sync. case <-ticker.C: // Scheduled periodic sync.
case <-cc.syncMachineTrigger: // Immediate sync request. case <-cc.syncMachineTrigger: // Immediate sync request.
case <-dockerRestarted: // Docker daemon restarted -- engine version may have changed. case <-dockerRestarted: // Docker daemon restarted -- engine version may have changed.
} }
// A pending restart signal can race with context cancellation and win the select.
if ctx.Err() != nil {
return nil
}
if err := cc.syncMachineInfo(ctx); err != nil { if err := cc.syncMachineInfo(ctx); err != nil {
slog.Error("Failed to sync machine info to cluster store.", "err", err) slog.Error("Failed to sync machine info to cluster store.", "err", err)
} }
@@ -548,31 +552,33 @@ func (cc *clusterController) RequestMachineSync() {
// write if the info is unchanged since the last successful write. // write if the info is unchanged since the last successful write.
func (cc *clusterController) syncMachineInfo(ctx context.Context) error { func (cc *clusterController) syncMachineInfo(ctx context.Context) error {
publishedInfo, err := cc.store.GetMachine(ctx, cc.state.ID) publishedInfo, err := cc.store.GetMachine(ctx, cc.state.ID)
if err != nil { if err != nil && !errors.Is(err, store.ErrMachineNotFound) {
return fmt.Errorf("get machine from store: %w", err) return fmt.Errorf("get machine from store: %w", err)
} }
info := cc.machine.Info(ctx) info := cc.machine.Info(ctx)
if publishedInfo != nil {
// Info leaves the Docker engine version empty when the engine is unavailable. Keep the previously // Info leaves the Docker engine version empty when the engine is unavailable. Keep the previously
// published version in that case rather than overwriting it with an empty value. // published version in that case rather than overwriting it with an empty value.
if info.DockerVersion == "" { if info.DockerVersion == "" {
info.DockerVersion = publishedInfo.DockerVersion info.DockerVersion = publishedInfo.DockerVersion
} }
// Skip the write if nothing changed since the last successful sync.
if proto.Equal(info, publishedInfo) { if proto.Equal(info, publishedInfo) {
return nil return nil
} }
}
if err = cc.store.UpdateMachine(ctx, info); err != nil { if err = cc.store.UpdateMachine(ctx, info); err != nil {
if errors.Is(err, store.ErrMachineNotFound) { if !errors.Is(err, store.ErrMachineNotFound) {
// This should not happen but let's try to recreate it.
if createErr := cc.store.CreateMachine(ctx, info); createErr != nil {
return fmt.Errorf("create machine in store: %w", createErr)
}
return nil
}
return fmt.Errorf("update machine in store: %w", err) return fmt.Errorf("update machine in store: %w", err)
} }
// The machine row is missing (not created yet or lost). Recreate it.
if err = cc.store.CreateMachine(ctx, info); err != nil {
return fmt.Errorf("create machine in store: %w", err)
}
}
slog.Info("Synced machine info to cluster store.", "id", info.Id, "name", info.Name) slog.Info("Synced machine info to cluster store.", "id", info.Id, "name", info.Name)
return nil return nil
-2
View File
@@ -49,8 +49,6 @@ func (s *Service) WatchDaemonRestart(ctx context.Context) <-chan struct{} {
ch := make(chan struct{}, 1) ch := make(chan struct{}, 1)
go func() { go func() {
defer close(ch)
boff := backoff.WithContext(backoff.NewExponentialBackOff( boff := backoff.WithContext(backoff.NewExponentialBackOff(
backoff.WithInitialInterval(1*time.Second), backoff.WithInitialInterval(1*time.Second),
backoff.WithMaxInterval(30*time.Second), backoff.WithMaxInterval(30*time.Second),