refactor: cleanup Corrosion container on reset, move admin socket to runtime dir

This commit is contained in:
Pasha Sviderski
2026-05-25 21:41:35 +10:00
parent 71da32549a
commit 442b5c62bf
5 changed files with 87 additions and 22 deletions
+4 -3
View File
@@ -60,11 +60,12 @@ func (c *Config) Write(path, owner string) error {
return nil
}
func MkDataDir(dir, owner string) error {
// MkDir creates a data or runtime directory for the Corrosion service with 0700 permissions and the specified owner.
func MkDir(dir, owner string) error {
parent, _ := filepath.Split(dir)
// Use 0711 for parent directories to allow `owner` to access its nested data directory.
// Use 0711 for parent directories to allow `owner` to access its nested directory.
if err := os.MkdirAll(parent, 0o711); err != nil {
return fmt.Errorf("create directory %q: %w", parent, err)
return fmt.Errorf("create directory '%s': %w", parent, err)
}
if err := os.Mkdir(dir, 0o700); err != nil {
if !os.IsExist(err) {
+30 -1
View File
@@ -24,7 +24,10 @@ type DockerService struct {
Client *client.Client
Image string
Name string
// DataDir holds the corrosion config, schema, and db files.
DataDir string
// RunDir holds ephemeral runtime state like the admin socket.
RunDir string
User string
}
@@ -89,6 +92,23 @@ func (s *DockerService) Restart(ctx context.Context) error {
return nil
}
// Cleanup gracefully stops and removes the Corrosion container.
func (s *DockerService) Cleanup(ctx context.Context) error {
if err := s.Client.ContainerStop(ctx, s.Name, container.StopOptions{}); err != nil {
if errdefs.IsNotFound(err) {
return nil
}
return fmt.Errorf("stop container '%s': %w", s.Name, err)
}
if err := s.Client.ContainerRemove(ctx, s.Name, container.RemoveOptions{
RemoveVolumes: true,
}); err != nil {
return fmt.Errorf("remove container '%s': %w", s.Name, err)
}
slog.Debug("Corrosion container removed.", "name", s.Name)
return nil
}
func (s *DockerService) Running() bool {
c, err := s.Client.ContainerInspect(context.Background(), s.Name)
if err != nil {
@@ -115,13 +135,22 @@ func (s *DockerService) hostConfig() *container.HostConfig {
RestartPolicy: container.RestartPolicy{
Name: container.RestartPolicyUnlessStopped,
},
LogConfig: container.LogConfig{
Type: "local",
},
Mounts: []mount.Mount{
// Bind mount the data directory at the same path inside the container to simplify path handling.
// Bind mount the data and runtime directories at the same paths inside the container
// to simplify path handling.
{
Type: mount.TypeBind,
Source: s.DataDir,
Target: s.DataDir,
},
{
Type: mount.TypeBind,
Source: s.RunDir,
Target: s.RunDir,
},
},
}
}
+2
View File
@@ -16,6 +16,8 @@ type Service interface {
Start(ctx context.Context) error
Stop(ctx context.Context) error
Restart(ctx context.Context) error
// Cleanup tears down the underlying service resources.
Cleanup(ctx context.Context) error
Running() bool
}
+32 -14
View File
@@ -55,6 +55,8 @@ const (
// DefaultCaddyAdminSockPath is the default path to the Caddy admin socket for validating the generated Caddy
// reverse proxy configuration.
DefaultCaddyAdminSockPath = "/run/uncloud/caddy/admin.sock"
// DefaultCorrosionRunDir is the default runtime directory for the Corrosion service.
DefaultCorrosionRunDir = "/run/uncloud/corrosion"
)
type Config struct {
@@ -63,7 +65,9 @@ type Config struct {
MachineSockPath string
UncloudSockPath string
CorrosionDir string
CorrosionDataDir string
// CorrosionRunDir is the runtime directory for the corrosion service.
CorrosionRunDir string
CorrosionAPIListenAddr netip.AddrPort
CorrosionAPIAddr netip.AddrPort
CorrosionAdminSockPath string
@@ -105,8 +109,11 @@ func (c *Config) SetDefaults() (*Config, error) {
}
cfg.DockerClient = cli
}
if cfg.CorrosionDir == "" {
cfg.CorrosionDir = filepath.Join(cfg.DataDir, "corrosion")
if cfg.CorrosionDataDir == "" {
cfg.CorrosionDataDir = filepath.Join(cfg.DataDir, "corrosion")
}
if cfg.CorrosionRunDir == "" {
cfg.CorrosionRunDir = DefaultCorrosionRunDir
}
if !cfg.CorrosionAPIListenAddr.IsValid() {
cfg.CorrosionAPIListenAddr = netip.AddrPortFrom(
@@ -117,7 +124,7 @@ func (c *Config) SetDefaults() (*Config, error) {
netip.AddrFrom4([4]byte{127, 0, 0, 1}), corroservice.DefaultAPIPort)
}
if cfg.CorrosionAdminSockPath == "" {
cfg.CorrosionAdminSockPath = filepath.Join(cfg.CorrosionDir, "admin.sock")
cfg.CorrosionAdminSockPath = filepath.Join(cfg.CorrosionRunDir, "admin.sock")
}
if cfg.CorrosionUser == "" {
cfg.CorrosionUser = corroservice.DefaultUser
@@ -131,7 +138,8 @@ func (c *Config) SetDefaults() (*Config, error) {
Client: cfg.DockerClient,
Image: corroservice.Image,
Name: "uncloud-corrosion",
DataDir: cfg.CorrosionDir,
DataDir: cfg.CorrosionDataDir,
RunDir: cfg.CorrosionRunDir,
User: fmt.Sprintf("%d:%d", uid, gid),
}
}
@@ -359,7 +367,7 @@ func (m *Machine) Run(ctx context.Context) error {
if err := m.configureCorrosion(); err != nil {
return fmt.Errorf("configure corrosion service: %w", err)
}
slog.Info("Configured corrosion service.", "dir", m.config.CorrosionDir)
slog.Info("Configured corrosion service.", "dir", m.config.CorrosionDataDir)
if err := m.config.CorrosionService.Start(ctx); err != nil {
return fmt.Errorf("start corrosion service: %w", err)
@@ -369,7 +377,7 @@ func (m *Machine) Run(ctx context.Context) error {
// Migrate the on-disk Corrosion store to 2026.5.14 (v1.0.0 upstream) if a v0.x store.db is detected,
// before any Corrosion start attempt. The legacy systemd unit (if installed) is stopped here too
// so we own the data dir exclusively.
if err := corromigrate.MigrateIfNeeded(ctx, m.config.CorrosionDir, m.config.CorrosionUser); err != nil {
if err := corromigrate.MigrateIfNeeded(ctx, m.config.CorrosionDataDir, m.config.CorrosionUser); err != nil {
return fmt.Errorf("migrate corrosion store: %w", err)
}
}
@@ -422,7 +430,7 @@ func (m *Machine) Run(ctx context.Context) error {
if err := m.configureCorrosion(); err != nil {
return fmt.Errorf("configure corrosion service: %w", err)
}
slog.Info("Configured corrosion service.", "dir", m.config.CorrosionDir)
slog.Info("Configured corrosion service.", "dir", m.config.CorrosionDataDir)
slog.Info("Starting cluster controller.")
// Update the proxy director's local address to the machine's management IP address, allowing
@@ -493,7 +501,7 @@ func (m *Machine) Run(ctx context.Context) error {
m.store,
proxyServer,
m.config.CorrosionService,
m.config.CorrosionDir,
m.config.CorrosionDataDir,
m.dockerService,
m.networkReady,
m.clusterReady,
@@ -595,11 +603,14 @@ func listenUnixSocket(path string) (net.Listener, error) {
}
func (m *Machine) configureCorrosion() error {
if err := corroservice.MkDataDir(m.config.CorrosionDir, m.config.CorrosionUser); err != nil {
if err := corroservice.MkDir(m.config.CorrosionDataDir, m.config.CorrosionUser); err != nil {
return fmt.Errorf("create corrosion data directory: %w", err)
}
configPath := filepath.Join(m.config.CorrosionDir, "config.toml")
schemaPath := filepath.Join(m.config.CorrosionDir, "schema.sql")
if err := corroservice.MkDir(m.config.CorrosionRunDir, m.config.CorrosionUser); err != nil {
return fmt.Errorf("create corrosion runtime directory: %w", err)
}
configPath := filepath.Join(m.config.CorrosionDataDir, "config.toml")
schemaPath := filepath.Join(m.config.CorrosionDataDir, "schema.sql")
// Use a loopback address as the gossip address (required) unless the machine has joined a cluster
// and has a management IP.
@@ -618,7 +629,7 @@ func (m *Machine) configureCorrosion() error {
}
cfg := corroservice.Config{
DB: corroservice.DBConfig{
Path: filepath.Join(m.config.CorrosionDir, "store.db"),
Path: filepath.Join(m.config.CorrosionDataDir, "store.db"),
SchemaPaths: []string{schemaPath},
},
Gossip: corroservice.GossipConfig{
@@ -630,7 +641,7 @@ func (m *Machine) configureCorrosion() error {
Addr: m.config.CorrosionAPIAddr,
},
Admin: corroservice.AdminConfig{
Path: filepath.Join(m.config.CorrosionDir, "admin.sock"),
Path: m.config.CorrosionAdminSockPath,
},
}
// TODO: change file permissions to 0640 root:uncloud to emphasize the owner is the machine, not corrosion.
@@ -658,6 +669,13 @@ func (m *Machine) cleanup() error {
}
}
// Remove the corrosion service container.
cleanupCtx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
if err := m.config.CorrosionService.Cleanup(cleanupCtx); err != nil {
errs = append(errs, fmt.Errorf("cleanup corrosion service: %w", err))
}
cancel()
if err := os.RemoveAll(m.config.DataDir); err != nil {
errs = append(errs,
fmt.Errorf("remove data directory with persistent machine state '%s': %w", m.config.DataDir, err))
+15
View File
@@ -57,6 +57,7 @@ fi
log "⏳ Stopping systemd services..."
systemctl stop uncloud.service || log "uncloud.service not running or doesn't exist."
# TODO: remove uncloud-corrosion.service handling in 0.22 once pre-0.20 systemd installs are gone.
systemctl stop uncloud-corrosion.service || log "uncloud-corrosion.service not running or doesn't exist."
systemctl disable uncloud.service || log "uncloud.service already disabled or doesn't exist."
systemctl disable uncloud-corrosion.service || log "uncloud-corrosion.service already disabled or doesn't exist."
@@ -64,15 +65,29 @@ log "✓ Systemd services stopped."
log "⏳ Removing systemd service files..."
rm -fv "${INSTALL_SYSTEMD_DIR}/uncloud.service"
# TODO: remove uncloud-corrosion.service handling in 0.22 once pre-0.20 systemd installs are gone.
rm -fv "${INSTALL_SYSTEMD_DIR}/uncloud-corrosion.service"
systemctl daemon-reload
log "✓ Systemd service files removed."
log "⏳ Removing binaries..."
rm -fv "${INSTALL_BIN_DIR}/uncloudd"
# TODO: remove uncloud-corrosion binary handling in 0.22 once pre-0.20 systemd installs are gone.
rm -fv "${INSTALL_BIN_DIR}/uncloud-corrosion"
log "✓ Binaries removed."
log "⏳ Removing uncloudd-managed corrosion Docker container..."
if command -v docker &> /dev/null; then
if docker inspect uncloud-corrosion &> /dev/null; then
docker rm -fv uncloud-corrosion || log "Failed to remove uncloud-corrosion container."
log "✓ uncloud-corrosion container removed."
else
log "uncloud-corrosion container not found."
fi
else
log "Docker CLI not found, skipping uncloud-corrosion container cleanup."
fi
log "⏳ Removing data and run directories..."
rm -rfv "${UNCLOUD_DATA_DIR}"
rm -rfv "${UNCLOUD_RUN_DIR}"