mirror of
https://github.com/psviderski/uncloud.git
synced 2026-08-26 19:13:34 +00:00
522 lines
17 KiB
Go
522 lines
17 KiB
Go
package e2e
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"strings"
|
|
"testing"
|
|
"time"
|
|
|
|
"github.com/psviderski/uncloud/internal/machine/api/pb"
|
|
"github.com/psviderski/uncloud/internal/ucind"
|
|
"github.com/psviderski/uncloud/pkg/api"
|
|
"github.com/stretchr/testify/assert"
|
|
"github.com/stretchr/testify/require"
|
|
)
|
|
|
|
func TestMachineOperations(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
name := "ucind-test.machine-ops"
|
|
ctx := context.Background()
|
|
c, _ := createTestCluster(t, name, ucind.CreateClusterOptions{Machines: 3}, true)
|
|
|
|
cli, err := c.Machines[0].Connect(ctx)
|
|
require.NoError(t, err)
|
|
defer cli.Close()
|
|
|
|
// waitMachineName waits until the connected machine's store view reports the given name for the machine.
|
|
// Updates are applied on the target machine and replicated through the cluster store, so they are not
|
|
// immediately visible when read back from the connected machine.
|
|
waitMachineName := func(t *testing.T, id, name string) {
|
|
t.Helper()
|
|
require.Eventually(t, func() bool {
|
|
m, err := cli.InspectMachine(ctx, id)
|
|
return err == nil && m.Machine.Name == name
|
|
}, 15*time.Second, 100*time.Millisecond, "machine %s should be visible as %q", id, name)
|
|
}
|
|
|
|
// RenameMachine subtests.
|
|
|
|
t.Run("rename machine by name", func(t *testing.T) {
|
|
// Get initial machine state.
|
|
machines, err := cli.ListMachines(ctx, nil)
|
|
require.NoError(t, err)
|
|
require.Len(t, machines, 3)
|
|
|
|
// Select the second machine to rename.
|
|
originalMachine := machines[1]
|
|
originalName := originalMachine.Machine.Name
|
|
newName := "renamed-machine-1"
|
|
|
|
// Rename the machine.
|
|
updatedMachine, err := cli.RenameMachine(ctx, originalName, newName)
|
|
require.NoError(t, err)
|
|
assert.Equal(t, newName, updatedMachine.Name)
|
|
assert.Equal(t, originalMachine.Machine.Id, updatedMachine.Id)
|
|
|
|
// Wait for the rename to propagate to the connected machine's store view.
|
|
waitMachineName(t, originalMachine.Machine.Id, newName)
|
|
|
|
// Ensure other machines are unaffected.
|
|
machines, err = cli.ListMachines(ctx, nil)
|
|
require.NoError(t, err)
|
|
require.Len(t, machines, 3)
|
|
for _, m := range machines {
|
|
if m.Machine.Id != originalMachine.Machine.Id {
|
|
assert.NotEqual(t, newName, m.Machine.Name)
|
|
}
|
|
}
|
|
|
|
// Verify we can inspect the machine by its new name.
|
|
inspectedMachine, err := cli.InspectMachine(ctx, newName)
|
|
require.NoError(t, err)
|
|
assert.Equal(t, newName, inspectedMachine.Machine.Name)
|
|
assert.Equal(t, originalMachine.Machine.Id, inspectedMachine.Machine.Id)
|
|
|
|
// Verify the old name no longer works.
|
|
_, err = cli.InspectMachine(ctx, originalName)
|
|
assert.ErrorIs(t, err, api.ErrNotFound)
|
|
})
|
|
|
|
t.Run("rename machine by ID", func(t *testing.T) {
|
|
// Get the third machine.
|
|
machines, err := cli.ListMachines(ctx, nil)
|
|
require.NoError(t, err)
|
|
require.Len(t, machines, 3)
|
|
|
|
// Find a machine that hasn't been renamed yet.
|
|
var targetMachine *pb.MachineMember
|
|
for _, m := range machines {
|
|
if m.Machine.Name != "renamed-machine-1" {
|
|
targetMachine = m
|
|
break
|
|
}
|
|
}
|
|
require.NotNil(t, targetMachine)
|
|
|
|
originalName := targetMachine.Machine.Name
|
|
machineID := targetMachine.Machine.Id
|
|
newName := "renamed-machine-2"
|
|
|
|
// Rename using ID instead of name.
|
|
updatedMachine, err := cli.RenameMachine(ctx, machineID, newName)
|
|
require.NoError(t, err)
|
|
assert.Equal(t, newName, updatedMachine.Name)
|
|
assert.Equal(t, machineID, updatedMachine.Id)
|
|
|
|
// Wait for the rename to propagate to the connected machine's store view.
|
|
waitMachineName(t, machineID, newName)
|
|
|
|
// Verify the rename was successful.
|
|
inspectedMachine, err := cli.InspectMachine(ctx, newName)
|
|
require.NoError(t, err)
|
|
assert.Equal(t, newName, inspectedMachine.Machine.Name)
|
|
assert.Equal(t, machineID, inspectedMachine.Machine.Id)
|
|
|
|
// Verify the old name no longer works.
|
|
_, err = cli.InspectMachine(ctx, originalName)
|
|
assert.ErrorIs(t, err, api.ErrNotFound)
|
|
})
|
|
|
|
t.Run("rename non-existent machine", func(t *testing.T) {
|
|
// Try to rename a machine that doesn't exist.
|
|
_, err := cli.RenameMachine(ctx, "non-existent-machine", "new-name")
|
|
assert.ErrorContains(t, err, "machine not found")
|
|
})
|
|
|
|
t.Run("rename to existing name", func(t *testing.T) {
|
|
// Get current machines.
|
|
machines, err := cli.ListMachines(ctx, nil)
|
|
require.NoError(t, err)
|
|
require.Len(t, machines, 3)
|
|
|
|
// Rename the connected machine to another machine's name. The uniqueness check runs on the connected
|
|
// machine, which already sees the other machine's name, so the collision is detected deterministically
|
|
// (a remote target's store view may lag behind the connected machine's).
|
|
connectedID := c.Machines[0].ID
|
|
var otherName string
|
|
for _, m := range machines {
|
|
if m.Machine.Id != connectedID {
|
|
otherName = m.Machine.Name
|
|
break
|
|
}
|
|
}
|
|
require.NotEmpty(t, otherName)
|
|
|
|
// This should fail because the name is already taken.
|
|
_, err = cli.RenameMachine(ctx, connectedID, otherName)
|
|
assert.Error(t, err)
|
|
})
|
|
|
|
t.Run("rename with empty name", func(t *testing.T) {
|
|
// Get a machine to rename.
|
|
machines, err := cli.ListMachines(ctx, nil)
|
|
require.NoError(t, err)
|
|
require.Len(t, machines, 3)
|
|
|
|
machineName := machines[0].Machine.Name
|
|
|
|
// Try to rename with empty string.
|
|
_, err = cli.RenameMachine(ctx, machineName, "")
|
|
assert.Error(t, err)
|
|
})
|
|
|
|
t.Run("service continuity after rename", func(t *testing.T) {
|
|
// Deploy a service on a specific machine.
|
|
machines, err := cli.ListMachines(ctx, nil)
|
|
require.NoError(t, err)
|
|
|
|
// Find a machine that hasn't been renamed to test with.
|
|
var targetMachine *pb.MachineMember
|
|
for _, m := range machines {
|
|
if m.Machine.Name != "renamed-machine-1" && m.Machine.Name != "renamed-machine-2" {
|
|
targetMachine = m
|
|
break
|
|
}
|
|
}
|
|
require.NotNil(t, targetMachine)
|
|
|
|
originalMachineName := targetMachine.Machine.Name
|
|
serviceName := "test-service-rename-continuity"
|
|
|
|
// Create a service on the specific machine.
|
|
spec := api.ServiceSpec{
|
|
Name: serviceName,
|
|
Mode: api.ServiceModeGlobal,
|
|
Container: api.ContainerSpec{
|
|
Image: "portainer/pause:latest",
|
|
},
|
|
Placement: api.Placement{
|
|
Machines: []string{originalMachineName},
|
|
},
|
|
}
|
|
|
|
_, err = cli.RunService(ctx, spec)
|
|
require.NoError(t, err)
|
|
|
|
t.Cleanup(func() {
|
|
err := cli.RemoveService(ctx, serviceName)
|
|
if err != nil && !errors.Is(err, api.ErrNotFound) {
|
|
assert.NoError(t, err)
|
|
}
|
|
})
|
|
|
|
// Verify service is running on the machine.
|
|
svc, err := cli.InspectService(ctx, serviceName)
|
|
require.NoError(t, err)
|
|
assert.Len(t, svc.Containers, 1)
|
|
assert.Equal(t, targetMachine.Machine.Id, svc.Containers[0].MachineID)
|
|
|
|
// Rename the machine.
|
|
newMachineName := "renamed-for-service-test"
|
|
_, err = cli.RenameMachine(ctx, originalMachineName, newMachineName)
|
|
require.NoError(t, err)
|
|
waitMachineName(t, targetMachine.Machine.Id, newMachineName)
|
|
|
|
// Verify service is still running on the renamed machine.
|
|
svc, err = cli.InspectService(ctx, serviceName)
|
|
require.NoError(t, err)
|
|
assert.Len(t, svc.Containers, 1)
|
|
assert.Equal(t, targetMachine.Machine.Id, svc.Containers[0].MachineID)
|
|
|
|
// The service spec's placement still references the old name,
|
|
// but the service should continue to run on the same machine id.
|
|
})
|
|
|
|
// UpdateMachine subtests.
|
|
|
|
t.Run("update machine name", func(t *testing.T) {
|
|
// Get initial machine state.
|
|
machines, err := cli.ListMachines(ctx, nil)
|
|
require.NoError(t, err)
|
|
require.Len(t, machines, 3)
|
|
|
|
// Select a machine to update.
|
|
targetMachine := machines[1]
|
|
originalName := targetMachine.Machine.Name
|
|
newName := "updated-machine-name"
|
|
|
|
// Update the machine name using UpdateMachine directly.
|
|
req := &pb.UpdateMachineRequest{
|
|
Name: &newName,
|
|
}
|
|
updatedMachine, err := cli.UpdateMachine(ctx, targetMachine.Machine.Id, req)
|
|
require.NoError(t, err)
|
|
assert.Equal(t, newName, updatedMachine.Name)
|
|
assert.Equal(t, targetMachine.Machine.Id, updatedMachine.Id)
|
|
|
|
// Wait for the change to propagate to the connected machine's store view.
|
|
waitMachineName(t, targetMachine.Machine.Id, newName)
|
|
|
|
// Verify old name no longer works.
|
|
_, err = cli.InspectMachine(ctx, originalName)
|
|
assert.ErrorIs(t, err, api.ErrNotFound)
|
|
})
|
|
|
|
t.Run("update machine public IP", func(t *testing.T) {
|
|
// Get a machine to update.
|
|
machines, err := cli.ListMachines(ctx, nil)
|
|
require.NoError(t, err)
|
|
|
|
// Find a machine that hasn't been renamed.
|
|
var targetMachine *pb.MachineMember
|
|
for _, m := range machines {
|
|
if m.Machine.Name != "updated-machine-name" {
|
|
targetMachine = m
|
|
break
|
|
}
|
|
}
|
|
require.NotNil(t, targetMachine)
|
|
|
|
// Create a new public IP (must be a valid public IP address).
|
|
newPublicIP := &pb.IP{
|
|
Ip: []byte{8, 8, 8, 8},
|
|
}
|
|
|
|
// Update the public IP.
|
|
req := &pb.UpdateMachineRequest{
|
|
PublicIp: newPublicIP,
|
|
}
|
|
updatedMachine, err := cli.UpdateMachine(ctx, targetMachine.Machine.Id, req)
|
|
require.NoError(t, err)
|
|
assert.Equal(t, newPublicIP.Ip, updatedMachine.PublicIp.Ip)
|
|
|
|
// Verify the change propagated to the connected machine's store view.
|
|
require.Eventually(t, func() bool {
|
|
inspected, err := cli.InspectMachine(ctx, targetMachine.Machine.Id)
|
|
return err == nil && inspected.Machine.PublicIp != nil &&
|
|
string(inspected.Machine.PublicIp.Ip) == string(newPublicIP.Ip)
|
|
}, 15*time.Second, 100*time.Millisecond, "public IP should be updated")
|
|
})
|
|
|
|
t.Run("remove machine public IP", func(t *testing.T) {
|
|
// Get machines.
|
|
machines, err := cli.ListMachines(ctx, nil)
|
|
require.NoError(t, err)
|
|
require.True(t, len(machines) > 0, "Need at least one machine")
|
|
|
|
// First, set a public IP on a machine.
|
|
targetMachine := machines[0]
|
|
setIPReq := &pb.UpdateMachineRequest{
|
|
PublicIp: &pb.IP{Ip: []byte{192, 0, 2, 1}}, // TEST-NET-1 address.
|
|
}
|
|
updatedMachine, err := cli.UpdateMachine(ctx, targetMachine.Machine.Id, setIPReq)
|
|
require.NoError(t, err)
|
|
require.NotNil(t, updatedMachine.PublicIp)
|
|
|
|
// Now test removing the public IP.
|
|
|
|
// Remove the public IP by setting it to empty.
|
|
emptyIP := &pb.IP{}
|
|
req := &pb.UpdateMachineRequest{
|
|
PublicIp: emptyIP,
|
|
}
|
|
removedIPMachine, err := cli.UpdateMachine(ctx, updatedMachine.Id, req)
|
|
require.NoError(t, err)
|
|
assert.Nil(t, removedIPMachine.PublicIp)
|
|
|
|
// Verify the removal propagated to the connected machine's store view.
|
|
require.Eventually(t, func() bool {
|
|
inspected, err := cli.InspectMachine(ctx, updatedMachine.Id)
|
|
return err == nil && inspected.Machine.PublicIp == nil
|
|
}, 15*time.Second, 100*time.Millisecond, "public IP should be removed")
|
|
})
|
|
|
|
t.Run("update machine endpoints", func(t *testing.T) {
|
|
// Get a machine to update.
|
|
machines, err := cli.ListMachines(ctx, nil)
|
|
require.NoError(t, err)
|
|
targetMachine := machines[0]
|
|
|
|
newEndpoints := []*pb.IPPort{
|
|
{
|
|
Ip: &pb.IP{Ip: []byte{10, 0, 0, 10}},
|
|
Port: 8080,
|
|
},
|
|
{
|
|
Ip: &pb.IP{Ip: []byte{10, 0, 0, 10}},
|
|
Port: 8443,
|
|
},
|
|
}
|
|
|
|
req := &pb.UpdateMachineRequest{
|
|
Endpoints: newEndpoints,
|
|
}
|
|
updatedMachine, err := cli.UpdateMachine(ctx, targetMachine.Machine.Id, req)
|
|
require.NoError(t, err)
|
|
assert.Equal(t, len(newEndpoints), len(updatedMachine.Network.Endpoints))
|
|
|
|
// Verify endpoints were updated.
|
|
for i, endpoint := range updatedMachine.Network.Endpoints {
|
|
assert.Equal(t, newEndpoints[i].Ip.Ip, endpoint.Ip.Ip)
|
|
assert.Equal(t, newEndpoints[i].Port, endpoint.Port)
|
|
}
|
|
|
|
// Verify other network fields remain unchanged.
|
|
assert.Equal(t, targetMachine.Machine.Network.Subnet.Ip.Ip, updatedMachine.Network.Subnet.Ip.Ip)
|
|
assert.Equal(t, targetMachine.Machine.Network.Subnet.Bits, updatedMachine.Network.Subnet.Bits)
|
|
assert.Equal(t, targetMachine.Machine.Network.ManagementIp.Ip, updatedMachine.Network.ManagementIp.Ip)
|
|
assert.Equal(t, targetMachine.Machine.Network.PublicKey, updatedMachine.Network.PublicKey)
|
|
})
|
|
|
|
t.Run("update multiple fields simultaneously", func(t *testing.T) {
|
|
// Get a machine to update.
|
|
machines, err := cli.ListMachines(ctx, nil)
|
|
require.NoError(t, err)
|
|
|
|
var targetMachine *pb.MachineMember
|
|
for _, m := range machines {
|
|
if m.Machine.Name != "updated-machine-name" {
|
|
targetMachine = m
|
|
break
|
|
}
|
|
}
|
|
require.NotNil(t, targetMachine)
|
|
|
|
// Update both name and public IP.
|
|
newName := "multi-update-machine"
|
|
newPublicIP := &pb.IP{
|
|
Ip: []byte{1, 1, 1, 1},
|
|
}
|
|
|
|
req := &pb.UpdateMachineRequest{
|
|
Name: &newName,
|
|
PublicIp: newPublicIP,
|
|
}
|
|
updatedMachine, err := cli.UpdateMachine(ctx, targetMachine.Machine.Id, req)
|
|
require.NoError(t, err)
|
|
assert.Equal(t, newName, updatedMachine.Name)
|
|
assert.Equal(t, newPublicIP.Ip, updatedMachine.PublicIp.Ip)
|
|
|
|
// Verify both changes propagated to the connected machine's store view.
|
|
require.Eventually(t, func() bool {
|
|
inspected, err := cli.InspectMachine(ctx, updatedMachine.Id)
|
|
if err != nil || inspected.Machine.PublicIp == nil {
|
|
return false
|
|
}
|
|
return inspected.Machine.Name == newName &&
|
|
string(inspected.Machine.PublicIp.Ip) == string(newPublicIP.Ip)
|
|
}, 15*time.Second, 100*time.Millisecond, "name and public IP should be updated")
|
|
})
|
|
|
|
t.Run("update non-existent machine", func(t *testing.T) {
|
|
// Try to update properties on a machine that doesn't exist.
|
|
nonExistentName := "should-be-updated"
|
|
req := &pb.UpdateMachineRequest{
|
|
Name: &nonExistentName,
|
|
}
|
|
_, err := cli.UpdateMachine(ctx, "non-existent-machine-id", req)
|
|
assert.ErrorContains(t, err, "machine not found")
|
|
})
|
|
|
|
t.Run("update to duplicate name", func(t *testing.T) {
|
|
machines, err := cli.ListMachines(ctx, nil)
|
|
require.NoError(t, err)
|
|
require.Len(t, machines, 3)
|
|
|
|
// Update the connected machine with another machine's name. The uniqueness check runs on the connected
|
|
// machine, which already sees the other machine's name, so the collision is detected deterministically
|
|
// (a remote target's store view may lag behind the connected machine's).
|
|
connectedID := c.Machines[0].ID
|
|
var otherName string
|
|
for _, m := range machines {
|
|
if m.Machine.Id != connectedID {
|
|
otherName = m.Machine.Name
|
|
break
|
|
}
|
|
}
|
|
require.NotEmpty(t, otherName)
|
|
|
|
req := &pb.UpdateMachineRequest{
|
|
Name: &otherName,
|
|
}
|
|
_, err = cli.UpdateMachine(ctx, connectedID, req)
|
|
assert.Error(t, err)
|
|
})
|
|
|
|
t.Run("update with empty request", func(t *testing.T) {
|
|
// Get a machine.
|
|
machines, err := cli.ListMachines(ctx, nil)
|
|
require.NoError(t, err)
|
|
|
|
targetMachine := machines[0]
|
|
|
|
// Update with no fields set (should be a no-op).
|
|
req := &pb.UpdateMachineRequest{}
|
|
updatedMachine, err := cli.UpdateMachine(ctx, targetMachine.Machine.Id, req)
|
|
require.NoError(t, err)
|
|
|
|
// Machine should remain unchanged.
|
|
assert.Equal(t, targetMachine.Machine.Name, updatedMachine.Name)
|
|
if targetMachine.Machine.PublicIp != nil && updatedMachine.PublicIp != nil {
|
|
assert.Equal(t, targetMachine.Machine.PublicIp.Ip, updatedMachine.PublicIp.Ip)
|
|
}
|
|
assert.Equal(t, len(targetMachine.Machine.Network.Endpoints), len(updatedMachine.Network.Endpoints))
|
|
})
|
|
|
|
t.Run("remove machine clears container records from cluster store", func(t *testing.T) {
|
|
// Deploy a global service with an HTTP ingress port so the auto-generated Caddyfile lists
|
|
// each container's IP as an upstream.
|
|
serviceName := "test-machine-rm-cleanup"
|
|
spec := api.ServiceSpec{
|
|
Name: serviceName,
|
|
Mode: api.ServiceModeGlobal,
|
|
Container: api.ContainerSpec{
|
|
Image: "portainer/pause:latest",
|
|
},
|
|
Ports: []api.PortSpec{
|
|
{
|
|
Hostname: "test-machine-rm-cleanup.example.com",
|
|
ContainerPort: 8000,
|
|
Protocol: api.ProtocolHTTP,
|
|
Mode: api.PortModeIngress,
|
|
},
|
|
},
|
|
}
|
|
|
|
t.Cleanup(func() {
|
|
err := cli.RemoveService(ctx, serviceName)
|
|
if err != nil && !errors.Is(err, api.ErrNotFound) {
|
|
assert.NoError(t, err)
|
|
}
|
|
})
|
|
_, err = cli.NewDeployment(spec, nil).Run(ctx)
|
|
require.NoError(t, err)
|
|
|
|
svc, err := cli.InspectService(ctx, serviceName)
|
|
require.NoError(t, err)
|
|
|
|
containers := serviceContainersByMachine(svc)
|
|
|
|
removedMachine := c.Machines[2]
|
|
removedIP := containers[removedMachine.ID][0].Container.UncloudNetworkIP().String()
|
|
keptIP := containers[c.Machines[0].ID][0].Container.UncloudNetworkIP().String()
|
|
|
|
// The Caddyfile contains both upstream IPs before the machine removal.
|
|
require.Eventually(t, func() bool {
|
|
cfg, err := cli.Caddy.GetConfig(ctx, nil)
|
|
if err != nil {
|
|
return false
|
|
}
|
|
return strings.Contains(cfg.Caddyfile, removedIP) && strings.Contains(cfg.Caddyfile, keptIP)
|
|
}, 15*time.Second, 100*time.Millisecond,
|
|
"expected Caddyfile to include both the to-be-removed and kept container IPs")
|
|
|
|
_, err = cli.RemoveMachine(ctx, &pb.RemoveMachineRequest{Id: removedMachine.ID})
|
|
require.NoError(t, err)
|
|
|
|
// After removal, the removed machine's container record should be gone from the cluster store,
|
|
// so the Caddy controller regenerates a Caddyfile without that upstream while keeping the
|
|
// upstreams for the still-running containers.
|
|
require.Eventually(t, func() bool {
|
|
cfg, err := cli.Caddy.GetConfig(ctx, nil)
|
|
if err != nil {
|
|
return false
|
|
}
|
|
return !strings.Contains(cfg.Caddyfile, removedIP) && strings.Contains(cfg.Caddyfile, keptIP)
|
|
}, 15*time.Second, 100*time.Millisecond,
|
|
"expected Caddyfile to drop the removed machine's container IP and keep the rest")
|
|
})
|
|
}
|