From e198fa5891a3abbbc141daf3b74e31957606850a Mon Sep 17 00:00:00 2001 From: Steven Miller Date: Tue, 2 Dec 2025 12:57:32 -0500 Subject: [PATCH 1/6] Volume readonly multi-attach --- cmd/api/api/volumes.go | 18 ++- lib/instances/create.go | 12 +- lib/instances/delete.go | 2 +- lib/instances/manager_test.go | 10 +- lib/instances/volumes_test.go | 221 +++++++++++++++++++++++++ lib/oapi/oapi.go | 136 ++++++++-------- lib/volumes/README.md | 12 +- lib/volumes/manager.go | 95 ++++++++--- lib/volumes/manager_test.go | 294 ++++++++++++++++++++++++++++++++++ lib/volumes/storage.go | 19 ++- lib/volumes/types.go | 21 +-- openapi.yaml | 32 ++-- 12 files changed, 738 insertions(+), 134 deletions(-) create mode 100644 lib/instances/volumes_test.go create mode 100644 lib/volumes/manager_test.go diff --git a/cmd/api/api/volumes.go b/cmd/api/api/volumes.go index ff931f15..63d9f134 100644 --- a/cmd/api/api/volumes.go +++ b/cmd/api/api/volumes.go @@ -140,12 +140,20 @@ func volumeToOAPI(vol volumes.Volume) oapi.Volume { SizeGb: vol.SizeGb, CreatedAt: vol.CreatedAt, } - if vol.AttachedTo != nil { - oapiVol.AttachedTo = vol.AttachedTo - } - if vol.MountPath != nil { - oapiVol.MountPath = vol.MountPath + + // Convert attachments + if len(vol.Attachments) > 0 { + attachments := make([]oapi.VolumeAttachmentInfo, len(vol.Attachments)) + for i, att := range vol.Attachments { + attachments[i] = oapi.VolumeAttachmentInfo{ + InstanceId: att.InstanceID, + MountPath: att.MountPath, + Readonly: att.Readonly, + } + } + oapiVol.Attachments = &attachments } + return oapiVol } diff --git a/lib/instances/create.go b/lib/instances/create.go index 4811cb50..ffe55df6 100644 --- a/lib/instances/create.go +++ b/lib/instances/create.go @@ -263,18 +263,14 @@ func (m *manager) createInstance( if len(req.Volumes) > 0 { log.DebugContext(ctx, "validating volumes", "id", id, "count", len(req.Volumes)) for _, volAttach := range req.Volumes { - // Check volume exists and is not attached - vol, err := m.volumeManager.GetVolume(ctx, volAttach.VolumeID) + // Check volume exists + _, err := m.volumeManager.GetVolume(ctx, volAttach.VolumeID) if err != nil { log.ErrorContext(ctx, "volume not found", "id", id, "volume_id", volAttach.VolumeID, "error", err) return nil, fmt.Errorf("volume %s: %w", volAttach.VolumeID, err) } - if vol.AttachedTo != nil { - log.ErrorContext(ctx, "volume already attached", "id", id, "volume_id", volAttach.VolumeID, "attached_to", *vol.AttachedTo) - return nil, fmt.Errorf("volume %s is already attached to instance %s", volAttach.VolumeID, *vol.AttachedTo) - } - // Mark volume as attached + // Mark volume as attached (AttachVolume handles multi-attach validation) if err := m.volumeManager.AttachVolume(ctx, volAttach.VolumeID, volumes.AttachVolumeRequest{ InstanceID: id, MountPath: volAttach.MountPath, @@ -287,7 +283,7 @@ func (m *manager) createInstance( // Add volume cleanup to stack volumeID := volAttach.VolumeID // capture for closure cu.Add(func() { - m.volumeManager.DetachVolume(ctx, volumeID) + m.volumeManager.DetachVolume(ctx, volumeID, id) }) } // Store volume attachments in metadata diff --git a/lib/instances/delete.go b/lib/instances/delete.go index 32630aef..62ed44a1 100644 --- a/lib/instances/delete.go +++ b/lib/instances/delete.go @@ -62,7 +62,7 @@ func (m *manager) deleteInstance( if len(inst.Volumes) > 0 { log.DebugContext(ctx, "detaching volumes", "id", id, "count", len(inst.Volumes)) for _, volAttach := range inst.Volumes { - if err := m.volumeManager.DetachVolume(ctx, volAttach.VolumeID); err != nil { + if err := m.volumeManager.DetachVolume(ctx, volAttach.VolumeID, id); err != nil { // Log error but continue with cleanup log.WarnContext(ctx, "failed to detach volume, continuing with cleanup", "id", id, "volume_id", volAttach.VolumeID, "error", err) } diff --git a/lib/instances/manager_test.go b/lib/instances/manager_test.go index dde54d5e..4b533e0a 100644 --- a/lib/instances/manager_test.go +++ b/lib/instances/manager_test.go @@ -222,7 +222,7 @@ func TestCreateAndDeleteInstance(t *testing.T) { // Verify volume file exists and is not attached assert.FileExists(t, p.VolumeData(vol.Id)) - assert.Nil(t, vol.AttachedTo, "Volume should not be attached yet") + assert.Empty(t, vol.Attachments, "Volume should not be attached yet") // Create instance with real nginx image and attached volume req := CreateInstanceRequest{ @@ -267,9 +267,9 @@ func TestCreateAndDeleteInstance(t *testing.T) { // Verify volume shows as attached vol, err = volumeManager.GetVolume(ctx, vol.Id) require.NoError(t, err) - require.NotNil(t, vol.AttachedTo, "Volume should be attached") - assert.Equal(t, inst.Id, *vol.AttachedTo) - assert.Equal(t, "/mnt/data", *vol.MountPath) + require.Len(t, vol.Attachments, 1, "Volume should be attached") + assert.Equal(t, inst.Id, vol.Attachments[0].InstanceID) + assert.Equal(t, "/mnt/data", vol.Attachments[0].MountPath) // Verify directories exist assert.DirExists(t, p.InstanceDir(inst.Id)) @@ -444,7 +444,7 @@ func TestCreateAndDeleteInstance(t *testing.T) { // Verify volume is detached but still exists vol, err = volumeManager.GetVolume(ctx, vol.Id) require.NoError(t, err) - assert.Nil(t, vol.AttachedTo, "Volume should be detached after instance deletion") + assert.Empty(t, vol.Attachments, "Volume should be detached after instance deletion") assert.FileExists(t, p.VolumeData(vol.Id), "Volume file should still exist") // Delete volume diff --git a/lib/instances/volumes_test.go b/lib/instances/volumes_test.go new file mode 100644 index 00000000..fa359f20 --- /dev/null +++ b/lib/instances/volumes_test.go @@ -0,0 +1,221 @@ +package instances + +import ( + "context" + "os" + "strings" + "testing" + "time" + + "github.com/onkernel/hypeman/lib/images" + "github.com/onkernel/hypeman/lib/paths" + "github.com/onkernel/hypeman/lib/system" + "github.com/onkernel/hypeman/lib/volumes" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// waitForExecAgent polls until exec-agent is ready +func waitForExecAgent(ctx context.Context, mgr *manager, instanceID string, timeout time.Duration) error { + deadline := time.Now().Add(timeout) + for time.Now().Before(deadline) { + logs, err := collectLogs(ctx, mgr, instanceID, 100) + if err == nil && strings.Contains(logs, "[exec-agent] listening on vsock port 2222") { + return nil + } + time.Sleep(500 * time.Millisecond) + } + return context.DeadlineExceeded +} + +// execWithRetry runs a command with retries until exec-agent is ready +func execWithRetry(ctx context.Context, vsockSocket string, command []string) (string, int, error) { + var output string + var code int + var err error + + for i := 0; i < 10; i++ { + output, code, err = execCommand(ctx, vsockSocket, command...) + if err == nil { + return output, code, nil + } + time.Sleep(500 * time.Millisecond) + } + return output, code, err +} + +// TestVolumeMultiAttachReadOnly tests that a volume can be: +// 1. Attached read-write to one instance, written to +// 2. Detached (by deleting the instance) +// 3. Attached read-only to multiple instances simultaneously +// 4. Data persists and is readable from all instances +func TestVolumeMultiAttachReadOnly(t *testing.T) { + // Require KVM + if _, err := os.Stat("/dev/kvm"); os.IsNotExist(err) { + t.Fatal("/dev/kvm not available - ensure KVM is enabled and user is in 'kvm' group") + } + + if testing.Short() { + t.Skip("Skipping integration test in short mode") + } + + manager, tmpDir := setupTestManager(t) + ctx := context.Background() + p := paths.New(tmpDir) + + // Setup: prepare image and system files + imageManager, err := images.NewManager(p, 1) + require.NoError(t, err) + + t.Log("Pulling alpine image...") + _, err = imageManager.CreateImage(ctx, images.CreateImageRequest{ + Name: "docker.io/library/alpine:latest", + }) + require.NoError(t, err) + + for i := 0; i < 60; i++ { + img, err := imageManager.GetImage(ctx, "docker.io/library/alpine:latest") + if err == nil && img.Status == images.StatusReady { + break + } + time.Sleep(1 * time.Second) + } + t.Log("Image ready") + + systemManager := system.NewManager(p) + t.Log("Ensuring system files...") + err = systemManager.EnsureSystemFiles(ctx) + require.NoError(t, err) + t.Log("System files ready") + + // Create volume + volumeManager := volumes.NewManager(p, 0) + t.Log("Creating volume...") + vol, err := volumeManager.CreateVolume(ctx, volumes.CreateVolumeRequest{ + Name: "shared-data", + SizeGb: 1, + }) + require.NoError(t, err) + t.Logf("Volume created: %s", vol.Id) + + // Phase 1: Create instance with volume attached read-write + t.Log("Phase 1: Creating writer instance with read-write volume...") + writerInst, err := manager.CreateInstance(ctx, CreateInstanceRequest{ + Name: "writer", + Image: "docker.io/library/alpine:latest", + Size: 512 * 1024 * 1024, + HotplugSize: 512 * 1024 * 1024, + OverlaySize: 1024 * 1024 * 1024, + Vcpus: 1, + NetworkEnabled: false, + Volumes: []VolumeAttachment{ + {VolumeID: vol.Id, MountPath: "/data", Readonly: false}, + }, + }) + require.NoError(t, err) + t.Logf("Writer instance created: %s", writerInst.Id) + + // Wait for exec-agent + err = waitForExecAgent(ctx, manager, writerInst.Id, 30*time.Second) + require.NoError(t, err, "exec-agent should be ready") + + // Write test file + t.Log("Writing test file to volume...") + _, code, err := execWithRetry(ctx, writerInst.VsockSocket, []string{ + "/bin/sh", "-c", "echo 'Hello from writer' > /data/test.txt", + }) + require.NoError(t, err) + require.Equal(t, 0, code, "Write command should succeed") + + // Verify file was written + output, code, err := execWithRetry(ctx, writerInst.VsockSocket, []string{"cat", "/data/test.txt"}) + require.NoError(t, err) + require.Equal(t, 0, code) + assert.Contains(t, output, "Hello from writer") + t.Log("Test file written successfully") + + // Delete writer instance (detaches volume) + t.Log("Deleting writer instance...") + err = manager.DeleteInstance(ctx, writerInst.Id) + require.NoError(t, err) + + // Verify volume is detached + vol, err = volumeManager.GetVolume(ctx, vol.Id) + require.NoError(t, err) + assert.Empty(t, vol.Attachments, "Volume should be detached") + + // Phase 2: Create two instances with the volume attached read-only + t.Log("Phase 2: Creating two reader instances with read-only volume...") + + reader1, err := manager.CreateInstance(ctx, CreateInstanceRequest{ + Name: "reader-1", + Image: "docker.io/library/alpine:latest", + Size: 512 * 1024 * 1024, + HotplugSize: 512 * 1024 * 1024, + OverlaySize: 1024 * 1024 * 1024, + Vcpus: 1, + NetworkEnabled: false, + Volumes: []VolumeAttachment{ + {VolumeID: vol.Id, MountPath: "/data", Readonly: true}, + }, + }) + require.NoError(t, err) + t.Logf("Reader 1 created: %s", reader1.Id) + + reader2, err := manager.CreateInstance(ctx, CreateInstanceRequest{ + Name: "reader-2", + Image: "docker.io/library/alpine:latest", + Size: 512 * 1024 * 1024, + HotplugSize: 512 * 1024 * 1024, + OverlaySize: 1024 * 1024 * 1024, + Vcpus: 1, + NetworkEnabled: false, + Volumes: []VolumeAttachment{ + {VolumeID: vol.Id, MountPath: "/data", Readonly: true}, + }, + }) + require.NoError(t, err) + t.Logf("Reader 2 created: %s", reader2.Id) + + // Verify volume has two attachments + vol, err = volumeManager.GetVolume(ctx, vol.Id) + require.NoError(t, err) + assert.Len(t, vol.Attachments, 2, "Volume should have 2 attachments") + + // Wait for exec-agent on both readers + err = waitForExecAgent(ctx, manager, reader1.Id, 30*time.Second) + require.NoError(t, err, "reader-1 exec-agent should be ready") + + err = waitForExecAgent(ctx, manager, reader2.Id, 30*time.Second) + require.NoError(t, err, "reader-2 exec-agent should be ready") + + // Verify data is readable from reader-1 + t.Log("Verifying data from reader-1...") + output, code, err = execWithRetry(ctx, reader1.VsockSocket, []string{"cat", "/data/test.txt"}) + require.NoError(t, err) + require.Equal(t, 0, code) + assert.Contains(t, output, "Hello from writer", "Reader 1 should see the file") + + // Verify data is readable from reader-2 + t.Log("Verifying data from reader-2...") + output, code, err = execWithRetry(ctx, reader2.VsockSocket, []string{"cat", "/data/test.txt"}) + require.NoError(t, err) + require.Equal(t, 0, code) + assert.Contains(t, output, "Hello from writer", "Reader 2 should see the file") + + // Verify write fails on read-only mount (reader-1) + t.Log("Verifying read-only enforcement...") + _, code, err = execWithRetry(ctx, reader1.VsockSocket, []string{ + "/bin/sh", "-c", "echo 'illegal write' > /data/illegal.txt", + }) + require.NoError(t, err, "Exec should succeed even if command fails") + assert.NotEqual(t, 0, code, "Write to read-only volume should fail with non-zero exit") + + t.Log("Multi-attach read-only test passed!") + + // Cleanup + t.Log("Cleaning up...") + manager.DeleteInstance(ctx, reader1.Id) + manager.DeleteInstance(ctx, reader2.Id) + volumeManager.DeleteVolume(ctx, vol.Id) +} diff --git a/lib/oapi/oapi.go b/lib/oapi/oapi.go index e5e3eee5..6ad0352a 100644 --- a/lib/oapi/oapi.go +++ b/lib/oapi/oapi.go @@ -257,8 +257,8 @@ type InstanceState string // Volume defines model for Volume. type Volume struct { - // AttachedTo Instance ID if attached - AttachedTo *string `json:"attached_to"` + // Attachments List of current attachments (empty if not attached) + Attachments *[]VolumeAttachmentInfo `json:"attachments,omitempty"` // CreatedAt Creation timestamp (RFC3339) CreatedAt time.Time `json:"created_at"` @@ -266,9 +266,6 @@ type Volume struct { // Id Unique identifier Id string `json:"id"` - // MountPath Mount path if attached - MountPath *string `json:"mount_path"` - // Name Volume name Name string `json:"name"` @@ -288,6 +285,18 @@ type VolumeAttachment struct { VolumeId string `json:"volume_id"` } +// VolumeAttachmentInfo defines model for VolumeAttachmentInfo. +type VolumeAttachmentInfo struct { + // InstanceId ID of the instance this volume is attached to + InstanceId string `json:"instance_id"` + + // MountPath Mount path in the guest + MountPath string `json:"mount_path"` + + // Readonly Whether the attachment is read-only + Readonly bool `json:"readonly"` +} + // GetInstanceLogsParams defines parameters for GetInstanceLogs. type GetInstanceLogsParams struct { // Tail Number of lines to return from end @@ -4972,64 +4981,65 @@ func (sh *strictHandler) GetVolume(w http.ResponseWriter, r *http.Request, id st // Base64 encoded, gzipped, json marshaled Swagger object var swaggerSpec = []string{ - "H4sIAAAAAAAC/+xcC28Tu5f/Kpb3f6V2NXm2ZWn+Wq1KeVWiULUXrrSULc74JPHFYw+2JyWgfveVHzOZ", - "yUwehTaXCiQkknn4vH7naaffcCyTVAoQRuPBN6zjCSTEfTwyhsSTd5JnCZzD5wy0sZdTJVNQhoF7KJGZ", - "MFcpMRP7jYKOFUsNkwIP8BkxE3Q9AQVo6lZBeiIzTtEQkHsPKI4wfCFJygEPcCcRpkOJITjCZpbaS9oo", - "Jsb4JsIKCJWCzzyZEcm4wYMR4RqiBbKndmlENLKvtNw7xXpDKTkQgW/cip8zpoDiwfuyGB+Kh+Xwb4iN", - "JX6sgBg4Sch4uSYESaCugzfHJ4jZ95CCESgQMaAdaI/bEaIy/gSqzWSHs6EiatYRYya+DDgxoM1uRTWr", - "n63ra0E8x9sKwYQ2RMTLZQMxtf8RSpmVi/Czyu2asao6eCamTEmRgDBoShQjQw66LN43/PrN02dXz16/", - "wwNLmWaxezXCZ2/O/8QDvNftdu26Nf4n0qQ8G19p9hUqyMB7L57gRUaOCv5RAolUMzSSCoU10M4kS4ho", - "WdRYDu29hBjE2SdAl3a9SxyhS9x7cYmrxuk7UjUlOLNvhIg1piY8ZQKW2jpaAr2XVXHsQ2iHy2tQMdGA", - "OBgDSkeIsjEzOkJEUESJnoBG1mn+jWIihDRIG6IMkgqBoOiamQki7rmqEpJZ61qqT1wS2urhCCfkyysQ", - "YxsXHu1FOCWWmmXr/96T1tdu6/DDTvjQ+vCf+aXd//lXo3xg7Np1EV/7GyiWYsTGmSL2ujOqmQBiAdY4", - "qsHZaoRWAGNUVoskf03ATEAhIxFxwbBY0l6yJMLrKOewpBG/YEPcqYFYTkFxMmsAca/bgOK/FDPOouE9", - "RJn+hOzLayBsV/MYPujWQdxtRnEDUw08PbGICj61CScFI73+afjY39SvpnGa6QpL/UV2XmfJEBSSIzRl", - "ymSEo+Ozt5WQ0y8WZsLAGJRb2WUpXceZT4K6BIRg/wIPxKDYxlKLP8MSizlmIHFr/UvBCA/wf3TmqbYT", - "8mzHr+xTrQ2QpShHlCKz5lCeB5flIX1N2ma0ITClITbGmTYyQYyCMGzEQKEdkhnZGoMARQxQxEbIRoZU", - "ySmjQKtmm0reslnchYENY5VnFwXhKlHFLeUtswyfV+NhfckLC0Mm0JiNyXBmqhmn163bv1nR+fpNqn6m", - "lFR15caSNoh4lKacxQ4hLZ1CzEYsRmBXQPYFtJOQeMIEFD5T1eqQ0CsVzBk1ZVxDGG+AbinneWLhSbRj", - "w2SSccNSDv6e3t0Utk7yp26lOmIjzIQAdQW5em6xUgJaN6bNhWyWy1I84qI+hWE2HluVlFV3yrRmYoxy", - "66IRA04HPguvrZ2cNeeMLcVBkGFDNLyyebjFYQq8DALvUZbZRCpABU680SpSMTElnNErJtKsERJLVfk8", - "Uy6t+UURGcrMuGjmDVYm4kpe5+sjmQnaqKyaOl4C4b4fqGpCG2KykICzxOpWfrL6nJOTn9aaIyzSZIaT", - "vOBaMEDSEOyOT5+ikZKJLR0MYQIUSsCQ0H0UHL3Hrs7GEW5ZTFECiRRIjkb/thwUrlKPchnnFqcLZUDh", - "IC5XAL0ipoG1ch7RhiQp2jl/fry3t3e4mLL7B61ur9U7+LPXHXTtv//FEfap1laSxEArJKN6wGDjkBmq", - "1M9BSz4FihIi2Ai0QeHJMmU9If2DRwMyjHv9PQqj/YNH7Xa7iQwIo2apZKKB1LPi3mam6PjSuDVfs60n", - "P2aHe2hsNpHlGz47+vOlbXkzrTpcxoR39JCJQel78XV+w33wX4dMNDZERcxd4NSFmBARbPr2boSYRiPC", - "+EIjnmach+sDK4mAuACkdMFmiV7XpfnXFpqcfQWKGhtjQ8a20fCI+7EOOMKfM8jgKpWaeeq18US4Y4uE", - "YcY4Re4NtGOFy0scd6la4PSXil8qJV3Z4MuOGuGnRb1uKdtnAs1MGMbd2GJWoXiw9+jxf3UPe/2SczNh", - "Hu3jjVgpwu5Cze5kDnejIianIKjPoBYG/lMsxdR6hfvi+LNxxgOnEsDzezVj2O6IifEVZQ3o/MvfRJQp", - "iI3ry9f7EO6QNF0PxeaqrohphfiliNyYW/Jesp5e7j6U790ulN/PdKY+ayH6SguS6olsEDXvlQnKn0Hw", - "hWmjQzvOdLkfLyQPE7zFNrlpslOpBsPMZkXLudmMpqE0OKr2OplgnzOodEPHb0+e9kNLWyVjvu6Tw8df", - "vhBz+Ihd68OvyVCN/94jD2Q+tHKi86NjGTm6xVSmCVpFs810aMOBfvcgJsIsbbC91mwsgKKTM0QoVaB1", - "OR/ky1eN3jvst3uPHrd73W67190kOyYkXkH79Oh4c+Ldvq/8BmQ4iOkARj+QnYPZ/KSQ8Gsy0+gyH7Nc", - "YnQ9AYGCmRaycxjFbNQf1Odd3zfeWrDC2gHWbQZWG0UPNxldEvov3NT09nH/YGncX2tVm8tgXb+dJ7IL", - "97B7S6bpUiFkeisZ+mty11oZSsO9bQz0FsNIKTjdz/iO2Uq7MsPL7bZxCXKRm7kqUn7bVXQwuBQt5EeB", - "dIDenZ6isDoaZgYVY32gaOeYy4yil7MU1JRpqZAghk1h165wngnBxNiu4KJubO/wGVL++uqXz0imPXX7", - "buq+rX7jYpIZKq+Fe0dPMoPsN8eyFSEUFKuX8HAeoNfSvRM4jWwAXahM/ONE0OGs/vhiFbMTE4GGNilr", - "IxXQ3UtRKpqDpnGEg8ZwhL34OMK5VPaj5859coRLlp47gUdVvdTM0Xpl5Ar7nzy1+SJ/dmFmpE3Ld+2b", - "OONdl7bdw9tOKZrqs7eLBdktps+rtqv9vrG9t1R/5R3q70yyP+GkuxyTciJro1Et8v3g0QCm8zMBVhQb", - "i8fZ4tjpjg4I5PVknfKqEwN5NrlqwmSw6gpMFl63uj+d04hWH0qwgIA4U8zMLmwy8jofAlGgjjKvc5el", - "nBDu8pz4xJgU39y4Sf2oIZa8sN0Pi9HR2Ylr3xIiyNjG+3eniLMRxLOYA8rcVL0WjN0W8Zvjk9aQ2ICf", - "9xCup2TGKcQ+nRBh18cRnoLSnm633W+7jX6ZgiApwwO81+61bU1t1eBE7EyK8fIYHOws6FwsOqGOdxMG", - "0FazOpVCe930u10/jxcm4JXMt2Q6f2s/I/KZfV3eDxScChec0arB9yieUV8D6CxJiJpZ2d1VFE8g/uRu", - "dVwdoJcK9Ippc+If+UGJNipp/BS9XsfUJLV82YossH8T4f1u78407PfWGsi+FSQzE6nYV6CW6MEdmnUp", - "0RNhQAnCkQY1BRV2SspOiAfvq+73/sPNh7LdnbrmukqlbrB16aAR9oEBtHki6ezORGw4ynRTDUI2nd3U", - "kNa/Mw4CwBqU7KYew3ws6atTomci3vXo2oKhnxCK8m3WfwrR+939LSB6YWfvAXnSWca5O60UxtLzvYRy", - "PO18syXNjU9uHHzLVPW2p+567m0pUSQBA0o7DhZsdP6qBSKW1NYnXnWhibN3XY7OK768lKp6VFRS3GIJ", - "8KHmbfsN9b2j6kX5DZMNYOKtmwMjWlot/ID9/VHO+UnOP/rPw1D2j/5zP5b9Y+9ofqDzfsDS3VZozg+Z", - "/AbfWvC9gJDs50pzoSk06WuqveKprRR8+czrNjVfweHvsm+Tsq+srpWV33z+eI/F38Jx743qv7sz8Rxv", - "TQoPQ6wwffil6r6HAmmPIleB+bPhbG7RcozrfGN0k/qrtCG4KgWXB5xu5rus/nLDk7uuvnLiWy/AcsIP", - "Mg26DST3w4JQjJVyzdJ67KfDQ3e7sW/rZdaDhpirtGqqqweiDpfjctm1uNOpgCTzMyq2t9SSA7JvIaLR", - "hWOwdQHCoGdTK137UpyDyZTQblzNiTboNeJMgEY7Vm1Kcg4UDWfoo+XqIyrgvBvZVwSS4cg9n10K+wYT", - "GWikHS9MjJGA67AgG6GPI8m5vP5vi9+Pbbf9tNR3XllZ/yH/iZZv3npZjETKKc4f8wJ3rtjR/ZyBms0J", - "hzPPc1LFXL/XbdzWqG1SBZ02qpSMjDtUwgwjHMnM+HPUTYx4zTezsuQE0wZhxMAX0wGLpZbnr+pQi3qt", - "F+NyHARDOxcXz3Z/B4wNc5JTWeHpzsODAhvCRtjwdftbjZX7uX/gl09b+c74PwzD/e7h/ZM+lmLEWWxQ", - "a44jywUTtiQWdDhztp0fOXhIDhIAPZfMhekgV6OP5PeW+kg47fDL+8gcH7+4l8RSKYiNP6z0sDYfSuVm", - "yd133Pmm+bmhKG953p2eNieWcDit881/OFnXK8//GsNPU9mFYw/ryOQCPghfDTJRCOd/tu6nsjiZ8kC3", - "XdwPpIMILnWUu/7m/FD+WyO/DrrvfsDb9DdbNhrvbtW3irN1P4tvbTsbBh4Id7+Xqujjobi5R1ouiZEL", - "Q+DSyeul21zhEPZWNrlCaLnFFlcuwe/dgA02uErKWrW9VQT4+9vc+o7Yd3fGzVG2NPL93tb66be1prkN", - "51Fsw42szcqXDauK+9jEKkrb7W5hvft5Mm7pl4MP8CjTtEhiy/bOfi4IdrcXWLe9Z/buAXdoLyBP2KX9", - "MreAXbEJMK9kTDiiMAUuU/dLcf8sjnCmePhBwaDj/3LFRGozeNx93MU3H27+PwAA//8Mq1KNYFIAAA==", + "H4sIAAAAAAAC/+xci28Tubr/VyzfXam9miSTtOXSrK6uSsujEoWqXVjpUk5xZr4kXjz2YHtSAur/fuTH", + "TOaVJoU2Sw9ISCTz8Pf6+Xs6/YojkaSCA9cKD79iFU0hIfbjgdYkmr4VLEvgDD5loLS5nEqRgtQU7EOJ", + "yLi+TImemm8xqEjSVFPB8RCfEj1FV1OQgGZ2FaSmImMxGgGy70GMAwyfSZIywEPcS7juxUQTHGA9T80l", + "pSXlE3wdYAkkFpzNHZkxyZjGwzFhCoIa2ROzNCIKmVc69p1ivZEQDAjH13bFTxmVEOPhu7IY74uHxehv", + "iLQhfiiBaDhOyGS5JjhJoKmD14fHiJr3kIQxSOARoC3oTroBikX0EWSXih6jI0nkvMcnlH8eMqJB6e2K", + "am5+tqmvmniWtxsE40oTHi2XDfjM/EfimBq5CDut3G4Yq6qDp3xGpeAJcI1mRFIyYqDK4n3Fr14fPb18", + "+uotHhrKcRbZVwN8+vrsTzzEO2EYmnUb/E+FTlk2uVT0C1SQgXeeP8F1Rg4K/lECiZBzNBYS+TXQ1jRL", + "CO8Y1BgOzb2EaMToR0AXZr0LHKAL3H9+gavGGVhSDSVYs6+FiBWmJiylHJbaOlgCvRdVccxDaIuJK5AR", + "UYAYaA1SBSimE6pVgAiPUUzUFBQym+YPFBHOhUZKE6mRkAh4jK6oniJin6sqIZl3roT8yASJO30c4IR8", + "fgl8YvzCo50Ap8RQM2z96x3pfAk7+++3/IfO+//OL23/32+t8oE2azdFfOVuoEjwMZ1kkpjr1qh6Coh6", + "WOOgAWejkbgCGC2zhif5awp6ChJpgYh1hsWS5pIh4V9HOYcljbgFW/xOA8RiBpKReQuI+2ELiv+SVFuL", + "+vdQTNVHZF5eAWGzmsPwXtgEcdiO4hamWnh6YhDl99Q6nBSM9Acn/uNg3X01i9JMVVga1Nl5lSUjkEiM", + "0YxKnRGGDk/fVFzOoFiYcg0TkHZlG6VUE2cuCKoSELz9CzwQjSLjSw3+NE0M5qiGxK71m4QxHuL/6i1C", + "bc/H2Z5b2YVa4yBLXo5ISebtrjx3Lstd+oqwTeMWx5R63xhlSosE0Ri4pmMKEm2RTIvOBDhIoiFGdIyM", + "Z0ilmNEY4qrZZoJ1TBS3bmBNX+XYRV64ilexSznLLMPn5WTUXPLcwJByNKETMprrasTph037tys6X79N", + "1U+lFLKp3EjELSIepCmjkUVIR6UQ0TGNEJgVkHkBbSUkmlIOxZ6panVE4kvpzRm0RVxNKGuBbinmOWL+", + "SbRl3GSSMU1TBu6e2l4XtlbyI7tSE7EBppyDvIRcPbdYKQGlWsNmLZrlshSPWK8fwyibTIxKyqo7oUpR", + "PkG5ddGYAouHLgqvzJ2sNReMLcWBl2FNNLw0cbjDYAasDAK3owyziZCACpw4o1WkonxGGI0vKU+zVkgs", + "VeWzTNqw5hZFZCQybb2ZM1iZiE157V4fi4zHrcpqqOMFEObqgaomlCY68wE4S4xuxUejzwU58XGlOfwi", + "bWY4zhOumgGSFmd3eHKExlIkJnXQhHKQKAFNfPVRcPQO2zwbB7hjMBUTSARHYjz+w3BQbJWml8sYMzit", + "pQHFBrGxAuJLoltYK8cRpUmSoq2zZ4c7Ozv79ZA92OuE/U5/789+OAzNv//HAXah1mSSREPHB6Omw6AT", + "Hxmq1M9ACTaDGCWE0zEojfyTZcpqSgZ7j4ZkFPUHOzGMd/cedbvdNjLAtZyngvIWUk+Le+uZoudS485i", + "za6afp8d7qGwWUeWr/j04M8XpuTNlOwxERHWUyPKh6XvxdfFDfvBfR1R3loQFT63xql1Md4jmPDtthGi", + "Co0JZbVCPM0Y89eHRhIOUQFIYZ3NEr2uCvOvDDQZ/QIxai2MNZmYQsMh7vsq4AB/yiCDy1Qo6qg32hP+", + "jkkSRhllMbJvoC0jXJ7i2EvVBGewVPxSKmnTBpd2NAgfFfm6oWye8TQzrimzbYt5heLezqPH/xPu9wel", + "zU25frSL12KlcLu1nN3K7O8GhU9OgccughoYuE+R4DOzK+wXy5/xMw44FQee32sYw1RHlE8uY9qCzr/c", + "TRRTCZG2dfnqPYR7JE1XQ7E9qyt8WiF+ySO3xpa8lmyGl7t35Tu3c+X3051p9lqIulScpGoqWkTNa2WC", + "8mcQfKZKK1+OU1WuxwvJfQevXia3dXYq2aDv2dxQcq7Xo2lJDQ6qtU7G6acMKtXQ4Zvjo4Evaatk9Jdd", + "sv/482ei9x/RK7X/JRnJyd875IH0h27s6HxvW0aMb9GVaYNWUWxT5ctwiL+5ERNgmrbYXik64RCj41NE", + "4liCUuV4kC9fNXp/f9DtP3rc7Ydhtx+uEx0TEt1A++TgcH3i4cBlfkMyGkbxEMbfEZ292VynkLArMlfo", + "Im+zXGB0NQWOvJlq0dm3YtaqD5r9rm9rb9WssLKBdZuG1Vrew3ZGl7j+c9s1vb3f31vq91da1cQyWFVv", + "54Hs3D5s3xJpulQIkd5KhsGK2LVShlJzbxMNvbobKTmn+2nfUZNpV3p4ud3WTkHOczNXRcpv24wOhhe8", + "g1wrMB6itycnyK+ORplGRVsfYrR1yEQWoxfzFOSMKiERJ5rOYNuscJZxTvnErGC9bmTusDmS7vrNL5+S", + "TDnq5t3Ufrv5jfNppmNxxe07apppZL5Zlo0IPqG4eQkH5yF6Jew7ntPAONBaZuIeJzwezZuP17OYrYhw", + "NDJBWWkhId6+4KWk2WsaB9hrDAfYiY8DnEtlPjru7CdLuGTpxSZwqGqmmqTAWQukX1KlzQaJMilNLld6", + "GG1Bkup5XtPkoN/+VpQf87Foa/vddSoc7t+2q9GWz72pJ3D/id3qsl/Jiaz0KA3v9Z3jfaryub4RxfjT", + "SVZvHd3RkD/PCZuUb5r65xHhsg0n3qo34MT1u1a2KBc0glUHC1p3VnNa4117K9vHR/XM2lVbC82Uglyt", + "g6z0UpmCG23vzliYe3di6dah6xRKPszIUbbsqgKyvj1KGqxIVuKkaR+zYSHKJNXzc+MKnTVGQCTIg8zp", + "xfpIS9peXsg61TrF19d2GuKMWpXzuakwaYQOTo9tiZwQTiYmpr49QYyOIZpHDFBmJxeNgGfH8K8Pjzsj", + "YoJqXqfZup1qq37zdEK4WR8HeAZSObphd9C1hylECpykFA/xTrffNXWL0YgVsTctWvgTsG7BwNH67+PY", + "8q59k9/oT6WCK6ebQRi6mQfX3p+Qxdir97dyfTgXV1ZFHU/BqrDmLI0aXB3oGHV5lsqShMi5kd1eRdEU", + "oo/2Vs/mWmqpQCZ4HrtHvlOitQKqm1Q0c8WGpHlQ9+xfB3g37N+Zht38soXsG04yPRWSfoHYEN27Q7Mu", + "JXrMNUhOGFIgZyD9NKq8CfHwXXX7vXt//b5sd6uuha5SoVpsXTrMhZ2XAKWfiHh+ZyK2HBe7rnokU/1c", + "N5A2uDMOPMBalGw7S6O89esqAKLmPNp26NqAoZ+QGOWj7H8K0bvh7gYQXZuePqCddJoxZk+E+db/Yl5T", + "9qe9ryblvHbBjYErS6u77chez3dbSiRJQINUloOajc5edoBHIjb5o1OdL5TNXR+uXUaep7rVHRWUFFdP", + "0d43dttuSy5lqTpRfsFkDZg46+bACJZmC99hf3dcdnFa9vfBM9/4/n3wzLW+f985WByavR+whJtyzflB", + "nl/gWwm+5+CD/UJp1jX5XH9Ftlc8tZGEL+8r3ibnKzj8lfatk/aV1XVj5rfo8d5j8lc7Ur9W/nd3Jl7g", + "rU3hvk3gu0M/Vd73UCDtUGQzMHf+ni4sWvZxva80Xif/Kg1dbwrBBTaOj5Dtqy/Lv2wf5a6zr5z4xhOw", + "nPCDDIN2SGd/vOGTsVKsWZqP/XB4CDfr+zaeZj1oiNlMq6G6piPqMTEpp131abIEkizOAZnaUgkGyLyF", + "iELnlsHOOXCNns6MdN0LfgY6k1zZfjAjSqNXiFEOCm0ZtUnBGMRoNEcfDFcfUAHn7cC8wpHwP2tg8wtu", + "3qA8A4WU5YXyCeJw5RekY/RhLBgTV/9r8Puha0d8S/fOSyPrP7R/guUDcieLFkhaxbmjdGDPblu6nzKQ", + "8wVhf658QaqYu/TD1rFTY7DnddqqUjLW9uAO1ZQwJDLtzqq3MeI0387Ksib/ajei4bPugcFSx/FX3VB1", + "vTaTcTHxgqGt8/On278cxpoxyaqs2Ol2h3sFtrgNP1S3k6/WzP3MPfDTh6389ME/DMPdcP/+SR8KPmY0", + "0qizwJHhgnKTEvN4NLe2XRzreEgbxAN6IZl1016u1j2S31u6R/yJkp9+jyzw8ZPvkkhICZF2B8Ie1vCh", + "lG6WtvuWPUO2OJsV5CXP25OT9sDiDwD2vroPx6tq5cVfvPhhMjt/LGUVmVzAB7FXvUwxuIMpm9+nojg5", + "9EDHLvZH6F4EGzrKVX97fCj/PZefB9133+Bt+7s4a7V3N7q38kNfP8ze2nQ09DwQZn+TVtHHQ9nmDmm5", + "JFrUmsCl0+1Lx1z+oPtGhlzetdxixJVL8GsasMaAq6Ssm8ZbhYO/v+HWN/i+uzNujrKlnu/XWOuHH2vN", + "chsuvNiag6z10pc1s4r7GGIVqe1mR1hvf5yIWzpx/gCPMs2KILZsdvZjQTDcnGPd9Mzs7QOu0J5DHrBL", + "8zK7gFmxDTAvRUQYimEGTKT2Jw/uWRzgTDL/g4Jhz/11kKlQevg4fBzi6/fX/w4AAP//SV8pbsRTAAA=", } // GetSwagger returns the content of the embedded swagger specification file diff --git a/lib/volumes/README.md b/lib/volumes/README.md index 665ab931..b3998e51 100644 --- a/lib/volumes/README.md +++ b/lib/volumes/README.md @@ -21,11 +21,21 @@ When an instance with volumes is created, each volume's raw disk file is passed The init process inside the guest reads the requested mount paths from the config disk and mounts each volume at its specified path. +## Multi-Attach (Read-Only Sharing) + +A single volume can be attached to multiple instances simultaneously if **all** attachments are read-only. This enables sharing static content (libraries, datasets, config files) across many VMs without duplication. + +**Rules:** +- First attachment can be read-write or read-only +- If any attachment is read-write, no other attachments are allowed +- If all existing attachments are read-only, additional read-only attachments are allowed +- Cannot add read-write attachment to a volume with existing attachments + ## Constraints - Volumes can only be attached at instance creation time (no hot-attach) -- A volume can only be attached to one instance at a time - Deleting an instance detaches its volumes but does not delete them +- Cannot delete a volume while it has any attachments ## Storage diff --git a/lib/volumes/manager.go b/lib/volumes/manager.go index ff9b5fe7..b82297d6 100644 --- a/lib/volumes/manager.go +++ b/lib/volumes/manager.go @@ -19,8 +19,12 @@ type Manager interface { DeleteVolume(ctx context.Context, id string) error // Attachment operations (called by instance manager) + // Multi-attach rules: + // - If no attachments: allow any mode (rw or ro) + // - If existing attachment is rw: reject all new attachments + // - If existing attachments are ro: only allow new ro attachments AttachVolume(ctx context.Context, id string, req AttachVolumeRequest) error - DetachVolume(ctx context.Context, id string) error + DetachVolume(ctx context.Context, volumeID string, instanceID string) error // GetVolumePath returns the path to the volume data file GetVolumePath(id string) string @@ -191,8 +195,8 @@ func (m *manager) DeleteVolume(ctx context.Context, id string) error { return err } - // Check if volume is attached - if meta.AttachedTo != nil { + // Check if volume has any attachments + if len(meta.Attachments) > 0 { return ErrInUse } @@ -208,6 +212,10 @@ func (m *manager) DeleteVolume(ctx context.Context, id string) error { } // AttachVolume marks a volume as attached to an instance +// Multi-attach rules (dynamic based on current state): +// - If no attachments: allow any mode (rw or ro) +// - If existing attachment is rw: reject all new attachments +// - If existing attachments are ro: only allow new ro attachments func (m *manager) AttachVolume(ctx context.Context, id string, req AttachVolumeRequest) error { lock := m.getVolumeLock(id) lock.Lock() @@ -218,35 +226,64 @@ func (m *manager) AttachVolume(ctx context.Context, id string, req AttachVolumeR return err } - // Check if already attached - if meta.AttachedTo != nil { - return ErrInUse + // Check if this instance is already attached + for _, att := range meta.Attachments { + if att.InstanceID == req.InstanceID { + return fmt.Errorf("volume already attached to instance %s", req.InstanceID) + } + } + + // Apply multi-attach rules + if len(meta.Attachments) > 0 { + // Check if any existing attachment is read-write + for _, att := range meta.Attachments { + if !att.Readonly { + return fmt.Errorf("volume has exclusive read-write attachment to instance %s", att.InstanceID) + } + } + // Existing attachments are all read-only, new attachment must also be read-only + if !req.Readonly { + return fmt.Errorf("cannot attach read-write: volume has existing read-only attachments") + } } - // Update attachment info - meta.AttachedTo = &req.InstanceID - meta.MountPath = &req.MountPath - meta.Readonly = req.Readonly + // Add new attachment + meta.Attachments = append(meta.Attachments, storedAttachment{ + InstanceID: req.InstanceID, + MountPath: req.MountPath, + Readonly: req.Readonly, + }) return saveMetadata(m.paths, meta) } -// DetachVolume marks a volume as detached from an instance -func (m *manager) DetachVolume(ctx context.Context, id string) error { - lock := m.getVolumeLock(id) +// DetachVolume removes the attachment for a specific instance +func (m *manager) DetachVolume(ctx context.Context, volumeID string, instanceID string) error { + lock := m.getVolumeLock(volumeID) lock.Lock() defer lock.Unlock() - meta, err := loadMetadata(m.paths, id) + meta, err := loadMetadata(m.paths, volumeID) if err != nil { return err } - // Clear attachment info - meta.AttachedTo = nil - meta.MountPath = nil - meta.Readonly = false + // Find and remove the attachment for this instance + found := false + newAttachments := make([]storedAttachment, 0, len(meta.Attachments)) + for _, att := range meta.Attachments { + if att.InstanceID == instanceID { + found = true + continue // Skip this attachment (remove it) + } + newAttachments = append(newAttachments, att) + } + + if !found { + return fmt.Errorf("volume not attached to instance %s", instanceID) + } + meta.Attachments = newAttachments return saveMetadata(m.paths, meta) } @@ -259,13 +296,21 @@ func (m *manager) GetVolumePath(id string) string { func (m *manager) metadataToVolume(meta *storedMetadata) *Volume { createdAt, _ := time.Parse(time.RFC3339, meta.CreatedAt) + // Convert stored attachments to domain attachments + attachments := make([]Attachment, len(meta.Attachments)) + for i, att := range meta.Attachments { + attachments[i] = Attachment{ + InstanceID: att.InstanceID, + MountPath: att.MountPath, + Readonly: att.Readonly, + } + } + return &Volume{ - Id: meta.Id, - Name: meta.Name, - SizeGb: meta.SizeGb, - CreatedAt: createdAt, - AttachedTo: meta.AttachedTo, - MountPath: meta.MountPath, - Readonly: meta.Readonly, + Id: meta.Id, + Name: meta.Name, + SizeGb: meta.SizeGb, + CreatedAt: createdAt, + Attachments: attachments, } } diff --git a/lib/volumes/manager_test.go b/lib/volumes/manager_test.go new file mode 100644 index 00000000..3f78f932 --- /dev/null +++ b/lib/volumes/manager_test.go @@ -0,0 +1,294 @@ +package volumes + +import ( + "context" + "os" + "testing" + + "github.com/onkernel/hypeman/lib/paths" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func setupTestManager(t *testing.T) (Manager, *paths.Paths, func()) { + t.Helper() + + // Create temp dir + tmpDir, err := os.MkdirTemp("", "volume-test-*") + require.NoError(t, err) + + p := paths.New(tmpDir) + + // Create required directories + require.NoError(t, os.MkdirAll(p.VolumesDir(), 0755)) + + manager := NewManager(p, 0) // 0 = unlimited storage + + cleanup := func() { + os.RemoveAll(tmpDir) + } + + return manager, p, cleanup +} + +func TestMultiAttach_FirstAttachmentRW(t *testing.T) { + manager, _, cleanup := setupTestManager(t) + defer cleanup() + ctx := context.Background() + + // Create a volume + vol, err := manager.CreateVolume(ctx, CreateVolumeRequest{ + Name: "test-vol", + SizeGb: 1, + }) + require.NoError(t, err) + + // First attachment as read-write should succeed + err = manager.AttachVolume(ctx, vol.Id, AttachVolumeRequest{ + InstanceID: "instance-1", + MountPath: "/data", + Readonly: false, + }) + assert.NoError(t, err) + + // Verify attachment + vol, err = manager.GetVolume(ctx, vol.Id) + require.NoError(t, err) + require.Len(t, vol.Attachments, 1) + assert.Equal(t, "instance-1", vol.Attachments[0].InstanceID) + assert.Equal(t, "/data", vol.Attachments[0].MountPath) + assert.False(t, vol.Attachments[0].Readonly) +} + +func TestMultiAttach_FirstAttachmentRO(t *testing.T) { + manager, _, cleanup := setupTestManager(t) + defer cleanup() + ctx := context.Background() + + // Create a volume + vol, err := manager.CreateVolume(ctx, CreateVolumeRequest{ + Name: "test-vol", + SizeGb: 1, + }) + require.NoError(t, err) + + // First attachment as read-only should succeed + err = manager.AttachVolume(ctx, vol.Id, AttachVolumeRequest{ + InstanceID: "instance-1", + MountPath: "/data", + Readonly: true, + }) + assert.NoError(t, err) + + // Verify attachment + vol, err = manager.GetVolume(ctx, vol.Id) + require.NoError(t, err) + require.Len(t, vol.Attachments, 1) + assert.True(t, vol.Attachments[0].Readonly) +} + +func TestMultiAttach_RejectSecondAttachWhenRW(t *testing.T) { + manager, _, cleanup := setupTestManager(t) + defer cleanup() + ctx := context.Background() + + // Create a volume + vol, err := manager.CreateVolume(ctx, CreateVolumeRequest{ + Name: "test-vol", + SizeGb: 1, + }) + require.NoError(t, err) + + // First attachment as read-write + err = manager.AttachVolume(ctx, vol.Id, AttachVolumeRequest{ + InstanceID: "instance-1", + MountPath: "/data", + Readonly: false, + }) + require.NoError(t, err) + + // Second attachment (either RO or RW) should fail + err = manager.AttachVolume(ctx, vol.Id, AttachVolumeRequest{ + InstanceID: "instance-2", + MountPath: "/data", + Readonly: true, // Even RO should fail when existing is RW + }) + assert.Error(t, err) + assert.Contains(t, err.Error(), "exclusive read-write attachment") +} + +func TestMultiAttach_AllowMultipleRO(t *testing.T) { + manager, _, cleanup := setupTestManager(t) + defer cleanup() + ctx := context.Background() + + // Create a volume + vol, err := manager.CreateVolume(ctx, CreateVolumeRequest{ + Name: "test-vol", + SizeGb: 1, + }) + require.NoError(t, err) + + // First attachment as read-only + err = manager.AttachVolume(ctx, vol.Id, AttachVolumeRequest{ + InstanceID: "instance-1", + MountPath: "/data", + Readonly: true, + }) + require.NoError(t, err) + + // Second attachment as read-only should succeed + err = manager.AttachVolume(ctx, vol.Id, AttachVolumeRequest{ + InstanceID: "instance-2", + MountPath: "/data", + Readonly: true, + }) + assert.NoError(t, err) + + // Verify both attachments + vol, err = manager.GetVolume(ctx, vol.Id) + require.NoError(t, err) + assert.Len(t, vol.Attachments, 2) +} + +func TestMultiAttach_RejectRWWhenExistingRO(t *testing.T) { + manager, _, cleanup := setupTestManager(t) + defer cleanup() + ctx := context.Background() + + // Create a volume + vol, err := manager.CreateVolume(ctx, CreateVolumeRequest{ + Name: "test-vol", + SizeGb: 1, + }) + require.NoError(t, err) + + // First attachment as read-only + err = manager.AttachVolume(ctx, vol.Id, AttachVolumeRequest{ + InstanceID: "instance-1", + MountPath: "/data", + Readonly: true, + }) + require.NoError(t, err) + + // Second attachment as read-write should fail + err = manager.AttachVolume(ctx, vol.Id, AttachVolumeRequest{ + InstanceID: "instance-2", + MountPath: "/data", + Readonly: false, + }) + assert.Error(t, err) + assert.Contains(t, err.Error(), "cannot attach read-write") +} + +func TestMultiAttach_RejectDuplicateInstance(t *testing.T) { + manager, _, cleanup := setupTestManager(t) + defer cleanup() + ctx := context.Background() + + // Create a volume + vol, err := manager.CreateVolume(ctx, CreateVolumeRequest{ + Name: "test-vol", + SizeGb: 1, + }) + require.NoError(t, err) + + // First attachment + err = manager.AttachVolume(ctx, vol.Id, AttachVolumeRequest{ + InstanceID: "instance-1", + MountPath: "/data", + Readonly: true, + }) + require.NoError(t, err) + + // Same instance trying to attach again should fail + err = manager.AttachVolume(ctx, vol.Id, AttachVolumeRequest{ + InstanceID: "instance-1", + MountPath: "/other", + Readonly: true, + }) + assert.Error(t, err) + assert.Contains(t, err.Error(), "already attached") +} + +func TestDetach_RemovesSpecificAttachment(t *testing.T) { + manager, _, cleanup := setupTestManager(t) + defer cleanup() + ctx := context.Background() + + // Create a volume + vol, err := manager.CreateVolume(ctx, CreateVolumeRequest{ + Name: "test-vol", + SizeGb: 1, + }) + require.NoError(t, err) + + // Attach to two instances + err = manager.AttachVolume(ctx, vol.Id, AttachVolumeRequest{ + InstanceID: "instance-1", + MountPath: "/data", + Readonly: true, + }) + require.NoError(t, err) + + err = manager.AttachVolume(ctx, vol.Id, AttachVolumeRequest{ + InstanceID: "instance-2", + MountPath: "/data", + Readonly: true, + }) + require.NoError(t, err) + + // Detach instance-1 + err = manager.DetachVolume(ctx, vol.Id, "instance-1") + assert.NoError(t, err) + + // Verify only instance-2 remains + vol, err = manager.GetVolume(ctx, vol.Id) + require.NoError(t, err) + require.Len(t, vol.Attachments, 1) + assert.Equal(t, "instance-2", vol.Attachments[0].InstanceID) +} + +func TestDetach_ErrorIfNotAttached(t *testing.T) { + manager, _, cleanup := setupTestManager(t) + defer cleanup() + ctx := context.Background() + + // Create a volume + vol, err := manager.CreateVolume(ctx, CreateVolumeRequest{ + Name: "test-vol", + SizeGb: 1, + }) + require.NoError(t, err) + + // Detach from instance that's not attached + err = manager.DetachVolume(ctx, vol.Id, "instance-1") + assert.Error(t, err) + assert.Contains(t, err.Error(), "not attached") +} + +func TestDeleteVolume_RejectIfAttached(t *testing.T) { + manager, _, cleanup := setupTestManager(t) + defer cleanup() + ctx := context.Background() + + // Create a volume + vol, err := manager.CreateVolume(ctx, CreateVolumeRequest{ + Name: "test-vol", + SizeGb: 1, + }) + require.NoError(t, err) + + // Attach it + err = manager.AttachVolume(ctx, vol.Id, AttachVolumeRequest{ + InstanceID: "instance-1", + MountPath: "/data", + Readonly: true, + }) + require.NoError(t, err) + + // Try to delete - should fail + err = manager.DeleteVolume(ctx, vol.Id) + assert.ErrorIs(t, err, ErrInUse) +} + diff --git a/lib/volumes/storage.go b/lib/volumes/storage.go index 9ac7cd2f..57387bee 100644 --- a/lib/volumes/storage.go +++ b/lib/volumes/storage.go @@ -15,15 +15,20 @@ import ( // data.raw # ext4-formatted sparse disk // metadata.json # Volume metadata +// storedAttachment represents an attachment in stored metadata +type storedAttachment struct { + InstanceID string `json:"instance_id"` + MountPath string `json:"mount_path"` + Readonly bool `json:"readonly"` +} + // storedMetadata represents volume metadata that is persisted to disk type storedMetadata struct { - Id string `json:"id"` - Name string `json:"name"` - SizeGb int `json:"size_gb"` - CreatedAt string `json:"created_at"` // RFC3339 format - AttachedTo *string `json:"attached_to,omitempty"` - MountPath *string `json:"mount_path,omitempty"` - Readonly bool `json:"readonly,omitempty"` + Id string `json:"id"` + Name string `json:"name"` + SizeGb int `json:"size_gb"` + CreatedAt string `json:"created_at"` // RFC3339 format + Attachments []storedAttachment `json:"attachments,omitempty"` } // ensureVolumeDir creates the volume directory diff --git a/lib/volumes/types.go b/lib/volumes/types.go index b2505b9e..d0810b60 100644 --- a/lib/volumes/types.go +++ b/lib/volumes/types.go @@ -2,17 +2,20 @@ package volumes import "time" +// Attachment represents a volume attached to an instance +type Attachment struct { + InstanceID string + MountPath string + Readonly bool +} + // Volume represents a persistent block storage volume type Volume struct { - Id string - Name string - SizeGb int - CreatedAt time.Time - - // Attachment state (nil/empty if not attached) - AttachedTo *string // Instance ID if attached - MountPath *string // Mount path in guest if attached - Readonly bool // Whether mounted read-only + Id string + Name string + SizeGb int + CreatedAt time.Time + Attachments []Attachment // List of current attachments (empty if not attached) } // CreateVolumeRequest is the domain request for creating a volume diff --git a/openapi.yaml b/openapi.yaml index 975e3ab7..33c895b7 100644 --- a/openapi.yaml +++ b/openapi.yaml @@ -326,6 +326,23 @@ components: description: Size in gigabytes example: 10 + VolumeAttachmentInfo: + type: object + required: [instance_id, mount_path, readonly] + properties: + instance_id: + type: string + description: ID of the instance this volume is attached to + example: inst-abc123 + mount_path: + type: string + description: Mount path in the guest + example: /mnt/data + readonly: + type: boolean + description: Whether the attachment is read-only + example: false + Volume: type: object required: [id, name, size_gb, created_at] @@ -342,16 +359,11 @@ components: type: integer description: Size in gigabytes example: 10 - attached_to: - type: string - description: Instance ID if attached - example: inst-abc123 - nullable: true - mount_path: - type: string - description: Mount path if attached - example: /mnt/data - nullable: true + attachments: + type: array + description: List of current attachments (empty if not attached) + items: + $ref: "#/components/schemas/VolumeAttachmentInfo" created_at: type: string format: date-time From 439eb66df90be55711db05db6365cf964ffbdcc1 Mon Sep 17 00:00:00 2001 From: Steven Miller Date: Tue, 2 Dec 2025 13:07:14 -0500 Subject: [PATCH 2/6] Mount with noload --- lib/instances/volumes_test.go | 29 ++++++++++++----------------- lib/system/init_script.go | 3 ++- 2 files changed, 14 insertions(+), 18 deletions(-) diff --git a/lib/instances/volumes_test.go b/lib/instances/volumes_test.go index fa359f20..db990be0 100644 --- a/lib/instances/volumes_test.go +++ b/lib/instances/volumes_test.go @@ -116,22 +116,17 @@ func TestVolumeMultiAttachReadOnly(t *testing.T) { t.Logf("Writer instance created: %s", writerInst.Id) // Wait for exec-agent - err = waitForExecAgent(ctx, manager, writerInst.Id, 30*time.Second) + err = waitForExecAgent(ctx, manager, writerInst.Id, 15*time.Second) require.NoError(t, err, "exec-agent should be ready") - // Write test file + // Write test file, sync, and verify in one command to ensure data persistence t.Log("Writing test file to volume...") - _, code, err := execWithRetry(ctx, writerInst.VsockSocket, []string{ - "/bin/sh", "-c", "echo 'Hello from writer' > /data/test.txt", + output, code, err := execWithRetry(ctx, writerInst.VsockSocket, []string{ + "/bin/sh", "-c", "echo 'Hello from writer' > /data/test.txt && sync && cat /data/test.txt", }) require.NoError(t, err) - require.Equal(t, 0, code, "Write command should succeed") - - // Verify file was written - output, code, err := execWithRetry(ctx, writerInst.VsockSocket, []string{"cat", "/data/test.txt"}) - require.NoError(t, err) - require.Equal(t, 0, code) - assert.Contains(t, output, "Hello from writer") + require.Equal(t, 0, code, "Write+verify command should succeed") + require.Contains(t, output, "Hello from writer", "File should contain test data") t.Log("Test file written successfully") // Delete writer instance (detaches volume) @@ -183,25 +178,25 @@ func TestVolumeMultiAttachReadOnly(t *testing.T) { assert.Len(t, vol.Attachments, 2, "Volume should have 2 attachments") // Wait for exec-agent on both readers - err = waitForExecAgent(ctx, manager, reader1.Id, 30*time.Second) + err = waitForExecAgent(ctx, manager, reader1.Id, 15*time.Second) require.NoError(t, err, "reader-1 exec-agent should be ready") - err = waitForExecAgent(ctx, manager, reader2.Id, 30*time.Second) + err = waitForExecAgent(ctx, manager, reader2.Id, 15*time.Second) require.NoError(t, err, "reader-2 exec-agent should be ready") // Verify data is readable from reader-1 t.Log("Verifying data from reader-1...") - output, code, err = execWithRetry(ctx, reader1.VsockSocket, []string{"cat", "/data/test.txt"}) + output1, code, err := execWithRetry(ctx, reader1.VsockSocket, []string{"cat", "/data/test.txt"}) require.NoError(t, err) require.Equal(t, 0, code) - assert.Contains(t, output, "Hello from writer", "Reader 1 should see the file") + assert.Contains(t, output1, "Hello from writer", "Reader 1 should see the file") // Verify data is readable from reader-2 t.Log("Verifying data from reader-2...") - output, code, err = execWithRetry(ctx, reader2.VsockSocket, []string{"cat", "/data/test.txt"}) + output2, code, err := execWithRetry(ctx, reader2.VsockSocket, []string{"cat", "/data/test.txt"}) require.NoError(t, err) require.Equal(t, 0, code) - assert.Contains(t, output, "Hello from writer", "Reader 2 should see the file") + assert.Contains(t, output2, "Hello from writer", "Reader 2 should see the file") // Verify write fails on read-only mount (reader-1) t.Log("Verifying read-only enforcement...") diff --git a/lib/system/init_script.go b/lib/system/init_script.go index fdfcb5fe..4e7d06a6 100644 --- a/lib/system/init_script.go +++ b/lib/system/init_script.go @@ -83,8 +83,9 @@ if [ -n "${VOLUME_MOUNTS:-}" ]; then mkdir -p "/overlay/newroot${path}" # Mount with appropriate options + # For read-only mounts, use noload to skip journal recovery (safe since no writes will occur) if [ "$mode" = "ro" ]; then - mount -t ext4 -o ro "$device" "/overlay/newroot${path}" + mount -t ext4 -o ro,noload "$device" "/overlay/newroot${path}" else mount -t ext4 "$device" "/overlay/newroot${path}" fi From 4aef0ab1cd2c741aba723b760b3c43345c7e1cf4 Mon Sep 17 00:00:00 2001 From: Steven Miller Date: Tue, 2 Dec 2025 13:38:43 -0500 Subject: [PATCH 3/6] Writeable overlay with multi-attach --- cmd/api/api/instances.go | 31 +++++++-- lib/instances/configdisk.go | 26 ++++--- lib/instances/create.go | 53 ++++++++++++-- lib/instances/storage.go | 12 ++++ lib/instances/types.go | 8 ++- lib/instances/volumes_test.go | 40 +++++++---- lib/oapi/oapi.go | 126 ++++++++++++++++++---------------- lib/paths/paths.go | 12 ++++ lib/system/init_script.go | 34 +++++++-- openapi.yaml | 8 +++ 10 files changed, 252 insertions(+), 98 deletions(-) diff --git a/cmd/api/api/instances.go b/cmd/api/api/instances.go index cac8c7ec..2d84ac08 100644 --- a/cmd/api/api/instances.go +++ b/cmd/api/api/instances.go @@ -104,10 +104,27 @@ func (s *ApiService) CreateInstance(ctx context.Context, request oapi.CreateInst if vol.Readonly != nil { readonly = *vol.Readonly } + overlay := false + if vol.Overlay != nil { + overlay = *vol.Overlay + } + var overlaySize int64 + if vol.OverlaySize != nil && *vol.OverlaySize != "" { + var overlaySizeBytes datasize.ByteSize + if err := overlaySizeBytes.UnmarshalText([]byte(*vol.OverlaySize)); err != nil { + return oapi.CreateInstance400JSONResponse{ + Code: "invalid_overlay_size", + Message: fmt.Sprintf("invalid overlay_size for volume %s: %v", vol.VolumeId, err), + }, nil + } + overlaySize = int64(overlaySizeBytes) + } volumes[i] = instances.VolumeAttachment{ - VolumeID: vol.VolumeId, - MountPath: vol.MountPath, - Readonly: readonly, + VolumeID: vol.VolumeId, + MountPath: vol.MountPath, + Readonly: readonly, + Overlay: overlay, + OverlaySize: overlaySize, } } } @@ -461,11 +478,17 @@ func instanceToOAPI(inst instances.Instance) oapi.Instance { if len(inst.Volumes) > 0 { oapiVolumes := make([]oapi.VolumeAttachment, len(inst.Volumes)) for i, vol := range inst.Volumes { - oapiVolumes[i] = oapi.VolumeAttachment{ + oapiVol := oapi.VolumeAttachment{ VolumeId: vol.VolumeID, MountPath: vol.MountPath, Readonly: lo.ToPtr(vol.Readonly), } + if vol.Overlay { + oapiVol.Overlay = lo.ToPtr(true) + overlaySizeStr := datasize.ByteSize(vol.OverlaySize).HR() + oapiVol.OverlaySize = lo.ToPtr(overlaySizeStr) + } + oapiVolumes[i] = oapiVol } oapiInst.Volumes = &oapiVolumes } diff --git a/lib/instances/configdisk.go b/lib/instances/configdisk.go index 60a9aac8..dfc6bdb5 100644 --- a/lib/instances/configdisk.go +++ b/lib/instances/configdisk.go @@ -107,22 +107,32 @@ GUEST_DNS="%s" // Build volume mounts section // Volumes are attached as /dev/vdd, /dev/vde, etc. (after vda=rootfs, vdb=overlay, vdc=config) + // For overlay volumes, two devices are used: base + overlay disk + // Format: device:path:mode[:overlay_device] volumeSection := "" if len(inst.Volumes) > 0 { var volumeLines strings.Builder - volumeLines.WriteString("\n# Volume mounts (device:path:readonly)\n") + volumeLines.WriteString("\n# Volume mounts (device:path:mode[:overlay_device])\n") volumeLines.WriteString("VOLUME_MOUNTS=\"") + deviceIdx := 0 // Track device index (starts at 'd' = vdd) for i, vol := range inst.Volumes { - // Device naming: vdd, vde, vdf, ... - device := fmt.Sprintf("/dev/vd%c", 'd'+i) - readonly := "rw" - if vol.Readonly { - readonly = "ro" - } + device := fmt.Sprintf("/dev/vd%c", 'd'+deviceIdx) if i > 0 { volumeLines.WriteString(" ") } - volumeLines.WriteString(fmt.Sprintf("%s:%s:%s", device, vol.MountPath, readonly)) + if vol.Overlay { + // Overlay mode: base device + overlay device + overlayDevice := fmt.Sprintf("/dev/vd%c", 'd'+deviceIdx+1) + volumeLines.WriteString(fmt.Sprintf("%s:%s:overlay:%s", device, vol.MountPath, overlayDevice)) + deviceIdx += 2 // Overlay uses 2 devices + } else { + mode := "rw" + if vol.Readonly { + mode = "ro" + } + volumeLines.WriteString(fmt.Sprintf("%s:%s:%s", device, vol.MountPath, mode)) + deviceIdx++ // Regular volume uses 1 device + } } volumeLines.WriteString("\"\n") volumeSection = volumeLines.String() diff --git a/lib/instances/create.go b/lib/instances/create.go index ffe55df6..faa32279 100644 --- a/lib/instances/create.go +++ b/lib/instances/create.go @@ -285,6 +285,15 @@ func (m *manager) createInstance( cu.Add(func() { m.volumeManager.DetachVolume(ctx, volumeID, id) }) + + // Create overlay disk for volumes with overlay enabled + if volAttach.Overlay { + log.DebugContext(ctx, "creating volume overlay disk", "id", id, "volume_id", volAttach.VolumeID, "size", volAttach.OverlaySize) + if err := m.createVolumeOverlayDisk(id, volAttach.VolumeID, volAttach.OverlaySize); err != nil { + log.ErrorContext(ctx, "failed to create volume overlay disk", "id", id, "volume_id", volAttach.VolumeID, "error", err) + return nil, fmt.Errorf("create volume overlay disk %s: %w", volAttach.VolumeID, err) + } + } } // Store volume attachments in metadata stored.Volumes = req.Volumes @@ -373,8 +382,16 @@ func validateCreateRequest(req CreateInstanceRequest) error { // validateVolumeAttachments validates volume attachment requests func validateVolumeAttachments(volumes []VolumeAttachment) error { - if len(volumes) > MaxVolumesPerInstance { - return fmt.Errorf("cannot attach more than %d volumes per instance", MaxVolumesPerInstance) + // Count total devices needed (each overlay volume needs 2 devices: base + overlay) + totalDevices := 0 + for _, vol := range volumes { + totalDevices++ + if vol.Overlay { + totalDevices++ // Overlay needs an additional device + } + } + if totalDevices > MaxVolumesPerInstance { + return fmt.Errorf("cannot attach more than %d volume devices per instance (overlay volumes count as 2)", MaxVolumesPerInstance) } seenPaths := make(map[string]bool) @@ -397,6 +414,16 @@ func validateVolumeAttachments(volumes []VolumeAttachment) error { return fmt.Errorf("duplicate mount path %q", cleanPath) } seenPaths[cleanPath] = true + + // Validate overlay mode requirements + if vol.Overlay { + if !vol.Readonly { + return fmt.Errorf("volume %s: overlay mode requires readonly=true", vol.VolumeID) + } + if vol.OverlaySize <= 0 { + return fmt.Errorf("volume %s: overlay_size is required when overlay=true", vol.VolumeID) + } + } } return nil @@ -552,12 +579,26 @@ func (m *manager) buildVMConfig(inst *Instance, imageInfo *images.Image, netConf } // Add attached volumes as additional disks + // For overlay volumes, add both base (readonly) and overlay disk for _, volAttach := range inst.Volumes { volumePath := m.volumeManager.GetVolumePath(volAttach.VolumeID) - disks = append(disks, vmm.DiskConfig{ - Path: &volumePath, - Readonly: ptr(volAttach.Readonly), - }) + if volAttach.Overlay { + // Base volume is always read-only when overlay is enabled + disks = append(disks, vmm.DiskConfig{ + Path: &volumePath, + Readonly: ptr(true), + }) + // Overlay disk is writable + overlayPath := m.paths.InstanceVolumeOverlay(inst.Id, volAttach.VolumeID) + disks = append(disks, vmm.DiskConfig{ + Path: &overlayPath, + }) + } else { + disks = append(disks, vmm.DiskConfig{ + Path: &volumePath, + Readonly: ptr(volAttach.Readonly), + }) + } } // Serial console configuration diff --git a/lib/instances/storage.go b/lib/instances/storage.go index 788c7666..a0a4aac5 100644 --- a/lib/instances/storage.go +++ b/lib/instances/storage.go @@ -87,6 +87,18 @@ func (m *manager) createOverlayDisk(id string, sizeBytes int64) error { return images.CreateEmptyExt4Disk(overlayPath, sizeBytes) } +// createVolumeOverlayDisk creates a sparse overlay disk for a volume attachment +func (m *manager) createVolumeOverlayDisk(instanceID, volumeID string, sizeBytes int64) error { + // Ensure vol-overlays directory exists + overlaysDir := m.paths.InstanceVolumeOverlaysDir(instanceID) + if err := os.MkdirAll(overlaysDir, 0755); err != nil { + return fmt.Errorf("create vol-overlays directory: %w", err) + } + + overlayPath := m.paths.InstanceVolumeOverlay(instanceID, volumeID) + return images.CreateEmptyExt4Disk(overlayPath, sizeBytes) +} + // deleteInstanceData removes all instance data from disk func (m *manager) deleteInstanceData(id string) error { instDir := m.paths.InstanceDir(id) diff --git a/lib/instances/types.go b/lib/instances/types.go index 38ad4c4f..82355f1e 100644 --- a/lib/instances/types.go +++ b/lib/instances/types.go @@ -20,9 +20,11 @@ const ( // VolumeAttachment represents a volume attached to an instance type VolumeAttachment struct { - VolumeID string // Volume ID - MountPath string // Mount path in guest - Readonly bool // Whether mounted read-only + VolumeID string // Volume ID + MountPath string // Mount path in guest + Readonly bool // Whether mounted read-only + Overlay bool // If true, create per-instance overlay for writes (requires Readonly=true) + OverlaySize int64 // Size of overlay disk in bytes (max diff from base) } // StoredMetadata represents instance metadata that is persisted to disk diff --git a/lib/instances/volumes_test.go b/lib/instances/volumes_test.go index db990be0..118df5f9 100644 --- a/lib/instances/volumes_test.go +++ b/lib/instances/volumes_test.go @@ -157,8 +157,9 @@ func TestVolumeMultiAttachReadOnly(t *testing.T) { require.NoError(t, err) t.Logf("Reader 1 created: %s", reader1.Id) + // Reader 2 uses overlay mode: can read base data AND write to its own overlay reader2, err := manager.CreateInstance(ctx, CreateInstanceRequest{ - Name: "reader-2", + Name: "reader-2-overlay", Image: "docker.io/library/alpine:latest", Size: 512 * 1024 * 1024, HotplugSize: 512 * 1024 * 1024, @@ -166,11 +167,11 @@ func TestVolumeMultiAttachReadOnly(t *testing.T) { Vcpus: 1, NetworkEnabled: false, Volumes: []VolumeAttachment{ - {VolumeID: vol.Id, MountPath: "/data", Readonly: true}, + {VolumeID: vol.Id, MountPath: "/data", Readonly: true, Overlay: true, OverlaySize: 100 * 1024 * 1024}, // 100MB overlay }, }) require.NoError(t, err) - t.Logf("Reader 2 created: %s", reader2.Id) + t.Logf("Reader 2 (overlay) created: %s", reader2.Id) // Verify volume has two attachments vol, err = volumeManager.GetVolume(ctx, vol.Id) @@ -189,24 +190,37 @@ func TestVolumeMultiAttachReadOnly(t *testing.T) { output1, code, err := execWithRetry(ctx, reader1.VsockSocket, []string{"cat", "/data/test.txt"}) require.NoError(t, err) require.Equal(t, 0, code) - assert.Contains(t, output1, "Hello from writer", "Reader 1 should see the file") + require.Contains(t, output1, "Hello from writer", "Reader 1 should see the file") - // Verify data is readable from reader-2 - t.Log("Verifying data from reader-2...") + // Verify data is readable from reader-2 (overlay mode) + t.Log("Verifying data from reader-2 (overlay)...") output2, code, err := execWithRetry(ctx, reader2.VsockSocket, []string{"cat", "/data/test.txt"}) require.NoError(t, err) require.Equal(t, 0, code) - assert.Contains(t, output2, "Hello from writer", "Reader 2 should see the file") + assert.Contains(t, output2, "Hello from writer", "Reader 2 should see the file from base volume") - // Verify write fails on read-only mount (reader-1) - t.Log("Verifying read-only enforcement...") - _, code, err = execWithRetry(ctx, reader1.VsockSocket, []string{ - "/bin/sh", "-c", "echo 'illegal write' > /data/illegal.txt", + // Verify overlay allows writes: append to the file and verify in one command + t.Log("Verifying overlay allows writes (append to file)...") + output2, code, err = execWithRetry(ctx, reader2.VsockSocket, []string{ + "/bin/sh", "-c", "echo 'Appended by overlay' >> /data/test.txt && sync && cat /data/test.txt", + }) + require.NoError(t, err) + require.Equal(t, 0, code, "Append to file should succeed with overlay") + assert.Contains(t, output2, "Hello from writer", "Reader 2 should still see original data") + assert.Contains(t, output2, "Appended by overlay", "Reader 2 should see appended data") + + // Verify reader-1 does NOT see the appended data AND write fails (all in one command) + t.Log("Verifying read-only enforcement and isolation on reader-1...") + output1, code, err = execWithRetry(ctx, reader1.VsockSocket, []string{ + "/bin/sh", "-c", "cat /data/test.txt && echo 'illegal' > /data/illegal.txt", }) - require.NoError(t, err, "Exec should succeed even if command fails") + require.NoError(t, err, "Exec should succeed even if write command fails") + // Code should be non-zero because the write fails assert.NotEqual(t, 0, code, "Write to read-only volume should fail with non-zero exit") + assert.Contains(t, output1, "Hello from writer", "Reader 1 should see original data") + assert.NotContains(t, output1, "Appended by overlay", "Reader 1 should NOT see overlay data (isolated)") - t.Log("Multi-attach read-only test passed!") + t.Log("Multi-attach with overlay test passed!") // Cleanup t.Log("Cleaning up...") diff --git a/lib/oapi/oapi.go b/lib/oapi/oapi.go index 6ad0352a..9629cf2a 100644 --- a/lib/oapi/oapi.go +++ b/lib/oapi/oapi.go @@ -278,6 +278,12 @@ type VolumeAttachment struct { // MountPath Path where volume is mounted in the guest MountPath string `json:"mount_path"` + // Overlay Create per-instance overlay for writes (requires readonly=true) + Overlay *bool `json:"overlay,omitempty"` + + // OverlaySize Max overlay size as human-readable string (e.g., "1GB"). Required if overlay=true. + OverlaySize *string `json:"overlay_size,omitempty"` + // Readonly Whether volume is mounted read-only Readonly *bool `json:"readonly,omitempty"` @@ -4981,65 +4987,67 @@ func (sh *strictHandler) GetVolume(w http.ResponseWriter, r *http.Request, id st // Base64 encoded, gzipped, json marshaled Swagger object var swaggerSpec = []string{ - "H4sIAAAAAAAC/+xci28Tubr/VyzfXam9miSTtOXSrK6uSsujEoWqXVjpUk5xZr4kXjz2YHtSAur/fuTH", - "TOaVJoU2Sw9ISCTz8Pf6+Xs6/YojkaSCA9cKD79iFU0hIfbjgdYkmr4VLEvgDD5loLS5nEqRgtQU7EOJ", - "yLi+TImemm8xqEjSVFPB8RCfEj1FV1OQgGZ2FaSmImMxGgGy70GMAwyfSZIywEPcS7juxUQTHGA9T80l", - "pSXlE3wdYAkkFpzNHZkxyZjGwzFhCoIa2ROzNCIKmVc69p1ivZEQDAjH13bFTxmVEOPhu7IY74uHxehv", - "iLQhfiiBaDhOyGS5JjhJoKmD14fHiJr3kIQxSOARoC3oTroBikX0EWSXih6jI0nkvMcnlH8eMqJB6e2K", - "am5+tqmvmniWtxsE40oTHi2XDfjM/EfimBq5CDut3G4Yq6qDp3xGpeAJcI1mRFIyYqDK4n3Fr14fPb18", - "+uotHhrKcRbZVwN8+vrsTzzEO2EYmnUb/E+FTlk2uVT0C1SQgXeeP8F1Rg4K/lECiZBzNBYS+TXQ1jRL", - "CO8Y1BgOzb2EaMToR0AXZr0LHKAL3H9+gavGGVhSDSVYs6+FiBWmJiylHJbaOlgCvRdVccxDaIuJK5AR", - "UYAYaA1SBSimE6pVgAiPUUzUFBQym+YPFBHOhUZKE6mRkAh4jK6oniJin6sqIZl3roT8yASJO30c4IR8", - "fgl8YvzCo50Ap8RQM2z96x3pfAk7+++3/IfO+//OL23/32+t8oE2azdFfOVuoEjwMZ1kkpjr1qh6Coh6", - "WOOgAWejkbgCGC2zhif5awp6ChJpgYh1hsWS5pIh4V9HOYcljbgFW/xOA8RiBpKReQuI+2ELiv+SVFuL", - "+vdQTNVHZF5eAWGzmsPwXtgEcdiO4hamWnh6YhDl99Q6nBSM9Acn/uNg3X01i9JMVVga1Nl5lSUjkEiM", - "0YxKnRGGDk/fVFzOoFiYcg0TkHZlG6VUE2cuCKoSELz9CzwQjSLjSw3+NE0M5qiGxK71m4QxHuL/6i1C", - "bc/H2Z5b2YVa4yBLXo5ISebtrjx3Lstd+oqwTeMWx5R63xhlSosE0Ri4pmMKEm2RTIvOBDhIoiFGdIyM", - "Z0ilmNEY4qrZZoJ1TBS3bmBNX+XYRV64ilexSznLLMPn5WTUXPLcwJByNKETMprrasTph037tys6X79N", - "1U+lFLKp3EjELSIepCmjkUVIR6UQ0TGNEJgVkHkBbSUkmlIOxZ6panVE4kvpzRm0RVxNKGuBbinmOWL+", - "SbRl3GSSMU1TBu6e2l4XtlbyI7tSE7EBppyDvIRcPbdYKQGlWsNmLZrlshSPWK8fwyibTIxKyqo7oUpR", - "PkG5ddGYAouHLgqvzJ2sNReMLcWBl2FNNLw0cbjDYAasDAK3owyziZCACpw4o1WkonxGGI0vKU+zVkgs", - "VeWzTNqw5hZFZCQybb2ZM1iZiE157V4fi4zHrcpqqOMFEObqgaomlCY68wE4S4xuxUejzwU58XGlOfwi", - "bWY4zhOumgGSFmd3eHKExlIkJnXQhHKQKAFNfPVRcPQO2zwbB7hjMBUTSARHYjz+w3BQbJWml8sYMzit", - "pQHFBrGxAuJLoltYK8cRpUmSoq2zZ4c7Ozv79ZA92OuE/U5/789+OAzNv//HAXah1mSSREPHB6Omw6AT", - "Hxmq1M9ACTaDGCWE0zEojfyTZcpqSgZ7j4ZkFPUHOzGMd/cedbvdNjLAtZyngvIWUk+Le+uZoudS485i", - "za6afp8d7qGwWUeWr/j04M8XpuTNlOwxERHWUyPKh6XvxdfFDfvBfR1R3loQFT63xql1Md4jmPDtthGi", - "Co0JZbVCPM0Y89eHRhIOUQFIYZ3NEr2uCvOvDDQZ/QIxai2MNZmYQsMh7vsq4AB/yiCDy1Qo6qg32hP+", - "jkkSRhllMbJvoC0jXJ7i2EvVBGewVPxSKmnTBpd2NAgfFfm6oWye8TQzrimzbYt5heLezqPH/xPu9wel", - "zU25frSL12KlcLu1nN3K7O8GhU9OgccughoYuE+R4DOzK+wXy5/xMw44FQee32sYw1RHlE8uY9qCzr/c", - "TRRTCZG2dfnqPYR7JE1XQ7E9qyt8WiF+ySO3xpa8lmyGl7t35Tu3c+X3051p9lqIulScpGoqWkTNa2WC", - "8mcQfKZKK1+OU1WuxwvJfQevXia3dXYq2aDv2dxQcq7Xo2lJDQ6qtU7G6acMKtXQ4Zvjo4Evaatk9Jdd", - "sv/482ei9x/RK7X/JRnJyd875IH0h27s6HxvW0aMb9GVaYNWUWxT5ctwiL+5ERNgmrbYXik64RCj41NE", - "4liCUuV4kC9fNXp/f9DtP3rc7Ydhtx+uEx0TEt1A++TgcH3i4cBlfkMyGkbxEMbfEZ292VynkLArMlfo", - "Im+zXGB0NQWOvJlq0dm3YtaqD5r9rm9rb9WssLKBdZuG1Vrew3ZGl7j+c9s1vb3f31vq91da1cQyWFVv", - "54Hs3D5s3xJpulQIkd5KhsGK2LVShlJzbxMNvbobKTmn+2nfUZNpV3p4ud3WTkHOczNXRcpv24wOhhe8", - "g1wrMB6itycnyK+ORplGRVsfYrR1yEQWoxfzFOSMKiERJ5rOYNuscJZxTvnErGC9bmTusDmS7vrNL5+S", - "TDnq5t3Ufrv5jfNppmNxxe07apppZL5Zlo0IPqG4eQkH5yF6Jew7ntPAONBaZuIeJzwezZuP17OYrYhw", - "NDJBWWkhId6+4KWk2WsaB9hrDAfYiY8DnEtlPjru7CdLuGTpxSZwqGqmmqTAWQukX1KlzQaJMilNLld6", - "GG1Bkup5XtPkoN/+VpQf87Foa/vddSoc7t+2q9GWz72pJ3D/id3qsl/Jiaz0KA3v9Z3jfaryub4RxfjT", - "SVZvHd3RkD/PCZuUb5r65xHhsg0n3qo34MT1u1a2KBc0glUHC1p3VnNa4117K9vHR/XM2lVbC82Uglyt", - "g6z0UpmCG23vzliYe3di6dah6xRKPszIUbbsqgKyvj1KGqxIVuKkaR+zYSHKJNXzc+MKnTVGQCTIg8zp", - "xfpIS9peXsg61TrF19d2GuKMWpXzuakwaYQOTo9tiZwQTiYmpr49QYyOIZpHDFBmJxeNgGfH8K8Pjzsj", - "YoJqXqfZup1qq37zdEK4WR8HeAZSObphd9C1hylECpykFA/xTrffNXWL0YgVsTctWvgTsG7BwNH67+PY", - "8q59k9/oT6WCK6ebQRi6mQfX3p+Qxdir97dyfTgXV1ZFHU/BqrDmLI0aXB3oGHV5lsqShMi5kd1eRdEU", - "oo/2Vs/mWmqpQCZ4HrtHvlOitQKqm1Q0c8WGpHlQ9+xfB3g37N+Zht38soXsG04yPRWSfoHYEN27Q7Mu", - "JXrMNUhOGFIgZyD9NKq8CfHwXXX7vXt//b5sd6uuha5SoVpsXTrMhZ2XAKWfiHh+ZyK2HBe7rnokU/1c", - "N5A2uDMOPMBalGw7S6O89esqAKLmPNp26NqAoZ+QGOWj7H8K0bvh7gYQXZuePqCddJoxZk+E+db/Yl5T", - "9qe9ryblvHbBjYErS6u77chez3dbSiRJQINUloOajc5edoBHIjb5o1OdL5TNXR+uXUaep7rVHRWUFFdP", - "0d43dttuSy5lqTpRfsFkDZg46+bACJZmC99hf3dcdnFa9vfBM9/4/n3wzLW+f985WByavR+whJtyzflB", - "nl/gWwm+5+CD/UJp1jX5XH9Ftlc8tZGEL+8r3ibnKzj8lfatk/aV1XVj5rfo8d5j8lc7Ur9W/nd3Jl7g", - "rU3hvk3gu0M/Vd73UCDtUGQzMHf+ni4sWvZxva80Xif/Kg1dbwrBBTaOj5Dtqy/Lv2wf5a6zr5z4xhOw", - "nPCDDIN2SGd/vOGTsVKsWZqP/XB4CDfr+zaeZj1oiNlMq6G6piPqMTEpp131abIEkizOAZnaUgkGyLyF", - "iELnlsHOOXCNns6MdN0LfgY6k1zZfjAjSqNXiFEOCm0ZtUnBGMRoNEcfDFcfUAHn7cC8wpHwP2tg8wtu", - "3qA8A4WU5YXyCeJw5RekY/RhLBgTV/9r8Puha0d8S/fOSyPrP7R/guUDcieLFkhaxbmjdGDPblu6nzKQ", - "8wVhf658QaqYu/TD1rFTY7DnddqqUjLW9uAO1ZQwJDLtzqq3MeI0387Ksib/ajei4bPugcFSx/FX3VB1", - "vTaTcTHxgqGt8/On278cxpoxyaqs2Ol2h3sFtrgNP1S3k6/WzP3MPfDTh6389ME/DMPdcP/+SR8KPmY0", - "0qizwJHhgnKTEvN4NLe2XRzreEgbxAN6IZl1016u1j2S31u6R/yJkp9+jyzw8ZPvkkhICZF2B8Ie1vCh", - "lG6WtvuWPUO2OJsV5CXP25OT9sDiDwD2vroPx6tq5cVfvPhhMjt/LGUVmVzAB7FXvUwxuIMpm9+nojg5", - "9EDHLvZH6F4EGzrKVX97fCj/PZefB9133+Bt+7s4a7V3N7q38kNfP8ze2nQ09DwQZn+TVtHHQ9nmDmm5", - "JFrUmsCl0+1Lx1z+oPtGhlzetdxixJVL8GsasMaAq6Ssm8ZbhYO/v+HWN/i+uzNujrKlnu/XWOuHH2vN", - "chsuvNiag6z10pc1s4r7GGIVqe1mR1hvf5yIWzpx/gCPMs2KILZsdvZjQTDcnGPd9Mzs7QOu0J5DHrBL", - "8zK7gFmxDTAvRUQYimEGTKT2Jw/uWRzgTDL/g4Jhz/11kKlQevg4fBzi6/fX/w4AAP//SV8pbsRTAAA=", + "H4sIAAAAAAAC/+xciW4bOZN+FYL7DyAvWqftbKzBYuGxcxiIE8OeZICNsw7VXZI4YZMdki1bCfzuCx7d", + "6kuHE1sT/wkQIFIfZN31VRXlrzgUcSI4cK3w8CtW4RRiYj8eak3C6TvB0hjO4XMKSpvLiRQJSE3BPhSL", + "lOurhOip+RaBCiVNNBUcD/EZ0VN0PQUJaGZXQWoqUhahESD7HkQ4wHBD4oQBHuJuzHU3IprgAOt5Yi4p", + "LSmf4NsASyCR4GzuthmTlGk8HBOmIKhse2qWRkQh80rbvpOvNxKCAeH41q74OaUSIjx8X2TjQ/6wGP0N", + "oTabH0kgGk5iMlkuCU5iqMvgzdEJouY9JGEMEngIqAWdSSdAkQg/gexQ0WV0JImcd/mE8pshIxqU3imJ", + "ZvWzdXlV2LO0rWCMK014uJw34DPzH4kiavgi7Kx0u6assgye8RmVgsfANZoRScmIgSqy9xW/fnP87OrZ", + "63d4aHaO0tC+GuCzN+d/4iHe7fV6Zt0a/VOhE5ZOrhT9AiXLwLsv/sBVQg5z+lEMsZBzNBYS+TVQa5rG", + "hLeN1RgKzb2YaMToJ0CXZr1LHKBL3H9xicvKGditakKwat/IItaomrCEcliq62CJ6b0ss2MeQi0mrkGG", + "RAFioDVIFaCITqhWASI8QhFRU1DIOM3vKCScC42UJlIjIRHwCF1TPUXEPlcWQjxvXwv5iQkStfs4wDG5", + "eQV8YuLCk90AJ8TsZsj6v/ek/aXXPvjQ8h/aH/4zu7TzP/9q5A+0WbvO4mt3A4WCj+kklcRct0rVU0DU", + "mzUOauZsJBKVDEbLtBZJ/pqCnoJEWiBig2G+pLlktvCvo4zCgkTcgg1xp2bEYgaSkXmDEfd7DVb8l6Ta", + "atS/hyKqPiHz8hoTNqs5G97v1Y2412zFDUQ10PSHsSjvU5tQkhPSH5z6j4NN/WoWJqkqkTSokvM6jUcg", + "kRijGZU6JQwdnb0thZxBvjDlGiYg7co2S6m6nbkkqAqG4PWf2wPRKDSx1NifprGxOaohtmv9S8IYD/F/", + "dBeptuvzbNet7FKtCZCFKEekJPPmUJ4Fl+UhfU3aplFDYEp8bAxTpUWMaARc0zEFiVok1aI9AQ6SaIgQ", + "HSMTGRIpZjSCqKy2mWBtk8VtGNgwVjlykWeuFFXsUk4zy+zzajKqL3lhzJByNKETMprrcsbp9+r6bxZ0", + "tn6TqJ9JKWRduKGIGlg8TBJGQ2shbZVASMc0RGBWQOYF1IpJOKUccp8pS3VEoivp1Rk0ZVxNKGsw3ULO", + "c5v5J1HLhMk4ZZomDNw9tbOp2VrOj+1KdYsNMOUc5BVk4rnDSjEo1Zg2K9ks4yV/xEb9CEbpZGJEUhTd", + "KVWK8gnKtIvGFFg0dFl4LXay2lwQttQOPA8bWsMrk4fbDGbAikbgPMoQGwsJKLcTp7QSV5TPCKPRFeVJ", + "2mgSS0X5PJU2rblFERmJVNto5hRW3MRCXuvrY5HyqFFYNXG8BMJcPVCWhNJEpz4Bp7GRrfhk5LnYTnxa", + "qw6/SJMaTjLAVVFA3BDsjk6P0ViK2EAHTSgHiWLQxFcfOUXvscXZOMBtY1MRgVhwJMbj3w0FuavUo1zK", + "mLHTCgzIHcTmCoiuiG4grZhHlCZxglrnz492d3cPqil7sN/u9dv9/T/7vWHP/PtfHGCXag2SJBraPhnV", + "Awad+MxQ3v0clGAziFBMOB2D0sg/WdxZTclg/8mQjML+YDeC8d7+k06n07QNcC3niaC8Yatn+b3NVNF1", + "0Li9WLOjpt+nhwcobDbh5Ss+O/zzpSl5UyW7TISEddWI8mHhe/51ccN+cF9HlDcWRHnMrVBqQ4yPCCZ9", + "OzdCVKExoaxSiCcpY/760HDCIcwNUthgs0Su69L8a2OajH6BCDUWxppMTKHhLO77KuAAf04hhatEKOp2", + "r7Un/B0DEkYpZRGyb6CWYS6DOPZSGeAMlrJfgJIWNjjYUdv4OMfrZmfzjN8z5Zoy27aYl3bc333y9L96", + "B/1Bwbkp10/28Eak5GG3gtktz/5ukMfkBHjkMqgxA/cpFHxmvMJ+sfSZOOMMpxTAs3s1ZZjqiPLJVUQb", + "rPMvdxNFVEKobV2+3odwlyTJelNsRnV5TMvZL0TkxtyS1ZL19HL/oXz3bqH8Yboz9V4LUVeKk0RNRQOr", + "Wa1MUPYMghuqtPLlOFXFejzn3HfwqmVyU2enhAZ9z2ZFyblZj6YBGhyWa52U088plKqho7cnxwNf0pa3", + "0V/2yMHTmxuiD57Qa3XwJR7Jyd+75JH0h1Z2dL63LSPGd+jKNJlWXmxT5ctwiL65ERNgmjToXik64RCh", + "kzNEokiCUsV8kC1fVnr/YNDpP3na6fd6nX5vk+wYk3DF3qeHR5tv3hs45Dcko2EYDWH8HdnZq811Cgm7", + "JnOFLrM2yyVG11PgyKupkp19K2aj+qDe7/q29lZFC2sbWHdpWG0UPWxndEnov7Bd07vH/f2lcX+tVk0u", + "g3X1dpbILuzD9i2RJEuZEMmdeBisyV1reSg097bR0KuGkUJwepj2HTVIu9TDy/S2MQS5yNRcZim7bREd", + "DC95G7lWYDRE705PkV8djVKN8rY+RKh1xEQaoZfzBOSMKiERJ5rOYMescJ5yTvnErGCjbmjusDmS7vrq", + "l89Iqtzu5t3Eflv9xsU01ZG45vYdNU01Mt8syYYFDyhWL+HMeYheC/uOpzQwAbSCTNzjhEejef3xKopp", + "hYSjkUnKSgsJ0c4lL4BmL2kcYC8xHGDHPg5wxpX56Kizn+zGBU0vnMBZVR1qktzOGkz6FVXaOEiYSmmw", + "XOFh1II40fOspsmMfudbrfyEj0VT2+++oXDv4K5djSY897YK4P4du9XFuJJtsjai1KLXd473qcrm+oYV", + "E08nabV1tHLI71P++hm/8zeUgGznqDDDC6beuJbUltVeRO4sgOBs/t8m7+zgJjy4Gpackpt8BwsYiEKV", + "GZfjIxvv+ynXTgedZ31nOs6WsGR0yvilGWNsfu4hg8l1Zaw6CJElyasm1/GGvsJ1XAtwbdd2sUew7qxF", + "Y7CpD7C81hvJPjmuFhuuAF1IppD3K011pZfyFKx0B3fsxNz7RuMva7pxDj2FQlg3fBQ1u66mrkaMggRL", + "nBUoqevHxDAIU0n1/MJkB6eNERAJ8jB1crFpw25tLy94nWqd4NtbOyBySi3z+cIU3TREh2cn1otjwsnE", + "uNS7U8ToGMJ5yACldphTwwD2ZMKbo5P2iBickZWutpVBtRW/eTom3KyPAzwDqdy+vc6gY8+XiAQ4SSge", + "4t1Ov2NKOSMRy2J3mk81JmAjpTFHm9JOIku79nMPIz+VCK6cbAa9nhsDce1DLFlMArt/K9eadKl2XSL2", + "O1gRVvKHEYMrjR2hDnqqNI6JnBve7VUUTiH8ZG91LfxUSxkyeOLEPfKdHG2EMdzwpg6fa5xmOMeTfxvg", + "vV7/3iTsRroN277lJNVTIekXiMym+/eo1qWbnnANkhOGFMgZSD+gKzohHr4vu9/7D7cfinq34lrIKhGq", + "QdeF823YRQlQ+g8Rze+NxYYTdLfliGQy4m3N0gb3RoE3sAYh22bbKOuGu6KIqDkPd5x1bUHRf5AIZdP9", + "f8qi93p7W7DoykD5EXnSWcqYPSTnpyGLEVYxnna/GhR+65IbA1epl73t2F7PvC0hksSgQSpLQUVH56/a", + "wEMRGfToROd7B+auT9euSMnQf9mjgoLgqhDtQ83b9hqwlN3VsfLLTDYwE6fdzDCCpWjhO/TvSozFAeLf", + "Bs/9LOC3wXM3Dfht93BxjvhhjKW3rdCcnW36ZXxrje8F+GS/EJoNTR7rr0F7+VNbAXxZq/UumC+n8Bfs", + "2wT2FcW1Evkt2t4PCP4qvzLYCP/dn4oX9tYkcN8m8A2znwr3PRaT9j0/g8DcTxLoQqPFGNf9SqNN8Fdh", + "Dr0qBee2cXKM7KhhGf6yfZT7Rl/Z5lsHYNnGjzIN2rml/T2LB2OFXLMUj/1w9tDbbuzbOsx61CZmkVZN", + "dPVA1GViUoRd1QG7BBIvjkaZ2lIJBsi8hYhCF5bA9gVwjZ7NDHedS34OOpVc2X4wI0qj14hRDgq1jNik", + "YAwiNJqjj4aqjyg3553AvMKR8L/0YPNLbt6gPAWFlKWF8gnicO0XpGP0cSwYE9d2YvGxY6eeS33nleH1", + "H/KfYPmZAceLFkhawbnThWCPs9t9P6cg54uN/VH7xVb53KXfa5zE1WadXqaNIiVjbc8yUU0JQyLV7vh+", + "EyFO8s2kLGvyrw8jGm50F4wttR19ZYeqyrUOxsXEM4ZaFxfPdn4FjA1zkhVZ7unWw70AG8KGP2dgJ1+N", + "yP3cPfDTp63sQMY/bIZ7vYOH3/pI8DGjoUbthR0ZKig3kJhHo7nV7eKky2NyEG/QC85smPZ8NfpIdm+p", + "j/hDNj+9jyzs4yf3klBICaF2Z+Qe1/ChADcL7t6yx+oWx9WCrOR5d3ranFj8mcjuV/fhZF2tvPgjID8M", + "svPHUtZtkzH4KHzV8xSBO5iyfT8V+cmhRzp2sb/L9yzY1FGs+pvzQ/FP3Pw81n3/Dd6mPxW0UXt3q76V", + "Hfr6YXxr29nQ00CY/ZleSR6Pxc2dpWWcaFFpAhcO/C8dc/mz/1sZcvnQcocRV8bBr2nABgOugrBWjbfy", + "AP9ww61viH33p9zMypZGvl9jrR9+rDXLdLiIYhsOsjaDLxuiiocYYuXQdrsjrHc/TsYtnDh/hEeZZnkS", + "WzY7+7FMsLe9wLrtmdm7R1yhvYAsYRfmZXYBs2KTwbwSIWEoghkwkdifPLhncYBTyfwPCoZd9wdTpkLp", + "4dPe0x6+/XD7/wEAAP//oKJwbNdUAAA=", } // GetSwagger returns the content of the embedded swagger specification file diff --git a/lib/paths/paths.go b/lib/paths/paths.go index 4dedc604..05a566d8 100644 --- a/lib/paths/paths.go +++ b/lib/paths/paths.go @@ -27,6 +27,8 @@ // ch.sock // vsock.sock // logs/ +// vol-overlays/ +// {volumeID}.raw // snapshots/ // snapshot-latest/ // config.json @@ -140,6 +142,16 @@ func (p *Paths) InstanceConfigDisk(id string) string { return filepath.Join(p.InstanceDir(id), "config.ext4") } +// InstanceVolumeOverlay returns the path to a volume's overlay disk for an instance. +func (p *Paths) InstanceVolumeOverlay(instanceID, volumeID string) string { + return filepath.Join(p.InstanceDir(instanceID), "vol-overlays", volumeID+".raw") +} + +// InstanceVolumeOverlaysDir returns the directory for volume overlays. +func (p *Paths) InstanceVolumeOverlaysDir(instanceID string) string { + return filepath.Join(p.InstanceDir(instanceID), "vol-overlays") +} + // InstanceSocket returns the path to instance API socket. func (p *Paths) InstanceSocket(id string) string { return filepath.Join(p.InstanceDir(id), "ch.sock") diff --git a/lib/system/init_script.go b/lib/system/init_script.go index 4e7d06a6..9b998517 100644 --- a/lib/system/init_script.go +++ b/lib/system/init_script.go @@ -71,7 +71,8 @@ else exit 1 fi -# Mount attached volumes (from config: VOLUME_MOUNTS="device:path:mode device:path:mode ...") +# Mount attached volumes (from config: VOLUME_MOUNTS="device:path:mode[:overlay_device] ...") +# Modes: ro (read-only), rw (read-write), overlay (base ro + per-instance overlay) if [ -n "${VOLUME_MOUNTS:-}" ]; then echo "overlay-init: mounting volumes" for vol in $VOLUME_MOUNTS; do @@ -82,14 +83,37 @@ if [ -n "${VOLUME_MOUNTS:-}" ]; then # Create mount point in overlay mkdir -p "/overlay/newroot${path}" - # Mount with appropriate options - # For read-only mounts, use noload to skip journal recovery (safe since no writes will occur) - if [ "$mode" = "ro" ]; then + if [ "$mode" = "overlay" ]; then + # Overlay mode: mount base read-only, create overlayfs with per-instance writable layer + overlay_device=$(echo "$vol" | cut -d: -f4) + + # Create temp mount points for base and overlay disk + base_mount="/mnt/vol-base-$(basename "$path")" + overlay_mount="/mnt/vol-overlay-$(basename "$path")" + mkdir -p "$base_mount" "$overlay_mount" + + # Mount base volume read-only (noload to skip journal recovery) + mount -t ext4 -o ro,noload "$device" "$base_mount" + + # Mount overlay disk (writable) + mount -t ext4 "$overlay_device" "$overlay_mount" + mkdir -p "$overlay_mount/upper" "$overlay_mount/work" + + # Create overlayfs combining base (lower) and instance overlay (upper) + mount -t overlay \ + -o "lowerdir=$base_mount,upperdir=$overlay_mount/upper,workdir=$overlay_mount/work" \ + overlay "/overlay/newroot${path}" + + echo "overlay-init: mounted volume $device at $path (overlay via $overlay_device)" + elif [ "$mode" = "ro" ]; then + # Read-only mount (noload to skip journal recovery for multi-attach safety) mount -t ext4 -o ro,noload "$device" "/overlay/newroot${path}" + echo "overlay-init: mounted volume $device at $path (ro)" else + # Read-write mount mount -t ext4 "$device" "/overlay/newroot${path}" + echo "overlay-init: mounted volume $device at $path (rw)" fi - echo "overlay-init: mounted volume $device at $path ($mode)" done fi diff --git a/openapi.yaml b/openapi.yaml index 33c895b7..bd6ac20e 100644 --- a/openapi.yaml +++ b/openapi.yaml @@ -71,6 +71,14 @@ components: type: boolean description: Whether volume is mounted read-only default: false + overlay: + type: boolean + description: Create per-instance overlay for writes (requires readonly=true) + default: false + overlay_size: + type: string + description: Max overlay size as human-readable string (e.g., "1GB"). Required if overlay=true. + example: "1GB" PortMapping: type: object From 495e5b5b96d6b00386fe471396bc7400f761229e Mon Sep 17 00:00:00 2001 From: Steven Miller Date: Tue, 2 Dec 2025 13:40:45 -0500 Subject: [PATCH 4/6] note about overlays --- lib/volumes/README.md | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/lib/volumes/README.md b/lib/volumes/README.md index b3998e51..10bbb9b9 100644 --- a/lib/volumes/README.md +++ b/lib/volumes/README.md @@ -31,6 +31,16 @@ A single volume can be attached to multiple instances simultaneously if **all** - If all existing attachments are read-only, additional read-only attachments are allowed - Cannot add read-write attachment to a volume with existing attachments +## Overlay Mode + +When attaching a volume with `overlay: true`, the instance gets copy-on-write semantics: +- Base volume is mounted read-only (shared) +- A per-instance overlay disk captures all writes +- Instance sees combined view: base data + its local changes +- Other instances don't see each other's overlay writes (isolated) + +This allows multiple instances to share a common base (e.g., dataset, model weights) while each can make local modifications without affecting others. Requires `readonly: true` and `overlay_size` specifying the max size of per-instance writes. + ## Constraints - Volumes can only be attached at instance creation time (no hot-attach) From 71086cd378cf3e96853656d7d202f0eba7e58ab8 Mon Sep 17 00:00:00 2001 From: Steven Miller Date: Tue, 2 Dec 2025 13:57:35 -0500 Subject: [PATCH 5/6] fix test --- lib/instances/resource_limits_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/instances/resource_limits_test.go b/lib/instances/resource_limits_test.go index fce71951..c2882492 100644 --- a/lib/instances/resource_limits_test.go +++ b/lib/instances/resource_limits_test.go @@ -29,7 +29,7 @@ func TestValidateVolumeAttachments_MaxVolumes(t *testing.T) { err := validateVolumeAttachments(volumes) assert.Error(t, err) - assert.Contains(t, err.Error(), "cannot attach more than 23 volumes") + assert.Contains(t, err.Error(), "cannot attach more than 23") } func TestValidateVolumeAttachments_SystemDirectory(t *testing.T) { From 0b41fe096c3b21415eaa58260055eed1fcb5667b Mon Sep 17 00:00:00 2001 From: Steven Miller Date: Tue, 2 Dec 2025 17:10:05 -0500 Subject: [PATCH 6/6] Address PR review --- lib/instances/resource_limits_test.go | 64 +++++++++++++++++ lib/instances/storage.go | 5 +- lib/instances/volumes_test.go | 85 +++++++++++++++++++++++ lib/system/init_script.go | 5 +- lib/volumes/manager_test.go | 98 +++++++++++++++++++++++++++ 5 files changed, 255 insertions(+), 2 deletions(-) diff --git a/lib/instances/resource_limits_test.go b/lib/instances/resource_limits_test.go index c2882492..ac27b561 100644 --- a/lib/instances/resource_limits_test.go +++ b/lib/instances/resource_limits_test.go @@ -83,6 +83,70 @@ func TestValidateVolumeAttachments_Empty(t *testing.T) { assert.NoError(t, err) } +func TestValidateVolumeAttachments_OverlayRequiresReadonly(t *testing.T) { + // Overlay=true with Readonly=false should fail + volumes := []VolumeAttachment{{ + VolumeID: "vol-1", + MountPath: "/mnt/data", + Readonly: false, // Invalid: overlay requires readonly=true + Overlay: true, + OverlaySize: 100 * 1024 * 1024, + }} + + err := validateVolumeAttachments(volumes) + assert.Error(t, err) + assert.Contains(t, err.Error(), "overlay mode requires readonly=true") +} + +func TestValidateVolumeAttachments_OverlayRequiresSize(t *testing.T) { + // Overlay=true without OverlaySize should fail + volumes := []VolumeAttachment{{ + VolumeID: "vol-1", + MountPath: "/mnt/data", + Readonly: true, + Overlay: true, + OverlaySize: 0, // Invalid: overlay requires size + }} + + err := validateVolumeAttachments(volumes) + assert.Error(t, err) + assert.Contains(t, err.Error(), "overlay_size is required") +} + +func TestValidateVolumeAttachments_OverlayValid(t *testing.T) { + // Valid overlay configuration + volumes := []VolumeAttachment{{ + VolumeID: "vol-1", + MountPath: "/mnt/data", + Readonly: true, + Overlay: true, + OverlaySize: 100 * 1024 * 1024, // 100MB + }} + + err := validateVolumeAttachments(volumes) + assert.NoError(t, err) +} + +func TestValidateVolumeAttachments_OverlayCountsAsTwoDevices(t *testing.T) { + // 12 regular volumes + 12 overlay volumes = 12 + 24 = 36 devices (exceeds 23) + // But let's be more precise: 11 overlay volumes = 22 devices, + 1 regular = 23 (at limit) + // 12 overlay volumes = 24 devices (exceeds limit) + volumes := make([]VolumeAttachment, 12) + for i := range volumes { + volumes[i] = VolumeAttachment{ + VolumeID: "vol-" + string(rune('a'+i)), + MountPath: "/mnt/vol" + string(rune('a'+i)), + Readonly: true, + Overlay: true, + OverlaySize: 100 * 1024 * 1024, + } + } + + err := validateVolumeAttachments(volumes) + assert.Error(t, err) + assert.Contains(t, err.Error(), "cannot attach more than 23") +} + // createTestManager creates a manager with specified limits for testing func createTestManager(t *testing.T, limits ResourceLimits) *manager { t.Helper() diff --git a/lib/instances/storage.go b/lib/instances/storage.go index a0a4aac5..35efee99 100644 --- a/lib/instances/storage.go +++ b/lib/instances/storage.go @@ -87,7 +87,10 @@ func (m *manager) createOverlayDisk(id string, sizeBytes int64) error { return images.CreateEmptyExt4Disk(overlayPath, sizeBytes) } -// createVolumeOverlayDisk creates a sparse overlay disk for a volume attachment +// createVolumeOverlayDisk creates a sparse overlay disk for a volume attachment. +// Cleanup note: If instance creation fails after this point, the overlay disk is +// cleaned up automatically by deleteInstanceData() which removes the entire instance +// directory (including vol-overlays/) via the cleanup stack in createInstance(). func (m *manager) createVolumeOverlayDisk(instanceID, volumeID string, sizeBytes int64) error { // Ensure vol-overlays directory exists overlaysDir := m.paths.InstanceVolumeOverlaysDir(instanceID) diff --git a/lib/instances/volumes_test.go b/lib/instances/volumes_test.go index 118df5f9..9c877352 100644 --- a/lib/instances/volumes_test.go +++ b/lib/instances/volumes_test.go @@ -228,3 +228,88 @@ func TestVolumeMultiAttachReadOnly(t *testing.T) { manager.DeleteInstance(ctx, reader2.Id) volumeManager.DeleteVolume(ctx, vol.Id) } + +// TestOverlayDiskCleanupOnDelete verifies that vol-overlays/ directory is removed +// when an instance with overlay volumes is deleted. +func TestOverlayDiskCleanupOnDelete(t *testing.T) { + // Skip in short mode - this is an integration test + if testing.Short() { + t.Skip("skipping integration test in short mode") + } + + // Require KVM access + if _, err := os.Stat("/dev/kvm"); os.IsNotExist(err) { + t.Skip("/dev/kvm not available - skipping VM test") + } + + manager, tmpDir := setupTestManager(t) + ctx := context.Background() + p := paths.New(tmpDir) + + // Setup: prepare image and system files + imageManager, err := images.NewManager(p, 1) + require.NoError(t, err) + + t.Log("Pulling alpine image...") + _, err = imageManager.CreateImage(ctx, images.CreateImageRequest{ + Name: "docker.io/library/alpine:latest", + }) + require.NoError(t, err) + + for i := 0; i < 60; i++ { + img, err := imageManager.GetImage(ctx, "docker.io/library/alpine:latest") + if err == nil && img.Status == images.StatusReady { + break + } + time.Sleep(1 * time.Second) + } + + systemManager := system.NewManager(p) + err = systemManager.EnsureSystemFiles(ctx) + require.NoError(t, err) + + // Create volume + volumeManager := volumes.NewManager(p, 0) + vol, err := volumeManager.CreateVolume(ctx, volumes.CreateVolumeRequest{ + Name: "cleanup-test-vol", + SizeGb: 1, + }) + require.NoError(t, err) + + // Create instance with overlay volume + inst, err := manager.CreateInstance(ctx, CreateInstanceRequest{ + Name: "overlay-cleanup-test", + Image: "docker.io/library/alpine:latest", + Size: 512 * 1024 * 1024, + HotplugSize: 512 * 1024 * 1024, + OverlaySize: 1024 * 1024 * 1024, + Vcpus: 1, + NetworkEnabled: false, + Volumes: []VolumeAttachment{ + {VolumeID: vol.Id, MountPath: "/data", Readonly: true, Overlay: true, OverlaySize: 100 * 1024 * 1024}, + }, + }) + require.NoError(t, err) + + // Verify vol-overlays directory exists + overlaysDir := p.InstanceVolumeOverlaysDir(inst.Id) + _, err = os.Stat(overlaysDir) + require.NoError(t, err, "vol-overlays directory should exist after instance creation") + + // Verify overlay disk file exists + overlayDisk := p.InstanceVolumeOverlay(inst.Id, vol.Id) + _, err = os.Stat(overlayDisk) + require.NoError(t, err, "overlay disk file should exist after instance creation") + + // Delete the instance + err = manager.DeleteInstance(ctx, inst.Id) + require.NoError(t, err) + + // Verify instance directory is removed (which includes vol-overlays/) + instanceDir := p.InstanceDir(inst.Id) + _, err = os.Stat(instanceDir) + assert.True(t, os.IsNotExist(err), "instance directory should be removed after deletion") + + // Cleanup + volumeManager.DeleteVolume(ctx, vol.Id) +} diff --git a/lib/system/init_script.go b/lib/system/init_script.go index 9b998517..2c6a5e97 100644 --- a/lib/system/init_script.go +++ b/lib/system/init_script.go @@ -87,7 +87,10 @@ if [ -n "${VOLUME_MOUNTS:-}" ]; then # Overlay mode: mount base read-only, create overlayfs with per-instance writable layer overlay_device=$(echo "$vol" | cut -d: -f4) - # Create temp mount points for base and overlay disk + # Create temp mount points for base and overlay disk. + # These persist for the lifetime of the VM but are NOT leaked - they exist inside + # the ephemeral guest rootfs (which is itself an overlayfs) and are destroyed + # when the VM terminates along with all guest state. base_mount="/mnt/vol-base-$(basename "$path")" overlay_mount="/mnt/vol-overlay-$(basename "$path")" mkdir -p "$base_mount" "$overlay_mount" diff --git a/lib/volumes/manager_test.go b/lib/volumes/manager_test.go index 3f78f932..3752e230 100644 --- a/lib/volumes/manager_test.go +++ b/lib/volumes/manager_test.go @@ -2,6 +2,7 @@ package volumes import ( "context" + "fmt" "os" "testing" @@ -292,3 +293,100 @@ func TestDeleteVolume_RejectIfAttached(t *testing.T) { assert.ErrorIs(t, err, ErrInUse) } +func TestMultiAttach_ConcurrentAttachments(t *testing.T) { + manager, _, cleanup := setupTestManager(t) + defer cleanup() + ctx := context.Background() + + // Create a volume + vol, err := manager.CreateVolume(ctx, CreateVolumeRequest{ + Name: "concurrent-vol", + SizeGb: 1, + }) + require.NoError(t, err) + + // Launch multiple goroutines trying to attach simultaneously + const numGoroutines = 10 + results := make(chan error, numGoroutines) + + for i := 0; i < numGoroutines; i++ { + go func(instanceNum int) { + err := manager.AttachVolume(ctx, vol.Id, AttachVolumeRequest{ + InstanceID: fmt.Sprintf("instance-%d", instanceNum), + MountPath: "/data", + Readonly: true, + }) + results <- err + }(i) + } + + // Collect results + var successCount, errorCount int + for i := 0; i < numGoroutines; i++ { + err := <-results + if err == nil { + successCount++ + } else { + errorCount++ + } + } + + // All should succeed since all are read-only + assert.Equal(t, numGoroutines, successCount, "All read-only attachments should succeed") + assert.Equal(t, 0, errorCount, "No errors expected for concurrent read-only attachments") + + // Verify final state has all attachments + vol, err = manager.GetVolume(ctx, vol.Id) + require.NoError(t, err) + assert.Len(t, vol.Attachments, numGoroutines, "Should have all attachments") +} + +func TestMultiAttach_ConcurrentRWConflict(t *testing.T) { + manager, _, cleanup := setupTestManager(t) + defer cleanup() + ctx := context.Background() + + // Create a volume + vol, err := manager.CreateVolume(ctx, CreateVolumeRequest{ + Name: "rw-conflict-vol", + SizeGb: 1, + }) + require.NoError(t, err) + + // Launch multiple goroutines trying to attach read-write simultaneously + const numGoroutines = 5 + results := make(chan error, numGoroutines) + + for i := 0; i < numGoroutines; i++ { + go func(instanceNum int) { + err := manager.AttachVolume(ctx, vol.Id, AttachVolumeRequest{ + InstanceID: fmt.Sprintf("instance-%d", instanceNum), + MountPath: "/data", + Readonly: false, // All trying read-write + }) + results <- err + }(i) + } + + // Collect results + var successCount, errorCount int + for i := 0; i < numGoroutines; i++ { + err := <-results + if err == nil { + successCount++ + } else { + errorCount++ + } + } + + // Only ONE should succeed (first one gets exclusive lock) + assert.Equal(t, 1, successCount, "Exactly one read-write attachment should succeed") + assert.Equal(t, numGoroutines-1, errorCount, "Others should fail due to exclusive lock") + + // Verify final state has exactly one attachment + vol, err = manager.GetVolume(ctx, vol.Id) + require.NoError(t, err) + assert.Len(t, vol.Attachments, 1, "Should have exactly one attachment") + assert.False(t, vol.Attachments[0].Readonly, "Attachment should be read-write") +} +