compose: retained named volumes and strict reconciliation

README.md

@@ -80,7 +80,7 @@ hypeman rm --force --all
 
 ### Compose
 
-`hypeman compose` applies a small declarative workload file for images or Dockerfiles, instances, restart/health settings, and ingresses. See [lib/compose/README.md](lib/compose/README.md#compose).
+`hypeman compose` applies a small declarative workload file for images or Dockerfiles, retained volumes, instances, restart/health settings, and ingresses. See [lib/compose/README.md](lib/compose/README.md#compose).
 
 More ingress features:
 - Automatic certs

lib/compose/README.md

@@ -86,20 +86,28 @@ hypeman compose down -f hypeman.compose.yaml
 
 If a managed instance or ingress exists but the rendered spec changed, `up` reports that replacement is required and exits without changing resources. Re-run with `--replace` to recreate changed resources.
 
+Retained volumes are never deleted by `up` or `down`. Passing `--volumes` to `down` also deletes the volumes owned by the file and **destroys their data**:
+
+```sh
+hypeman compose down -f hypeman.compose.yaml --volumes
+```
+
 All compose commands honor global output flags such as `--format json`, `--format yaml`, and `--transform`.
 
 ### How It Works
 
-`plan` renders the desired resources from the compose file, checks whether referenced images exist, then compares the desired instances and ingresses against existing resources.
+`plan` renders the desired resources from the compose file, checks whether referenced images exist, then compares the desired volumes, instances, and ingresses against existing resources. Owned instances and ingresses that are no longer declared in the file are planned for deletion (pruning); resources without compose ownership tags are never touched.
 
 `up` applies the plan in order:
 
 1. build Dockerfile services whose generated images are missing
 2. ensure referenced images exist and are ready
-3. create or replace instances
-4. create or replace ingresses
+3. create declared volumes
+4. delete owned instances and ingresses that are no longer declared (pruning frees unique keys such as ingress hostnames before they are reused)
+5. create or replace instances
+6. create or replace ingresses
 
-`down` deletes only instances and ingresses tagged as owned by the compose file. Images are left in place because they can be shared by normal `hypeman run` usage or other compose files.
+`down` deletes only instances and ingresses tagged as owned by the compose file. Volumes owned by the file are retained and reported as skipped unless `--volumes` is passed. Images are left in place because they can be shared by normal `hypeman run` usage or other compose files.
 
 Instances and ingresses get compose ownership tags:
 
@@ -110,8 +118,49 @@ hypeman.compose.resource
 hypeman.compose.hash
 ```
 
+Volumes get the same ownership tags except `hypeman.compose.service`, because a volume can be shared by multiple services.
+
 The hash is computed from the rendered resource spec before ownership tags are added. Re-running the same file is idempotent: matching resources are reported as unchanged, changed managed resources require `--replace`, and unmanaged resources with the same name are reported as conflicts.
 
+### Retained Volumes
+
+Top-level `volumes` declare named volumes backed by the Hypeman volume API. Services attach them with `volumes` mount declarations:
+
+```yaml
+version: 1
+name: stateful
+
+volumes:
+  data:
+    size_gb: 10
+  logs:
+    name: stateful-logs-explicit # optional explicit name
+    size_gb: 1
+
+services:
+  db:
+    image: postgres:16
+    volumes:
+      - data:/var/lib/postgresql/data        # shorthand: volume:/abs/path[:ro|rw]
+      - volume: logs                          # mapping form
+        mount_path: /var/log/db
+        readonly: true
+```
+
+By default, volumes are named `<compose name>-<volume key>`; set `name` for a stable external name, just like services and ingresses.
+
+Volumes are created before the instances that mount them and are **retained**: instance replacement (via `--replace`), `compose down`, and pruning never delete them, and `compose up` on an existing volume reports it as unchanged. Deleting a retained volume requires the explicit destructive option `hypeman compose down --volumes`.
+
+Volumes are immutable once created. Changing a declared volume (for example `size_gb`) makes `plan` report a conflict and blocks `up`, rather than silently recreating the volume and losing data. To resize, restore the original spec or delete the volume explicitly with `compose down --volumes`.
+
+If instance replacement fails after the old instance was deleted, the retained volume and its data are untouched; re-running `compose up --replace` recreates the instance on the same volume.
+
+Mount declarations are validated strictly: the referenced volume must be declared, the mount path must be absolute, and duplicate mount paths or mounting the same volume twice in one service are rejected.
+
+### Strict Parsing
+
+Compose files are parsed strictly: unknown fields and duplicate keys fail validation before any resource is applied. This applies at every level, including volume mount mappings.
+
 ### Interpolation
 
 String values can embed local files or environment variables:

lib/compose/compose.go

@@ -14,13 +14,18 @@ const (
 	composeResourceInstance = "instance"
 	composeResourceIngress  = "ingress"
 	composeResourceBuild    = "build"
+	composeResourceVolume   = "volume"
 )
 
 type Runner struct {
 	file   string
 	spec   composeSpec
 	client hypeman.Client
 	opts   []option.RequestOption
+
+	// volumeIDsByName caches the compose volume name→ID lookup for a single
+	// Up apply pass so each instance create doesn't re-list volumes.
+	volumeIDsByName map[string]string
 }
 
 type UpOptions struct {
@@ -30,6 +35,13 @@ type UpOptions struct {
 	Verbose     bool
 }
 
+type DownOptions struct {
+	Verbose bool
+	// Volumes also deletes retained volumes owned by the compose file.
+	// This destroys their data and cannot be undone.
+	Volumes bool
+}
+
 type Plan struct {
 	Name    string   `json:"name"`
 	File    string   `json:"file"`
@@ -53,11 +65,17 @@ type Action struct {
 	Service string `json:"service,omitempty"`
 	Reason  string `json:"reason"`
 
-	instanceID    string
-	ingressID     string
-	instanceInput hypeman.InstanceNewParams
-	ingressInput  hypeman.IngressNewParams
-	buildInput    *desiredBuild
+	instanceID string
+	ingressID  string
+	volumeID   string
+	// claimedIngressIDs lists additional owned ingress IDs this action claims
+	// (e.g. ambiguous rename candidates under a conflict) so prune planning
+	// does not also propose deleting them.
+	claimedIngressIDs []string
+	instanceInput     hypeman.InstanceNewParams
+	ingressInput      hypeman.IngressNewParams
+	volumeInput       hypeman.VolumeNewParams
+	buildInput        *desiredBuild
 }
 
 func NewRunner(file string, client hypeman.Client, opts ...option.RequestOption) (*Runner, error) {

lib/compose/compose_test.go

@@ -88,7 +88,7 @@ func TestBuildComposeInstanceInputIncludesPolicyFields(t *testing.T) {
 		},
 	}
 
-	input := buildComposeInstanceInput("hypeship-otel-otelcol", service)
+	input := buildComposeInstanceInput("hypeship-otel-otelcol", service, nil)
 	inputJSON := map[string]any{}
 	inputData, err := json.Marshal(input)
 	require.NoError(t, err)
@@ -138,7 +138,7 @@ func TestDesiredResourcesUseDeterministicNamesAndTags(t *testing.T) {
 		},
 	}
 
-	_, instances, ingresses, images, err := runner.desiredResources()
+	_, _, instances, ingresses, images, err := runner.desiredResources()
 	require.NoError(t, err)
 
 	require.Equal(t, []string{"otel/opentelemetry-collector-contrib:0.108.0"}, images)
@@ -177,7 +177,7 @@ func TestDesiredResourcesUseExplicitResourceNames(t *testing.T) {
 		},
 	}
 
-	_, instances, ingresses, _, err := runner.desiredResources()
+	_, _, instances, ingresses, _, err := runner.desiredResources()
 	require.NoError(t, err)
 
 	require.Len(t, instances, 1)
@@ -270,7 +270,7 @@ services:
 	require.NoError(t, err)
 
 	runner := Runner{file: composePath, spec: spec}
-	builds, instances, _, images, err := runner.desiredResources()
+	builds, _, instances, _, images, err := runner.desiredResources()
 	require.NoError(t, err)
 
 	require.Empty(t, images)
@@ -280,18 +280,18 @@ services:
 	assert.Regexp(t, `^compose/worker-stack/worker:[a-f0-9]{12}$`, builds[0].Image)
 	assert.Equal(t, builds[0].Image, instances[0].Input.Image)
 
-	again, _, _, _, err := runner.desiredResources()
+	again, _, _, _, _, err := runner.desiredResources()
 	require.NoError(t, err)
 	require.Equal(t, builds[0].Image, again[0].Image)
 
 	require.NoError(t, os.WriteFile(filepath.Join(dir, ".dockerignore"), []byte("*.tmp\n"), 0644))
-	dockerignoreChanged, _, _, _, err := runner.desiredResources()
+	dockerignoreChanged, _, _, _, _, err := runner.desiredResources()
 	require.NoError(t, err)
 	require.NotEqual(t, builds[0].Image, dockerignoreChanged[0].Image)
 
 	require.NoError(t, os.Remove(filepath.Join(dir, ".dockerignore")))
 	require.NoError(t, os.WriteFile(filepath.Join(dir, "worker"), []byte("echo changed\n"), 0644))
-	changed, _, _, _, err := runner.desiredResources()
+	changed, _, _, _, _, err := runner.desiredResources()
 	require.NoError(t, err)
 	require.NotEqual(t, builds[0].Image, changed[0].Image)
 }
@@ -475,4 +475,22 @@ func TestPlanIngressActionConflictsWhenRenameCandidateIsAmbiguous(t *testing.T)
 	assert.Equal(t, "conflict", action.Action)
 	assert.Equal(t, "multiple owned ingresses for service have changed names", action.Reason)
 	assert.Empty(t, action.ingressID)
+	// The conflict still claims both candidates so prune planning does not
+	// also propose deleting them.
+	assert.ElementsMatch(t, []string{"old-http-id", "old-grpc-id"}, action.claimedIngressIDs)
+}
+
+func TestPruneActionsSkipsIngressesClaimedByConflict(t *testing.T) {
+	owned := []hypeman.Ingress{
+		{ID: "old-http-id", Name: "app-api-http", Tags: composeTags("app", "api", composeResourceIngress, "http-hash")},
+		{ID: "old-grpc-id", Name: "app-api-grpc", Tags: composeTags("app", "api", composeResourceIngress, "grpc-hash")},
+	}
+	actions := []Action{{
+		Action:            "conflict",
+		Type:              "ingress",
+		Name:              "app-api-public",
+		claimedIngressIDs: []string{"old-http-id", "old-grpc-id"},
+	}}
+
+	assert.Empty(t, pruneActions(nil, owned, actions))
 }

lib/compose/desired.go

@@ -24,7 +24,13 @@ type desiredIngress struct {
 	Input   hypeman.IngressNewParams
 }
 
-func (r *Runner) desiredResources() ([]desiredBuild, []desiredInstance, []desiredIngress, []string, error) {
+type desiredVolume struct {
+	Name  string
+	Hash  string
+	Input hypeman.VolumeNewParams
+}
+
+func (r *Runner) desiredResources() ([]desiredBuild, []desiredVolume, []desiredInstance, []desiredIngress, []string, error) {
 	serviceNames := make([]string, 0, len(r.spec.Services))
 	imageSet := map[string]struct{}{}
 	for name, service := range r.spec.Services {
@@ -41,6 +47,33 @@ func (r *Runner) desiredResources() ([]desiredBuild, []desiredInstance, []desire
 	}
 	sort.Strings(images)
 
+	volumeKeys := make([]string, 0, len(r.spec.Volumes))
+	for key := range r.spec.Volumes {
+		volumeKeys = append(volumeKeys, key)
+	}
+	sort.Strings(volumeKeys)
+	volumeNames := make(map[string]string, len(volumeKeys))
+	volumes := make([]desiredVolume, 0, len(volumeKeys))
+	for _, key := range volumeKeys {
+		volumeSpec := r.spec.Volumes[key]
+		volumeName := composeVolumeName(r.spec.Name, key, volumeSpec)
+		volumeNames[key] = volumeName
+		volumeInput := hypeman.VolumeNewParams{
+			Name:   volumeName,
+			SizeGB: volumeSpec.SizeGB,
+		}
+		volumeHash, err := shortHash(volumeInput)
+		if err != nil {
+			return nil, nil, nil, nil, nil, err
+		}
+		volumeInput.Tags = composeVolumeTags(r.spec.Name, volumeHash)
+		volumes = append(volumes, desiredVolume{
+			Name:  volumeName,
+			Hash:  volumeHash,
+			Input: volumeInput,
+		})
+	}
+
 	var builds []desiredBuild
 	instances := make([]desiredInstance, 0, len(serviceNames))
 	var ingresses []desiredIngress
@@ -49,16 +82,16 @@ func (r *Runner) desiredResources() ([]desiredBuild, []desiredInstance, []desire
 		if service.Dockerfile != "" {
 			build, err := r.desiredBuildForService(serviceName, service)
 			if err != nil {
-				return nil, nil, nil, nil, err
+				return nil, nil, nil, nil, nil, err
 			}
 			builds = append(builds, build)
 			service.Image = build.Image
 		}
 		instanceName := composeInstanceName(r.spec.Name, serviceName, service)
-		instanceInput := buildComposeInstanceInput(instanceName, service)
+		instanceInput := buildComposeInstanceInput(instanceName, service, volumeNames)
 		instanceHash, err := shortHash(instanceInput)
 		if err != nil {
-			return nil, nil, nil, nil, err
+			return nil, nil, nil, nil, nil, err
 		}
 		instanceInput.Tags = composeTags(r.spec.Name, serviceName, composeResourceInstance, instanceHash)
 		instances = append(instances, desiredInstance{
@@ -73,7 +106,7 @@ func (r *Runner) desiredResources() ([]desiredBuild, []desiredInstance, []desire
 			ingressInput := buildComposeIngressInput(instanceName, ingressName, ingressSpec)
 			ingressHash, err := shortHash(ingressInput)
 			if err != nil {
-				return nil, nil, nil, nil, err
+				return nil, nil, nil, nil, nil, err
 			}
 			ingressInput.Tags = composeTags(r.spec.Name, serviceName, composeResourceIngress, ingressHash)
 			ingresses = append(ingresses, desiredIngress{
@@ -84,14 +117,33 @@ func (r *Runner) desiredResources() ([]desiredBuild, []desiredInstance, []desire
 			})
 		}
 	}
-	return builds, instances, ingresses, images, nil
+	return builds, volumes, instances, ingresses, images, nil
 }
 
-func buildComposeInstanceInput(instanceName string, service composeServiceSpec) hypeman.InstanceNewParams {
+// buildComposeInstanceInput renders the instance create params for a service.
+// volumeNames maps compose volume keys to their resolved Hypeman volume names.
+// Mounts carry the volume *name* (not the server-assigned ID) so the rendered
+// hash stays stable across volume creation; names are resolved to IDs at apply
+// time, immediately before instance creation.
+func buildComposeInstanceInput(instanceName string, service composeServiceSpec, volumeNames map[string]string) hypeman.InstanceNewParams {
 	input := hypeman.InstanceNewParams{
 		Name:  instanceName,
 		Image: service.Image,
 	}
+	if len(service.Volumes) > 0 {
+		mounts := make([]hypeman.VolumeMountParam, 0, len(service.Volumes))
+		for _, mount := range service.Volumes {
+			volumeMount := hypeman.VolumeMountParam{
+				VolumeID:  volumeNames[mount.Volume],
+				MountPath: mount.MountPath,
+			}
+			if mount.Readonly {
+				volumeMount.Readonly = hypeman.Bool(true)
+			}
+			mounts = append(mounts, volumeMount)
+		}
+		input.Volumes = mounts
+	}
 	if len(service.Entrypoint) > 0 {
 		input.Entrypoint = service.Entrypoint
 	}
@@ -233,6 +285,13 @@ func composeIngressName(composeName, serviceName string, index int, ingress comp
 	return fmt.Sprintf("%s-%s-%d", composeName, serviceName, index)
 }
 
+func composeVolumeName(composeName, key string, volume composeVolumeSpec) string {
+	if volume.Name != "" {
+		return volume.Name
+	}
+	return composeName + "-" + key
+}
+
 func composeTags(composeName, serviceName, resource, hash string) map[string]string {
 	return map[string]string{
 		composeTagName:     composeName,
@@ -242,6 +301,16 @@ func composeTags(composeName, serviceName, resource, hash string) map[string]str
 	}
 }
 
+// composeVolumeTags tags a retained volume with compose ownership. Volumes are
+// shared across services, so they carry no service tag.
+func composeVolumeTags(composeName, hash string) map[string]string {
+	return map[string]string{
+		composeTagName:     composeName,
+		composeTagResource: composeResourceVolume,
+		composeTagHash:     hash,
+	}
+}
+
 func shortHash(v any) (string, error) {
 	data, err := json.Marshal(v)
 	if err != nil {