diff --git a/CHANGELOG.md b/CHANGELOG.md index 5a035665e..e293f92d6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,6 @@ # Changelog +- **Added** Files inside a directory tagged with a `CACHEDIR.TAG` file ([Cache Directory Tagging spec](https://bford.info/cachedir/)) are no longer tracked as inputs automatically, so a task that rewrites a tool's own cache there can still be cached ([#794](https://github.com/voidzero-dev/vite-task/pull/794)). - **Changed** When a task isn't cached because it wrote a file it also read, `vp run --last-details` now says the task read and wrote the file, and shows the `cache: { input, output }` exclusions that let the task be cached ([#784](https://github.com/voidzero-dev/vite-task/pull/784)). - **Changed** The run summary now says a task that wrote a file it also read was `not cached because it modified its inputs`, and the statistics in `vp run --verbose` and `vp run --last-details` use the singular for a count of one, e.g. `1 task • 1 cache miss` ([#783](https://github.com/voidzero-dev/vite-task/pull/783)). - **Fixed** An invalid glob in `--filter` no longer shows its error message twice ([#763](https://github.com/voidzero-dev/vite-task/pull/763)). diff --git a/crates/vt/src/session/execute/cache_update.rs b/crates/vt/src/session/execute/cache_update.rs index ee6a04f6e..db366646d 100644 --- a/crates/vt/src/session/execute/cache_update.rs +++ b/crates/vt/src/session/execute/cache_update.rs @@ -205,6 +205,7 @@ pub(super) async fn update_cache( /// user-configured input negatives, and by tool-reported `ignoreInput` paths. /// `path_writes` is filtered by user-configured output negatives and /// tool-reported `ignoreOutput` paths before read-write overlap detection. +/// `path_reads` also drops paths inside directories tagged with `CACHEDIR.TAG`. fn observe_fspy( #[cfg(fspy)] path_accesses: Option<&fspy::PathAccessIterable>, metadata: &CacheMetadata, @@ -219,6 +220,11 @@ fn observe_fspy( path_accesses.map(|raw| { let tracked = TrackedPathAccesses::from_raw(raw, workspace_root); + // Reads inside directories tagged with `CACHEDIR.TAG` aren't + // inputs, like `ignoreInput`. Writes there stay outputs, so a + // cache hit still restores them (e.g. Cargo tags `target/`). + let tagged_cache_dirs = + super::cachedir_tag::find_tagged_dirs(tracked.path_reads.keys(), workspace_root); let filtered_path_reads: HashMap = // fspy can be attached for auto-output-only tasks. In that // mode reads must not become inferred inputs. @@ -231,6 +237,7 @@ fn observe_fspy( .filter(|(path, _)| { !fspy.input_negative_globs.is_match(path.as_str()) && !is_ignored(path, ignored_input_rels) + && !is_ignored(path, &tagged_cache_dirs) }) .map(|(path, read)| (path.clone(), *read)) .collect() diff --git a/crates/vt/src/session/execute/cachedir_tag.rs b/crates/vt/src/session/execute/cachedir_tag.rs new file mode 100644 index 000000000..7c985f444 --- /dev/null +++ b/crates/vt/src/session/execute/cachedir_tag.rs @@ -0,0 +1,111 @@ +//! Find cache directories marked with a `CACHEDIR.TAG` file, as described by +//! the Cache Directory Tagging spec (). +//! +//! Reads inside a tagged directory are left out of automatic input tracking, +//! the same way as paths a tool reports through `ignoreInput`. +#![cfg(fspy)] + +use std::io::Read as _; + +use rustc_hash::FxHashSet; +use vt_path::{AbsolutePath, RelativePath, RelativePathBuf}; + +const TAG_FILE_NAME: &str = "CACHEDIR.TAG"; +const SIGNATURE: &[u8; 43] = b"Signature: 8a477f597d28d172789f06886806bc55"; + +/// Return the directories that hold a valid `CACHEDIR.TAG` among `paths` and +/// their ancestors. Only directories inside the workspace are considered; the +/// workspace root itself is not. +/// +/// Every directory is checked at most once, so the cost is bounded by the +/// number of distinct directories in `paths`. +pub fn find_tagged_dirs<'a>( + paths: impl IntoIterator, + workspace_root: &AbsolutePath, +) -> FxHashSet { + let mut checked: FxHashSet<&'a RelativePath> = FxHashSet::default(); + let mut tagged = FxHashSet::default(); + for path in paths { + if path.as_path().starts_with("..") { + continue; + } + let mut dir: &'a RelativePath = path; + // A checked directory's ancestors were checked along with it, so the + // walk can stop at the first one already seen. + while !dir.as_str().is_empty() && checked.insert(dir) { + if has_valid_tag(&workspace_root.join(dir)) { + tagged.insert(dir.to_relative_path_buf()); + } + let Some(parent) = dir.parent() else { break }; + dir = parent; + } + } + tagged +} + +/// Whether `dir` contains a regular file named `CACHEDIR.TAG` that starts +/// with the spec's signature. +fn has_valid_tag(dir: &AbsolutePath) -> bool { + let Ok(mut file) = std::fs::File::open(dir.join(TAG_FILE_NAME)) else { + return false; + }; + if !file.metadata().is_ok_and(|metadata| metadata.is_file()) { + return false; + } + let mut header = [0u8; SIGNATURE.len()]; + file.read_exact(&mut header).is_ok() && &header == SIGNATURE +} + +#[cfg(test)] +mod tests { + use vt_path::AbsolutePathBuf; + + use super::*; + + fn rel(path: &str) -> RelativePathBuf { + RelativePathBuf::new(path).unwrap() + } + + fn write(root: &AbsolutePath, path: &str, content: &[u8]) { + let path = root.join(path); + std::fs::create_dir_all(path.parent().unwrap()).unwrap(); + std::fs::write(path, content).unwrap(); + } + + #[test] + fn finds_tagged_ancestors_and_rejects_invalid_tags() { + let temp = tempfile::tempdir().unwrap(); + let root = AbsolutePathBuf::new(temp.path().canonicalize().unwrap()).unwrap(); + + let mut valid_tag = SIGNATURE.to_vec(); + valid_tag.extend_from_slice(b"\n# This file is a cache directory tag.\n"); + write(&root, "valid/CACHEDIR.TAG", &valid_tag); + write(&root, "valid/sub/file.txt", b""); + write(&root, "exact/CACHEDIR.TAG", SIGNATURE); + write( + &root, + "wrong-signature/CACHEDIR.TAG", + b"Signature: 00000000000000000000000000000000", + ); + write(&root, "wrong-signature/file.txt", b""); + write(&root, "truncated/CACHEDIR.TAG", &SIGNATURE[..42]); + write(&root, "truncated/file.txt", b""); + write(&root, "tag-is-dir/CACHEDIR.TAG/file.txt", &valid_tag); + write(&root, "untagged/file.txt", b""); + + let paths = [ + rel("valid/sub/file.txt"), + rel("valid/sub/other.txt"), + rel("exact"), + rel("wrong-signature/file.txt"), + rel("truncated/file.txt"), + rel("tag-is-dir/CACHEDIR.TAG/file.txt"), + rel("untagged/file.txt"), + ]; + let tagged = find_tagged_dirs(&paths, &root); + + let expected: FxHashSet = + [rel("valid"), rel("exact")].into_iter().collect(); + assert_eq!(tagged, expected); + } +} diff --git a/crates/vt/src/session/execute/mod.rs b/crates/vt/src/session/execute/mod.rs index 0426f060f..9148870d9 100644 --- a/crates/vt/src/session/execute/mod.rs +++ b/crates/vt/src/session/execute/mod.rs @@ -1,4 +1,6 @@ mod cache_update; +#[cfg(fspy)] +mod cachedir_tag; pub mod fingerprint; pub mod glob; mod hash; diff --git a/crates/vt_bin/tests/e2e_snapshots/fixtures/cachedir_tag/invalid-signature/CACHEDIR.TAG b/crates/vt_bin/tests/e2e_snapshots/fixtures/cachedir_tag/invalid-signature/CACHEDIR.TAG new file mode 100644 index 000000000..811f183d5 --- /dev/null +++ b/crates/vt_bin/tests/e2e_snapshots/fixtures/cachedir_tag/invalid-signature/CACHEDIR.TAG @@ -0,0 +1 @@ +Signature: 00000000000000000000000000000000 diff --git a/crates/vt_bin/tests/e2e_snapshots/fixtures/cachedir_tag/invalid-signature/state.txt b/crates/vt_bin/tests/e2e_snapshots/fixtures/cachedir_tag/invalid-signature/state.txt new file mode 100644 index 000000000..702f63634 --- /dev/null +++ b/crates/vt_bin/tests/e2e_snapshots/fixtures/cachedir_tag/invalid-signature/state.txt @@ -0,0 +1 @@ +iiiiiiii diff --git a/crates/vt_bin/tests/e2e_snapshots/fixtures/cachedir_tag/package.json b/crates/vt_bin/tests/e2e_snapshots/fixtures/cachedir_tag/package.json new file mode 100644 index 000000000..9a2bc69b9 --- /dev/null +++ b/crates/vt_bin/tests/e2e_snapshots/fixtures/cachedir_tag/package.json @@ -0,0 +1,4 @@ +{ + "name": "cachedir-tag", + "private": true +} diff --git a/crates/vt_bin/tests/e2e_snapshots/fixtures/cachedir_tag/snapshots.toml b/crates/vt_bin/tests/e2e_snapshots/fixtures/cachedir_tag/snapshots.toml new file mode 100644 index 000000000..4fbcdcabf --- /dev/null +++ b/crates/vt_bin/tests/e2e_snapshots/fixtures/cachedir_tag/snapshots.toml @@ -0,0 +1,37 @@ +[[e2e]] +name = "read_write_inside_tagged_dir_is_cached" +comment = """ +A directory holding a valid `CACHEDIR.TAG` is a cache directory. A task that reads and rewrites a file anywhere inside it is cached, and changing that file afterwards doesn't cause a cache miss. +""" +steps = [ + ["vt", "run", "tagged"], + { argv = ["vt", "run", "tagged"], comment = "cache hit" }, + ["vtt", "replace-file-content", "tagged/sub/state.txt", "i", "x"], + { argv = ["vt", "run", "tagged"], comment = "still a cache hit: the file is inside the tagged directory" }, +] + +[[e2e]] +name = "read_write_without_tag_is_not_cached" +comment = """ +Without a `CACHEDIR.TAG`, the same task modifies its inputs and isn't cached. +""" +steps = [["vt", "run", "untagged"], ["vt", "run", "untagged"]] + +[[e2e]] +name = "invalid_tag_is_not_a_cache_dir" +comment = """ +A `CACHEDIR.TAG` with the wrong signature, or a directory named `CACHEDIR.TAG`, doesn't mark a cache directory. +""" +steps = [["vt", "run", "invalid-signature"], ["vt", "run", "tag-is-dir"]] + +[[e2e]] +name = "writes_inside_tagged_dir_are_still_restored" +comment = """ +The tag only affects inputs. A file the task writes inside a tagged directory is still an output, so a cache hit restores it. +""" +steps = [ + ["vt", "run", "tagged-write"], + { argv = ["vtt", "rm", "-rf", "tagged/out"], comment = "delete the output" }, + { argv = ["vt", "run", "tagged-write"], comment = "cache hit, restores the output" }, + { argv = ["vtt", "print-file", "tagged/out/built.txt"], comment = "restored" }, +] diff --git a/crates/vt_bin/tests/e2e_snapshots/fixtures/cachedir_tag/snapshots/invalid_tag_is_not_a_cache_dir.md b/crates/vt_bin/tests/e2e_snapshots/fixtures/cachedir_tag/snapshots/invalid_tag_is_not_a_cache_dir.md new file mode 100644 index 000000000..60fc415a6 --- /dev/null +++ b/crates/vt_bin/tests/e2e_snapshots/fixtures/cachedir_tag/snapshots/invalid_tag_is_not_a_cache_dir.md @@ -0,0 +1,21 @@ +# invalid_tag_is_not_a_cache_dir + +A `CACHEDIR.TAG` with the wrong signature, or a directory named `CACHEDIR.TAG`, doesn't mark a cache directory. + +## `vt run invalid-signature` + +``` +$ vtt replace-file-content invalid-signature/state.txt i ! + +--- +vt run: cachedir-tag#invalid-signature not cached because it modified its inputs. (Run `vt run --last-details` for full details) +``` + +## `vt run tag-is-dir` + +``` +$ vtt replace-file-content tag-is-dir/state.txt i ! + +--- +vt run: cachedir-tag#tag-is-dir not cached because it modified its inputs. (Run `vt run --last-details` for full details) +``` diff --git a/crates/vt_bin/tests/e2e_snapshots/fixtures/cachedir_tag/snapshots/read_write_inside_tagged_dir_is_cached.md b/crates/vt_bin/tests/e2e_snapshots/fixtures/cachedir_tag/snapshots/read_write_inside_tagged_dir_is_cached.md new file mode 100644 index 000000000..e8f91b0a4 --- /dev/null +++ b/crates/vt_bin/tests/e2e_snapshots/fixtures/cachedir_tag/snapshots/read_write_inside_tagged_dir_is_cached.md @@ -0,0 +1,36 @@ +# read_write_inside_tagged_dir_is_cached + +A directory holding a valid `CACHEDIR.TAG` is a cache directory. A task that reads and rewrites a file anywhere inside it is cached, and changing that file afterwards doesn't cause a cache miss. + +## `vt run tagged` + +``` +$ vtt replace-file-content tagged/sub/state.txt i ! +``` + +## `vt run tagged` + +cache hit + +``` +$ vtt replace-file-content tagged/sub/state.txt i ! ◉ cache hit, replaying + +--- +vt run: cache hit. +``` + +## `vtt replace-file-content tagged/sub/state.txt i x` + +``` +``` + +## `vt run tagged` + +still a cache hit: the file is inside the tagged directory + +``` +$ vtt replace-file-content tagged/sub/state.txt i ! ◉ cache hit, replaying + +--- +vt run: cache hit. +``` diff --git a/crates/vt_bin/tests/e2e_snapshots/fixtures/cachedir_tag/snapshots/read_write_without_tag_is_not_cached.md b/crates/vt_bin/tests/e2e_snapshots/fixtures/cachedir_tag/snapshots/read_write_without_tag_is_not_cached.md new file mode 100644 index 000000000..c9815cfe9 --- /dev/null +++ b/crates/vt_bin/tests/e2e_snapshots/fixtures/cachedir_tag/snapshots/read_write_without_tag_is_not_cached.md @@ -0,0 +1,21 @@ +# read_write_without_tag_is_not_cached + +Without a `CACHEDIR.TAG`, the same task modifies its inputs and isn't cached. + +## `vt run untagged` + +``` +$ vtt replace-file-content untagged/state.txt i ! + +--- +vt run: cachedir-tag#untagged not cached because it modified its inputs. (Run `vt run --last-details` for full details) +``` + +## `vt run untagged` + +``` +$ vtt replace-file-content untagged/state.txt i ! + +--- +vt run: cachedir-tag#untagged not cached because it modified its inputs. (Run `vt run --last-details` for full details) +``` diff --git a/crates/vt_bin/tests/e2e_snapshots/fixtures/cachedir_tag/snapshots/writes_inside_tagged_dir_are_still_restored.md b/crates/vt_bin/tests/e2e_snapshots/fixtures/cachedir_tag/snapshots/writes_inside_tagged_dir_are_still_restored.md new file mode 100644 index 000000000..5be9d22d7 --- /dev/null +++ b/crates/vt_bin/tests/e2e_snapshots/fixtures/cachedir_tag/snapshots/writes_inside_tagged_dir_are_still_restored.md @@ -0,0 +1,35 @@ +# writes_inside_tagged_dir_are_still_restored + +The tag only affects inputs. A file the task writes inside a tagged directory is still an output, so a cache hit restores it. + +## `vt run tagged-write` + +``` +$ vtt write-file tagged/out/built.txt built +``` + +## `vtt rm -rf tagged/out` + +delete the output + +``` +``` + +## `vt run tagged-write` + +cache hit, restores the output + +``` +$ vtt write-file tagged/out/built.txt built ◉ cache hit, replaying + +--- +vt run: cache hit. +``` + +## `vtt print-file tagged/out/built.txt` + +restored + +``` +built +``` diff --git a/crates/vt_bin/tests/e2e_snapshots/fixtures/cachedir_tag/tag-is-dir/CACHEDIR.TAG/placeholder.txt b/crates/vt_bin/tests/e2e_snapshots/fixtures/cachedir_tag/tag-is-dir/CACHEDIR.TAG/placeholder.txt new file mode 100644 index 000000000..16ed6b447 --- /dev/null +++ b/crates/vt_bin/tests/e2e_snapshots/fixtures/cachedir_tag/tag-is-dir/CACHEDIR.TAG/placeholder.txt @@ -0,0 +1 @@ +Signature: 8a477f597d28d172789f06886806bc55 diff --git a/crates/vt_bin/tests/e2e_snapshots/fixtures/cachedir_tag/tag-is-dir/state.txt b/crates/vt_bin/tests/e2e_snapshots/fixtures/cachedir_tag/tag-is-dir/state.txt new file mode 100644 index 000000000..702f63634 --- /dev/null +++ b/crates/vt_bin/tests/e2e_snapshots/fixtures/cachedir_tag/tag-is-dir/state.txt @@ -0,0 +1 @@ +iiiiiiii diff --git a/crates/vt_bin/tests/e2e_snapshots/fixtures/cachedir_tag/tagged/CACHEDIR.TAG b/crates/vt_bin/tests/e2e_snapshots/fixtures/cachedir_tag/tagged/CACHEDIR.TAG new file mode 100644 index 000000000..f629c8738 --- /dev/null +++ b/crates/vt_bin/tests/e2e_snapshots/fixtures/cachedir_tag/tagged/CACHEDIR.TAG @@ -0,0 +1,3 @@ +Signature: 8a477f597d28d172789f06886806bc55 +# This file is a cache directory tag. +# For information about cache directory tags, see https://bford.info/cachedir/ diff --git a/crates/vt_bin/tests/e2e_snapshots/fixtures/cachedir_tag/tagged/sub/state.txt b/crates/vt_bin/tests/e2e_snapshots/fixtures/cachedir_tag/tagged/sub/state.txt new file mode 100644 index 000000000..702f63634 --- /dev/null +++ b/crates/vt_bin/tests/e2e_snapshots/fixtures/cachedir_tag/tagged/sub/state.txt @@ -0,0 +1 @@ +iiiiiiii diff --git a/crates/vt_bin/tests/e2e_snapshots/fixtures/cachedir_tag/untagged/state.txt b/crates/vt_bin/tests/e2e_snapshots/fixtures/cachedir_tag/untagged/state.txt new file mode 100644 index 000000000..702f63634 --- /dev/null +++ b/crates/vt_bin/tests/e2e_snapshots/fixtures/cachedir_tag/untagged/state.txt @@ -0,0 +1 @@ +iiiiiiii diff --git a/crates/vt_bin/tests/e2e_snapshots/fixtures/cachedir_tag/vite-task.json b/crates/vt_bin/tests/e2e_snapshots/fixtures/cachedir_tag/vite-task.json new file mode 100644 index 000000000..cee5eeb38 --- /dev/null +++ b/crates/vt_bin/tests/e2e_snapshots/fixtures/cachedir_tag/vite-task.json @@ -0,0 +1,19 @@ +{ + "tasks": { + "tagged": { + "command": "vtt replace-file-content tagged/sub/state.txt i !" + }, + "untagged": { + "command": "vtt replace-file-content untagged/state.txt i !" + }, + "invalid-signature": { + "command": "vtt replace-file-content invalid-signature/state.txt i !" + }, + "tag-is-dir": { + "command": "vtt replace-file-content tag-is-dir/state.txt i !" + }, + "tagged-write": { + "command": "vtt write-file tagged/out/built.txt built" + } + } +} diff --git a/crates/vt_path/src/relative.rs b/crates/vt_path/src/relative.rs index a85db3f37..1b835ba33 100644 --- a/crates/vt_path/src/relative.rs +++ b/crates/vt_path/src/relative.rs @@ -98,6 +98,19 @@ impl RelativePath { // it remains valid UTF-8 and contains no backslash separators. Some(unsafe { Self::assume_portable(stripped_path.to_str().unwrap()) }) } + + /// Returns the path without its final component, or [`None`] if the path + /// is empty. The parent of a single-component path is the empty path. + #[must_use] + pub fn parent(&self) -> Option<&Self> { + if self.0.is_empty() { + return None; + } + let parent = self.0.rfind('/').map_or("", |index| &self.0[..index]); + // SAFETY: A prefix of a portable path that ends before a `/` separator + // is still portable. + Some(unsafe { Self::assume_portable(parent) }) + } } /// A owned relative path buf with the same guarantees as `RelativePath` @@ -449,6 +462,15 @@ mod tests { assert_eq!(joined_path.as_str(), "baz"); } + #[test] + fn parent() { + let rel_path = RelativePathBuf::new("foo/bar/baz").unwrap(); + let parents: Vec<&str> = std::iter::successors(rel_path.parent(), |p| p.parent()) + .map(RelativePath::as_str) + .collect(); + assert_eq!(parents, ["foo/bar", "foo", ""]); + } + #[test] fn clean() { let rel_path = RelativePathBuf::new("../foo/../bar").unwrap(); diff --git a/docs/inputs.md b/docs/inputs.md index d9f48b807..df68581f1 100644 --- a/docs/inputs.md +++ b/docs/inputs.md @@ -248,6 +248,18 @@ If the file is temporary or shouldn't affect caching, exclude it from both `inpu For a file outside the package directory, use the object form with `"base": "workspace"`. +### Cache Directories (`CACHEDIR.TAG`) + +A directory that contains a `CACHEDIR.TAG` file following the [Cache Directory Tagging spec](https://bford.info/cachedir/) is treated as a cache directory. Reads of files anywhere inside it are left out of automatic input tracking, so a tool that reads and rewrites its own cache on every run (such as a test runner's results cache) doesn't stop the task from being cached. Files the task writes there are still outputs and are restored on a cache hit. Cargo already writes this tag in `target/`. + +The tag must be a regular file whose content starts with exactly: + +```text +Signature: 8a477f597d28d172789f06886806bc55 +``` + +Only directories inside the workspace are checked. Explicit `input` globs still match files inside a tagged directory. + ### Cache Disabled `input` is part of the `cache` object, so it cannot be used when caching is disabled with `cache: false`.