chore: refactor config file structure and layout (#54)

Changes:
- Consolidates the configuration and lock file into a single file.
- Update design doc to match new config file design
- Refactored a lot of code and restructured `config.rs` to be more readable, otherwise it would have been a massive single mod.

Author: ptsouchlos

.embd/config.toml

@@ -0,0 +1,14 @@
+[infra.metadata]
+remote = "git@github.com:ptsouchlos/infra.git"
+commit_hash = "79932afc417d21900038dcd8f8154c3661c15f76"
+folder = "infra"
+allow_untracked = false
+include = [
+    "just/**",
+    "rust/**",
+]
+
+[infra.files]
+"just/shells.just" = "sha256:ca031066da941b192b18859a4c8ebda1ecfe91ff14f57fe31489c6e83cb3bfa7"
+"rust/just/clippy.just" = "sha256:5ce758826bbd8f4f3ffd626792edb06ab1fe41c591f7cd099c59336c46a1081f"
+"rust/just/format.just" = "sha256:cb2e821139f1e314f5b87e4282bb75524d9d90cdd0dee1eaa524b8fa4b84f762"

.embd/embd.lock

@@ -1,5 +1,5 @@
 # Import shared just recipes
-import "infra/rust/just/shells.just"
+import "infra/just/shells.just"
 import "infra/rust/just/clippy.just"
 import "infra/rust/just/format.just"
 

.embd/embd.toml

@@ -29,7 +29,7 @@ cargo install --path .
 embd add -l <repo link>.git -f <local folder>
 ```
 
-This will clone the repository to the given folder and create a `config.toml` and `embd.lock` file in the `.embd` folder. These files should be commited to VCS. You can also filter the contents of the repository using `-i` and `-e` flags. These use glob filters to include or exclude certain folders or files. For example, to exclude all Markdown and include all text files, you could the following:
+This will clone the repository to the given folder and create an `embd.toml` file in the `.embd` folder. This file tracks both the pinned remote/commit for each embed and the hashes of the files it manages, and should be commited to VCS. You can also filter the contents of the repository using `-i` and `-e` flags. These use glob filters to include or exclude certain folders or files. For example, to exclude all Markdown and include all text files, you could the following:
 
 ```bash
 embd add -l <repo> -f <folder> -i "**.txt" -e "**.md"
@@ -43,7 +43,7 @@ To update all embedded projects, run:
 embd update
 ```
 
-To update a specific project, use the name of the repo. This corresponds to the key of the projects entry in the `.embd/config.toml` file:
+To update a specific project, use the name of the repo. This corresponds to the key of the projects entry in the `.embd/embd.toml` file:
 
 ```bash
 embd update infra
@@ -55,7 +55,7 @@ To update a project to a new commit, tag or branch, use the `-r` or `--rev` flag
 embd update infra --rev abcd1234
 ```
 
-This will update the files on disk and update the commit hash tracked in the config file and lock file. In addition, the entries in the lock file for the given project will also be updated. Updates can also be forced using `--force` and untracked files can be removed using `--overwrite`. See `embd update -h` for more details.
+This will update the files on disk and update the commit hash and file manifest tracked for that project in `embd.toml`. Updates can also be forced using `--force` and untracked files can be removed using `--overwrite`. See `embd update -h` for more details.
 
 ### Check Status of Embeds
 

.justfile

@@ -1,16 +1,16 @@
 # Design of `embd`
 
-The basic goal of `embd` is to provide a simple way to "embed" one git repository into another. The prime source of inspiration for this project is the [beman-submodule](https://github.com/bemanproject/beman-submodule) tool, which is implemented in Python and is available on PyPI. `embd`, on the other hand, is implemented in Rust and will be available as a static binary which provides many options for distibution and installation.
+The basic goal of `embd` is to provide a simple way to "embed" one git repository into another. The prime source of inspiration for this project is the [beman-submodule](https://github.com/bemanproject/beman-submodule) tool, which is implemented in Python and is available on PyPI. `embd`, on the other hand, is implemented in Rust and will be available as a static binary which provides many options for distribution and installation.
 
-An additional goal of `embd` is to be more general purpose than `beman-submodule` which is more closely tied to the goals and setup that the various `Beman` projects use.
+An additional goal of `embd` is to be more general purpose than `beman-submodule`, which is more closely tied to the goals and setup that the various `Beman` projects use.
 
 ## Why?
 
-Both `beman-submodule` and `embd` aim to be alternatives to either [git submodules](https://git-scm.com/book/en/v2/Git-Tools-Submodules) and [git subtree](https://www.atlassian.com/git/tutorials/git-subtree). [Eddie Nolan](https://www.ednolan.com) put together a set of [slides](https://www.ednolan.com/toolchains_slides.pdf) to show the differences and limitations of both `git subtree` and `git submodule`, so I won't go into that too much here. The gist is that `git submodule` provides a subpar user experience (users have to remember to run `git submodule update --init`) and `git subtree`s force merge commits in your history.
+Both `beman-submodule` and `embd` aim to be alternatives to either [git submodules](https://git-scm.com/book/en/v2/Git-Tools-Submodules) or [git subtree](https://www.atlassian.com/git/tutorials/git-subtree). [Eddie Nolan](https://www.ednolan.com) put together a set of [slides](https://www.ednolan.com/toolchains_slides.pdf) to show the differences and limitations of both `git subtree` and `git submodule`, so I won't go into that too much here. The gist is that `git submodule` provides a subpar user experience (users have to remember to run `git submodule update --init`) and `git subtree`s force merge commits in your history.
 
 ## What's the solution?
 
-The general idea is to pull the dependant repository as source into the parent repository. This puts the burden of keeping dependencies up to date on the maintainer. `embd` helps in making that process easier for the maintainer.
+The general idea is to pull the dependent repository as source into the parent repository. This puts the burden of keeping dependencies up to date on the maintainer. `embd` helps in making that process easier for the maintainer.
 
 ## Design
 
@@ -22,22 +22,26 @@ The general idea is to pull the dependant repository as source into the parent r
 | `update` | Update all embeds to match the config file. |                                          `rev`: Specify the commit, tag or branch to advance to \n `force`: Overwrite any local modification \n `overwrite`: Delete all untracked files (requries `force`) \n `quiet`: Print summaries, not each file updates.                                          |
 | `status` |       Show the status of all embeds.        |                                                                                                                           `quiet`: Should summaries per embed, not per file.                                                                                                                            |
 
-### Config File
+### Configuration File
 
-The configuration file describes all the embedded sources.
+`embd` keeps a single consolidated file, `.embd/embd.toml`, describing all the embedded sources. Each entry has a `metadata` table which includes data like the remote repository URL, commit, folder, and include/exclude filters. The config entry also includes a `files` table of sha256 hashes per tracked file. This is used to detect local drift without having to re-pull or clone files from the remote.
 
 ```toml
-[repo1]
-remote="https://example.git"
-commit_hash=123abcd1234
-folder=/example
-
-[repo2]
-remote="https://example2.git"
-commit_hash=123abcd1234
-folder=/example2
+[repo1.metadata]
+remote = "https://example.git"
+commit_hash = "123abcd1234"
+folder = "example"
+allow_untracked = false
+
+[repo1.files]
+"a.txt" = "sha256:..."
+
+[repo2.metadata]
+remote = "https://example2.git"
+commit_hash = "123abcd1234"
+folder = "example2"
+allow_untracked = false
+
+[repo2.files]
+"b.txt" = "sha256:..."
 ```
-
-### Lockfile
-
-In addition to the config file, `embd` maintains a lockfile to check for differences in the local files and changes to the subfolder an embed is put into. This serves as an easy way to verify file status without pull the files again from the remote location.

README.md

@@ -4,9 +4,8 @@ use anyhow::{Context, Result, bail};
 use tempfile::tempdir;
 
 use crate::color;
-use crate::config::{self, EmbdEntry};
+use crate::config::{self, EmbdEntry, FileLocks, Metadata};
 use crate::filter::Filter;
-use crate::lockfile::{self, LockEntry};
 use crate::{filesystem, git, paths};
 
 /// Input arguments for the `add` command.
@@ -101,38 +100,34 @@ pub(crate) fn execute(args: AddArgs) -> Result<()> {
     // so we never leave on-disk state without a matching config entry.
     filesystem::copy_dir(tmp_dir.path(), &folder_abs, &filter)?;
 
-    let lock_path = paths::lock_path(&root);
-
     let result = (|| -> Result<()> {
-        let lock_entry = LockEntry::build_from_path(&folder_abs, commit_hash.clone())?;
-        let mut lock = lockfile::Lockfile::load_or_default(&lock_path)?;
-        lock.upsert(repo_name.clone(), lock_entry);
-        lock.save(&lock_path)?;
+        // Hash the destination folder as-copied (not just what the filter
+        // allowed) so a pre-existing --allow-untracked folder's contents are
+        // reflected too, matching what `copy_dir` actually left on disk.
+        let files = FileLocks::build_from_path(&folder_abs)?;
         config.insert(
             repo_name.clone(),
             EmbdEntry {
-                remote: link,
-                commit_hash,
-                folder: folder_rel,
-                allow_untracked: args.allow_untracked,
-                include,
-                exclude,
+                metadata: Metadata {
+                    remote: link,
+                    commit_hash,
+                    folder: folder_rel,
+                    allow_untracked: args.allow_untracked,
+                    include,
+                    exclude,
+                },
+                files,
             },
         )?;
         config.save(&config_path)?;
         Ok(())
     })();
 
     if let Err(e) = result {
-        // If there was an error, try to rollback to the previous state.
+        // If there was an error, try to rollback to the previous state. Nothing
+        // is written to config.toml unless the whole closure above succeeds, so
+        // there's no separate on-disk manifest entry to clean up here.
         rollback(&folder_abs, folder_existed, args.allow_untracked);
-        // Drop the entry we may have written to the lock file, leaving any other
-        // entries untouched.
-        if let Ok(mut lock) = lockfile::Lockfile::load(&lock_path)
-            && lock.remove(&repo_name).is_some()
-        {
-            let _ = lock.save(&lock_path);
-        }
         // Propagate the error up to the caller
         return Err(e);
     }

docs/design.md

@@ -2,8 +2,7 @@ use anyhow::Result;
 
 use crate::color;
 use crate::commands::common::select_entries;
-use crate::config;
-use crate::lockfile::{self, EntryReport, EntryState, FileChange, Lockfile};
+use crate::config::{self, EntryReport, EntryState, FileChange};
 use crate::paths;
 
 #[derive(clap::Args, Debug)]
@@ -19,12 +18,11 @@ pub(crate) fn execute(args: StatusArgs) -> Result<()> {
     let root = paths::find_git_root()?;
     let config_path = paths::config_path(&root);
     let config = config::load_or_default(&config_path)?;
-    let lock = Lockfile::load_or_default(&paths::lock_path(&root))?;
     let selected = select_entries(&config, &args.names)?;
 
     let mut any_drift = false;
     for (name, entry) in selected {
-        let report = lockfile::inspect_entry(&root, name, entry, lock.get(name));
+        let report = config::inspect_entry(&root, name, entry);
         any_drift |= report.has_drift();
         print_report(&report, args.quiet);
     }
@@ -45,16 +43,10 @@ pub(crate) fn print_report(report: &EntryReport, quiet: bool) {
             println!("{header}: {}", color::bad("folder missing"));
             return;
         }
-        EntryState::Missing => {
-            println!("{header}: {}", color::bad("missing from lock file"));
-            return;
-        }
         EntryState::Compared => {}
     }
 
-    let summary = if let Some((local, wanted)) = &report.stale {
-        color::warn(&format!("stale (folder at {local}, config wants {wanted})"))
-    } else if report.changes.is_empty() {
+    let summary = if report.changes.is_empty() {
         color::ok("clean")
     } else {
         color::warn("drift")