diff --git a/README.md b/README.md index 0ba469b..b84da44 100644 --- a/README.md +++ b/README.md @@ -56,7 +56,7 @@ See the [configuration section](#configuration) for more options. #### Start an instance ```bash -haze start [--name ] [database] [php-version] [services] [vX.Y.Z] +haze start [--name ] [--detach] [database] [php-version] [services] [vX.Y.Z] ``` Where `database` is one of `sqlite`, `mysql`, `mariadb`, `pgsql` or `oracle` @@ -70,6 +70,10 @@ release instead of using the local sources. Use `--name ` to give the instance a specific name instead of a randomly generated one. For example: +Use `--detach` to give the instance +[its own sources](#instances-with-their-own-sources) instead of sharing +`sources_root` with all other instances. + Additionally, you can use the following options when starting an instance: - `s3`: set up an S3 server and configure to Nextcloud to use it as primary @@ -134,6 +138,24 @@ haze list haze clean ``` +### Instances with their own sources + +By default every instance shares the sources configured as `sources_root`, so +all instances always run the same code. An instance started with `--detach` gets +its own `git worktree` of `sources_root` instead, created in `worktree_dir`, +which lets you run several instances on different branches at the same time. + +```bash +haze start --name my-fix --detach +``` + +The worktree is checked out with a detached head. Every app in one of the +`app_directories` that gets enabled during setup receives its own worktree as +well, other apps keep running from the shared app directory. + +The worktrees are removed together with the instance, by `haze stop` or +`haze clean`. + ## Controlling running instances The following commands run against the most recently started instance and allow @@ -358,6 +380,7 @@ options sources_root = "/path/to/sources" # path of the nextcloud sources. required app_directories = ["/path/to/sources/more_app"] # paths to additional app directories. work_dir = "/path/to/temp/dir" # path to temporary directory. optional, defaults to "/tmp/haze" +worktree_dir = "/path/to/worktrees" # where to create the worktrees of detached instances. optional, defaults to "/worktrees" [auto_setup] # optional enabled = false # whether or not to automatically install nextcloud on `haze start`. enabled by default diff --git a/nix/image/bootstrap b/nix/image/bootstrap index 2c9dcff..1c675f5 100755 --- a/nix/image/bootstrap +++ b/nix/image/bootstrap @@ -20,9 +20,26 @@ shadow-setup echo $"Running as ($HAZE_UID):($HAZE_GID)" -mkdir /var/www/html/core/skeleton /var/www/html/build/integration/vendor /var/www/html/build/integration/output /var/www/html/build/integration/work /var/www/html/core/skeleton /var/www/.composer/cache /var/www/html/apps/spreed/tests/integration/vendor/composer +mkdir /var/www/html/core/skeleton \ + /var/www/html/build/integration/vendor \ + /var/www/html/build/integration/output \ + /var/www/html/build/integration/work \ + /var/www/html/core/skeleton \ + /var/www/.composer/cache \ + /var/www/html/apps/spreed/tests/integration/vendor/composer chown -R $"($HAZE_UID):($HAZE_GID)" /var/www/html/data /var/www/html/config -chown $"($HAZE_UID):($HAZE_GID)" /var/www/html/core/skeleton /var/www/html/build/integration/vendor /var/www/html/build/integration/composer.lock /var/www/html/build/integration/output /var/www/html/build/integration/work /var/www/html/core/skeleton /var/www/.composer/cache /var/www/html/apps/spreed/tests/integration/vendor/composer +chown $"($HAZE_UID):($HAZE_GID)" /var/www/html/core/skeleton \ + /var/www/html/build/integration/vendor \ + /var/www/html/build/integration/composer.lock \ + /var/www/html/build/integration/output \ + /var/www/html/build/integration/work \ + /var/www/html/core/skeleton \ + /var/www/.composer/cache \ + /var/www/html/apps/spreed \ + /var/www/html/apps/spreed/tests \ + /var/www/html/apps/spreed/tests/integration \ + /var/www/html/apps/spreed/tests/integration/vendor \ + /var/www/html/apps/spreed/tests/integration/vendor/composer echo "{}\n" | save -f /var/www/html/build/integration/composer.lock diff --git a/src/args.rs b/src/args.rs index 777c6b2..94daee3 100644 --- a/src/args.rs +++ b/src/args.rs @@ -525,6 +525,7 @@ fn test_arg_parse() { sources_root: Default::default(), app_directories: Default::default(), work_dir: Default::default(), + worktree_dir: Default::default(), auto_setup: Default::default(), volume: vec![], blackfire: None, diff --git a/src/cloud.rs b/src/cloud.rs index c7faadd..131c93d 100644 --- a/src/cloud.rs +++ b/src/cloud.rs @@ -1,6 +1,7 @@ use crate::config::{HazeConfig, HazeVolumeConfig}; use crate::database::Database; use crate::exec::{exec, exec_io, exec_tty, ExitCode}; +use crate::git::{create_worktree, find_app_repo, init_submodules, remove_worktree}; use crate::mapping::{for_config, Mapping}; use crate::php::PhpVersion; use crate::service::Service; @@ -53,6 +54,116 @@ fn get_max_php_version(base_path: &Utf8Path) -> Option { const PINNED_FILE: &str = "/var/www/html/data/.haze-pinned"; +fn enabled_apps(config: &HazeConfig, options: &CloudOptions) -> Vec { + let mut apps: Vec = config.auto_setup.enable_apps.clone(); + for service in &options.services { + apps.extend(service.apps().iter().map(|app| app.to_string())); + + // Presets do not list apps in apps(), so we need to fetch them from the config. + for preset in config.preset.iter().filter(|p| p.name == service.name()) { + apps.extend(preset.apps.iter().cloned()); + } + } + apps.dedup(); + apps +} + +async fn creates_worktrees_for_instance( + config: &HazeConfig, + id: &str, + options: &CloudOptions, + git_ref: &str, +) -> Result<(Utf8PathBuf, Vec)> { + let worktree_dir = config.worktree_dir.join(id); + + create_dir_all(&worktree_dir) + .await + .into_diagnostic() + .wrap_err("Failed to create directory for worktrees")?; + + println!("Creating worktree of {} at {}", git_ref, worktree_dir); + create_worktree(&config.sources_root, &worktree_dir, git_ref)?; + init_submodules(&worktree_dir).wrap_err("Failed to setup submodules")?; + + for app in enabled_apps(config, options) { + let Some(app_repo) = find_app_repo(config, &app) else { + continue; + }; + let Some(dir_name) = app_repo.parent().and_then(|dir| dir.file_name()) else { + continue; + }; + let app_worktree = worktree_dir.join(dir_name).join(&app); + + println!("Creating worktree of {} at {}", app, app_worktree); + create_worktree(&app_repo, &app_worktree, "HEAD") + .wrap_err_with(|| format!("Failed to create worktree for {app}"))?; + } + + Ok((worktree_dir, worktree_volumes(config, id))) +} + +async fn remove_worktrees_for_instance(config: &HazeConfig, id: &str) -> Result<()> { + let worktree_dir = config.worktree_dir.join(id); + + remove_worktree(&worktree_dir)?; + + for app_dir in &config.app_directories { + let Some(dir_name) = app_dir.file_name() else { + continue; + }; + let Ok(app_worktrees) = worktree_dir.join(dir_name).read_dir_utf8() else { + continue; + }; + for app_worktree in app_worktrees.flatten() { + remove_worktree(app_worktree.path())?; + } + } + + Ok(()) +} + +fn worktree_volumes(config: &HazeConfig, id: &str) -> Vec { + // In order to be able to run git operations on the worktree within the container, + // we need to mount the original repository too. + fn repo_volume(repo: &Utf8Path) -> HazeVolumeConfig { + HazeVolumeConfig { + create: false, + source: repo.join(".git"), + read_only: false, + target: repo.join(".git"), + } + } + + let worktree_dir = config.worktree_dir.join(id); + + let mut volumes = vec![repo_volume(&config.sources_root)]; + + for app_dir in &config.app_directories { + let Some(dir_name) = app_dir.file_name() else { + continue; + }; + let Ok(app_worktrees) = worktree_dir.join(dir_name).read_dir_utf8() else { + continue; + }; + for app_worktree in app_worktrees.flatten() { + let app = app_worktree.file_name().to_string(); + let Some(app_repo) = find_app_repo(config, &app) else { + continue; + }; + + volumes.push(HazeVolumeConfig { + create: false, + source: app_worktree.into_path(), + read_only: false, + target: format!("/var/www/html/{dir_name}/{app}").into(), + }); + volumes.push(repo_volume(&app_repo)); + } + } + + volumes +} + #[derive(Clone, Debug, Eq, PartialEq, Default)] pub struct CloudOptions { pub name: Option, @@ -61,6 +172,7 @@ pub struct CloudOptions { pub services: Vec, pub app_packages: Vec, pub version: Option, + pub detach: bool, } impl CloudOptions { @@ -74,6 +186,7 @@ impl CloudOptions { services: vec![], app_packages: vec![], version: None, + detach: false, } } @@ -88,6 +201,7 @@ impl CloudOptions { let mut services = Vec::new(); let mut app_package = Vec::new(); let mut version = None; + let mut detach = false; while let Some(option) = args.peek() { if let Ok(db_option) = Database::from_str(option.as_ref()) { @@ -108,6 +222,9 @@ impl CloudOptions { } else if option.as_ref() == "--name" { let _ = args.next(); name = args.next().map(|s| s.into()); + } else if option.as_ref() == "--detach" { + let _ = args.next(); + detach = true; } else { break; } @@ -122,6 +239,7 @@ impl CloudOptions { services, app_packages: app_package, version, + detach, }) } } @@ -254,10 +372,10 @@ impl Cloud { .wrap_err("Failed to create directory for app packages")?; } - let source_root = if let Some(version) = options.version.as_deref() { - download_nc(config, version).await? - } else { - config.sources_root.clone() + let (source_root, source_volumes) = match (options.version.as_deref(), options.detach) { + (Some(version), _) => (download_nc(config, version).await?, Vec::new()), + (None, true) => creates_worktrees_for_instance(config, &id, &options, "HEAD").await?, + (None, false) => (config.sources_root.clone(), Vec::new()), }; let app_volumes = options @@ -289,11 +407,12 @@ impl Cloud { .collect::>>()?; let mappings = for_config(config) + .chain(source_volumes.iter().map(Mapping::from)) .chain(app_volumes.iter().map(Mapping::from)) .collect::>(); for mapping in &mappings { mapping - .create(&id, config) + .create(&id, config, &source_root) .await .wrap_err_with(|| format!("Failed to setup work directory {}", mapping.source))?; } @@ -561,7 +680,7 @@ impl Cloud { }) } - pub async fn destroy(self, docker: &Docker) -> Result<()> { + pub async fn destroy(self, docker: &Docker, config: &HazeConfig) -> Result<()> { for container in self.containers { let _ = docker .kill_container(container.trim_start_matches('/'), None) @@ -593,6 +712,8 @@ impl Cloud { } } + remove_worktrees_for_instance(config, &self.id).await?; + Ok(()) } @@ -701,6 +822,15 @@ impl Cloud { let db = labels.get("haze-db")?.parse().ok()?; let php = labels.get("haze-php")?.parse().ok()?; let version = labels.get("haze-version").cloned(); + let sources_root = cloud + .mounts + .as_ref()? + .iter() + .find(|mount| mount.destination.as_deref() == Some("/var/www/html"))? + .source + .as_deref(); + + let detach = sources_root.map(Utf8Path::new)? != config.sources_root; let found_services = labels .get("haze-services")? @@ -740,6 +870,7 @@ impl Cloud { services: found_services, app_packages: vec![], version, + detach, }, address, preset_config: HashMap::default(), @@ -847,15 +978,24 @@ impl Cloud { format!("/var/www/html/{path}").into() }; - let mut mappings = for_config(config).collect::>(); + let worktree_volumes = worktree_volumes(config, &self.id); + let mut mappings = for_config(config) + .chain(worktree_volumes.iter().map(Mapping::from)) + .collect::>(); mappings.sort_by_key(|mapping| usize::MAX - mapping.target.as_str().len()); + let source_root = if self.options.detach { + config.worktree_dir.join(&self.id) + } else { + config.sources_root.clone() + }; + for mapping in mappings { if let Some(rel_path) = path.strip_prefix(mapping.target.as_str()) { let rel_path = rel_path.trim_matches('/'); return Some( mapping - .source(&self.id, config, &config.sources_root) + .source(&self.id, config, &source_root) .join(rel_path), ); } diff --git a/src/config.rs b/src/config.rs index 9e4548f..392b453 100644 --- a/src/config.rs +++ b/src/config.rs @@ -16,6 +16,7 @@ pub struct HazeConfig { pub sources_root: Utf8PathBuf, pub app_directories: Vec, pub work_dir: Utf8PathBuf, + pub worktree_dir: Utf8PathBuf, pub auto_setup: HazeAutoSetupConfig, pub volume: Vec, pub blackfire: Option, @@ -29,6 +30,8 @@ pub struct RawHazeConfig { #[serde(default = "default_work_dir")] pub work_dir: Utf8PathBuf, #[serde(default)] + pub worktree_dir: Option, + #[serde(default)] pub app_directories: Vec, #[serde(default)] pub auto_setup: HazeAutoSetupConfig, @@ -57,6 +60,8 @@ impl From for HazeConfig { } } + let work_dir = normalize_path(raw.work_dir); + HazeConfig { sources_root: normalize_path(raw.sources_root), app_directories: raw @@ -64,7 +69,11 @@ impl From for HazeConfig { .into_iter() .map(normalize_path) .collect(), - work_dir: normalize_path(raw.work_dir), + worktree_dir: raw + .worktree_dir + .map(normalize_path) + .unwrap_or_else(|| work_dir.join("worktrees")), + work_dir, auto_setup: raw.auto_setup, volume: raw.volume, blackfire: raw.blackfire, diff --git a/src/git.rs b/src/git.rs index fc31255..2469de5 100644 --- a/src/git.rs +++ b/src/git.rs @@ -1,9 +1,10 @@ use crate::config::HazeConfig; use crate::Result; +use camino::{Utf8Path, Utf8PathBuf}; use git2::build::CheckoutBuilder; use git2::{Branch, BranchType, Repository, RepositoryState}; use indicatif::{MultiProgress, ProgressBar, ProgressStyle}; -use miette::{Context, IntoDiagnostic}; +use miette::{Context, IntoDiagnostic, Report}; use rayon::iter::{IntoParallelRefIterator, ParallelIterator}; use rayon::ThreadPoolBuilder; use std::fs::read_dir; @@ -69,6 +70,83 @@ pub fn checkout_all(config: &HazeConfig, mut name: &str, verbose: bool) -> Resul Ok(()) } +pub fn find_app_repo(config: &HazeConfig, app: &str) -> Option { + find_app_repos(config) + .ok()? + .filter_map(|dir| Utf8PathBuf::try_from(dir).ok()) + .find(|dir| dir.file_name().is_some_and(|name| name == app)) +} + +fn git(repo: &Utf8Path, args: &[&str]) -> Result { + let output = Command::new(GIT_BINARY) + .args(args) + .current_dir(repo) + .output() + .into_diagnostic() + .wrap_err_with(|| format!("Failed to run git {} for {}", args.join(" "), repo))?; + if !output.status.success() { + let err = String::from_utf8_lossy(&output.stderr); + return Err(Report::msg(format!( + "git {} failed in {}: {}", + args.join(" "), + repo, + err.trim() + ))); + } + Ok(String::from_utf8_lossy(&output.stdout).trim().into()) +} + +fn has_ref(repo: &Utf8Path, git_ref: &str) -> bool { + git( + repo, + &[ + "rev-parse", + "--verify", + "--quiet", + &format!("{git_ref}^{{commit}}"), + ], + ) + .is_ok() +} + +pub fn create_worktree(repo: &Utf8Path, dest: &Utf8Path, git_ref: &str) -> Result<()> { + if !has_ref(repo, git_ref) { + // The reference might be a tag that hasn't been fetched yet. + let _ = git(repo, &["fetch", "origin", "tag", git_ref]); + if !has_ref(repo, git_ref) { + return Err(Report::msg(format!("{git_ref} not found in {repo}"))); + } + } + + git( + repo, + &["worktree", "add", "--detach", dest.as_str(), git_ref], // Use a detached head so the worktree doesn't conflict with another one. + )?; + Ok(()) +} + +pub fn remove_worktree(repo: &Utf8Path) -> Result<()> { + git(repo, &["worktree", "remove", "--force", repo.as_str()])?; + Ok(()) +} + +pub fn init_submodules(worktree: &Utf8Path) -> Result<()> { + git(worktree, &["submodule", "update", "--init", "--recursive"])?; + Ok(()) +} + +pub fn prune_worktrees(config: &HazeConfig) -> Result<()> { + let app_repos = find_app_repos(config)? + .filter_map(|repo| Utf8PathBuf::try_from(repo).ok()) + .collect::>(); + for repo in once(config.sources_root.clone()).chain(app_repos) { + if let Err(e) = git(&repo, &["worktree", "prune"]) { + eprintln!("Failed to prune worktrees of {repo}: {e:#}"); + } + } + Ok(()) +} + fn current_branch_name(repo: &Repository) -> Option { let (branch, _) = repo .branches(None) diff --git a/src/help.rs b/src/help.rs index 97af8c8..566bff7 100644 --- a/src/help.rs +++ b/src/help.rs @@ -89,6 +89,8 @@ fn subcommand_help(command: &dyn SubCommand) { print!(" {}", "[database type]".green()); print!(" {}", "[services]".green()); print!(" {}", "[vX.Y.Z]".green()); + print!(" {}", "[--name ]".green()); + print!(" {}", "[--detach]".green()); } let args = if let Some(args) = command.get_str("Args") { @@ -156,6 +158,17 @@ fn subcommand_help(command: &dyn SubCommand) { service.get_documentation().unwrap_or_default(), ); } + println!(); + + println!("{}", "Options:".yellow().bold()); + println!( + " {} name the instance instead of generating a random name", + "--name ".blue() + ); + println!( + " {} give the instance its own worktree of the sources", + "--detach".blue() + ); } if let Some(sub_commands) = command.sub_commands() { diff --git a/src/main.rs b/src/main.rs index 11c74f9..d7b14c5 100644 --- a/src/main.rs +++ b/src/main.rs @@ -5,7 +5,7 @@ use crate::cloud::{Cloud, CloudOptions}; use crate::config::HazeConfig; use crate::database::DatabaseFamily; use crate::exec::{container_logs, exec, exec_tty}; -use crate::git::{checkout_all, pull_all}; +use crate::git::{checkout_all, prune_worktrees, pull_all}; use crate::help::help; use crate::image::update_image; use crate::network::clear_networks; @@ -79,7 +79,7 @@ async fn main() -> Result { } for cloud in remove { - if let Err(e) = cloud.destroy(&docker).await { + if let Err(e) = cloud.destroy(&docker, &config).await { eprintln!("Error while removing cloud: {:#}", e); } } @@ -91,6 +91,11 @@ async fn main() -> Result { if let Some(id) = cache_dir.file_name().to_str() { if !retain.iter().any(|cloud| cloud.id == id) { let path = cache_dir.path(); + // By default, the worktrees live within the work dir, we do not want to delete it here. + if path == config.worktree_dir { + continue; + } + if path.is_dir() { remove_dir_all(&path).into_diagnostic().wrap_err_with(|| { format!("Failed to cleanup stray cache dir: {}", path.display()) @@ -103,6 +108,8 @@ async fn main() -> Result { } } } + + prune_worktrees(&config)?; } HazeArgs::List { filter } => { let list = Cloud::list(&docker, filter, &config).await?; @@ -119,13 +126,19 @@ async fn main() -> Result { Some(version) => format!(", v{version}"), None => String::new(), }; + let detached = if cloud.options.detach { + ", detached" + } else { + "" + }; println!( - "Cloud {}{}, {}, {}{}, running on {}", + "Cloud {}{}, {}, {}{}{}, running on {}", cloud.id, pin, cloud.php().name(), services, version, + detached, cloud.address, ); } @@ -135,7 +148,7 @@ async fn main() -> Result { } HazeArgs::Stop { filter } => { let cloud = Cloud::get_by_filter(&docker, filter, &config).await?; - cloud.destroy(&docker).await?; + cloud.destroy(&docker, &config).await?; } HazeArgs::Logs { filter, @@ -290,7 +303,7 @@ async fn main() -> Result { ) .await { - cloud.destroy(&docker).await?; + cloud.destroy(&docker, &config).await?; return Err(e); } if let Some(app) = args @@ -307,7 +320,7 @@ async fn main() -> Result { } args.insert(0, "tests".to_string()); let result = cloud.exec(&docker, args, false, get_forward_env()).await?; - cloud.destroy(&docker).await?; + cloud.destroy(&docker, &config).await?; return Ok(result.into()); } HazeArgs::Integration { options, mut args } => { @@ -315,7 +328,7 @@ async fn main() -> Result { args.insert(0, "integration".to_string()); cloud.exec(&docker, args, false, get_forward_env()).await?; - cloud.destroy(&docker).await?; + cloud.destroy(&docker, &config).await?; } HazeArgs::Fmt { path } => { let cloud = Cloud::create(&docker, CloudOptions::new(&config), &config).await?; @@ -334,7 +347,7 @@ async fn main() -> Result { .and_then(|c| c.to_result()) { eprintln!("{}", String::from_utf8_lossy(&out_buffer)); - cloud.destroy(&docker).await?; + cloud.destroy(&docker, &config).await?; return Err(e); } out_buffer.clear(); @@ -348,7 +361,7 @@ async fn main() -> Result { ) .await { - cloud.destroy(&docker).await?; + cloud.destroy(&docker, &config).await?; return Err(e); } println!("Cleanup"); @@ -363,7 +376,7 @@ async fn main() -> Result { .and_then(|c| c.to_result()) { eprintln!("{}", String::from_utf8_lossy(&out_buffer)); - cloud.destroy(&docker).await?; + cloud.destroy(&docker, &config).await?; return Err(e); } if let Err(e) = cloud @@ -377,10 +390,10 @@ async fn main() -> Result { .and_then(|c| c.to_result()) { eprintln!("{}", String::from_utf8_lossy(&out_buffer)); - cloud.destroy(&docker).await?; + cloud.destroy(&docker, &config).await?; return Err(e); } - cloud.destroy(&docker).await?; + cloud.destroy(&docker, &config).await?; } HazeArgs::Shell { command, options } => { let cloud = setup(&docker, options, &config).await?; @@ -396,7 +409,7 @@ async fn main() -> Result { get_forward_env(), ) .await?; - cloud.destroy(&docker).await?; + cloud.destroy(&docker, &config).await?; } HazeArgs::Pin { filter } => { let cloud = Cloud::get_by_filter(&docker, filter, &config).await?; diff --git a/src/mapping.rs b/src/mapping.rs index 9c01da0..aef7d36 100644 --- a/src/mapping.rs +++ b/src/mapping.rs @@ -81,7 +81,12 @@ impl<'a> Mapping<'a> { } } - pub async fn create(&self, id: &str, config: &HazeConfig) -> Result<()> { + pub async fn create( + &self, + id: &str, + config: &HazeConfig, + source_root: &Utf8Path, + ) -> Result<()> { if !self.create { return Ok(()); } @@ -96,6 +101,19 @@ impl<'a> Mapping<'a> { MappingType::File => write(source, "").await.into_diagnostic()?, } + // For worktrees, pre-create mountpoints within /var/www/html to prevent docker from creating them as root. + if self.target.starts_with("/var/www/html") { + let local_target = source_root.join( + self.target + .strip_prefix("/var/www/html") + .unwrap_or(&self.target), + ); + match self.mapping_type { + MappingType::Folder => create_dir_all(local_target).await.into_diagnostic()?, + MappingType::File => write(local_target, "").await.into_diagnostic()?, + } + } + Ok(()) }