From cd9c9cd467759cbfda0be00e6ab5c050f1ef71d9 Mon Sep 17 00:00:00 2001 From: Marco Walz Date: Fri, 6 Feb 2026 19:17:21 +0100 Subject: [PATCH 1/6] feat: managed compose network with bitcoin --- .claude/CLAUDE.md | 18 +- crates/icp-cli/src/commands/network/stop.rs | 6 + .../assets/compose/docker-compose.test.yml | 11 + crates/icp-cli/tests/common/mod.rs | 16 + crates/icp-cli/tests/network_tests.rs | 167 ++++- crates/icp/src/manifest/network.rs | 19 + crates/icp/src/network/config.rs | 14 + crates/icp/src/network/managed/compose.rs | 576 ++++++++++++++++++ crates/icp/src/network/managed/mod.rs | 1 + crates/icp/src/network/managed/run.rs | 44 +- crates/icp/src/network/mod.rs | 25 + docs/guides/containerized-networks.md | 146 +++++ docs/guides/index.md | 2 +- docs/reference/configuration.md | 25 + docs/schemas/icp-yaml-schema.json | 38 ++ docs/schemas/network-yaml-schema.json | 38 ++ 16 files changed, 1139 insertions(+), 7 deletions(-) create mode 100644 crates/icp-cli/tests/assets/compose/docker-compose.test.yml create mode 100644 crates/icp/src/network/managed/compose.rs diff --git a/.claude/CLAUDE.md b/.claude/CLAUDE.md index 66895697..2f92edb6 100644 --- a/.claude/CLAUDE.md +++ b/.claude/CLAUDE.md @@ -236,7 +236,7 @@ This project uses Snafu for error handling. - Every new *primary erroring action* gets its own error variant. There is no `MyError::Io { source: io::Error }`, instead (hypothetically) `OpenSocket` and `WriteSocket` should be separate. `snafu(context(false))` is not permitted. `snafu(transparent)` should *only* be used for source error types defined elsewhere in this repo, *not* for foreign error types. - Every error regarding a file in some way (processing, creating, etc.) should contain the file path of the error. It is okay to add 'dummy' file path parameters only used in error handling routes. For 'basic' file ops and JSON/YML loading use the functions in `icp::fs`, whose errors include the file path and can be made `snafu(transparent)`. -## Examples +## Examples and Templates The `examples/` directory contains working project templates demonstrating: @@ -284,3 +284,19 @@ Before finalizing recipe-related changes, ask the user: - What branch or version to compare with Then verify documentation matches the recipe templates from the specified source. + +### Dependency Versions in Examples/Templates + +When creating or updating examples and templates: + +1. **Always use the latest versions** of dependencies like `candid`, `ic-cdk`, `ic-cdk-macros`, etc. Use `cargo search --limit 1` to check the current latest version on crates.io before specifying dependency versions. + +2. **When updating a single example**, ask the developer whether other examples should also be updated to maintain consistency across all examples. + +3. **Schema references** in YAML files (`$schema=`) should point to the latest stable release tag (e.g., `v0.1.0`), not beta versions. + +4. **Related repository**: The `icp-cli-templates` repository (https://github.com/dfinity/icp-cli-templates) contains cargo-generate templates that should follow the same versioning guidelines. + +5. **Motoko imports**: When writing Motoko code, prefer `mo:core` over `mo:base`. See https://docs.internetcomputer.org/motoko/core for documentation. + +6. **Gitignore for ICP projects**: Only add `.icp/cache/` to `.gitignore`, NOT the entire `.icp/` directory. The `.icp/data/` directory contains mainnet canister ID mappings that should be version controlled to preserve deployment information. diff --git a/crates/icp-cli/src/commands/network/stop.rs b/crates/icp-cli/src/commands/network/stop.rs index 35220bea..a860a760 100644 --- a/crates/icp-cli/src/commands/network/stop.rs +++ b/crates/icp-cli/src/commands/network/stop.rs @@ -65,6 +65,12 @@ pub async fn exec(ctx: &Context, cmd: &Cmd) -> Result<(), anyhow::Error> { &id[..12] )); } + ChildLocator::Compose { project_name, .. } => { + let _ = ctx.term.write_line(&format!( + "Stopping compose network (project: {})...", + project_name + )); + } } stop_network(&descriptor.child_locator).await?; diff --git a/crates/icp-cli/tests/assets/compose/docker-compose.test.yml b/crates/icp-cli/tests/assets/compose/docker-compose.test.yml new file mode 100644 index 00000000..767274da --- /dev/null +++ b/crates/icp-cli/tests/assets/compose/docker-compose.test.yml @@ -0,0 +1,11 @@ +# Test compose file for icp-cli compose network testing +# This is a minimal configuration that starts only the IC network launcher +services: + icp-network: + image: ghcr.io/dfinity/icp-cli-network-launcher:v11.0.0 + ports: + - "0:4943" + environment: + - ICP_CLI_NETWORK_LAUNCHER_INTERFACE_VERSION=1.0.0 + volumes: + - "${ICP_STATUS_DIR:-/tmp/status}:/app/status" diff --git a/crates/icp-cli/tests/common/mod.rs b/crates/icp-cli/tests/common/mod.rs index 7ec02d52..3786e5fd 100644 --- a/crates/icp-cli/tests/common/mod.rs +++ b/crates/icp-cli/tests/common/mod.rs @@ -49,6 +49,22 @@ environments: network: docker-network "#; +/// A network manifest for a compose-based network +pub(crate) const NETWORK_COMPOSE: &str = r#" +networks: + - name: compose-network + mode: managed + compose: + file: compose/docker-compose.test.yml + gateway-service: icp-network +"#; + +pub(crate) const ENVIRONMENT_COMPOSE: &str = r#" +environments: + - name: compose-environment + network: compose-network +"#; + /// This ID is dependent on the toplogy being served by pocket-ic /// NOTE: If the topology is changed (another subnet is added, etc) the ID may change. /// References: diff --git a/crates/icp-cli/tests/network_tests.rs b/crates/icp-cli/tests/network_tests.rs index b811c203..9aeb9d24 100644 --- a/crates/icp-cli/tests/network_tests.rs +++ b/crates/icp-cli/tests/network_tests.rs @@ -14,8 +14,8 @@ use serial_test::file_serial; use test_tag::tag; use crate::common::{ - ENVIRONMENT_DOCKER, ENVIRONMENT_RANDOM_PORT, NETWORK_DOCKER, NETWORK_RANDOM_PORT, TestContext, - TestNetwork, clients, + ENVIRONMENT_COMPOSE, ENVIRONMENT_DOCKER, ENVIRONMENT_RANDOM_PORT, NETWORK_COMPOSE, + NETWORK_DOCKER, NETWORK_RANDOM_PORT, TestContext, TestNetwork, clients, }; use icp::{ fs::{read_to_string, write_string}, @@ -579,6 +579,169 @@ async fn cannot_override_ic() { .stderr(contains("`ic` is a reserved network name")); } +#[tag(docker)] +#[tokio::test] +async fn network_compose() { + let ctx = TestContext::new(); + + let project_dir = ctx.create_project_dir("compose-network"); + + // Copy compose file to project + ctx.copy_asset_dir("compose", &project_dir.join("compose")); + + // Project manifest + write_string( + &project_dir.join("icp.yaml"), + &formatdoc! {r#" + {NETWORK_COMPOSE} + {ENVIRONMENT_COMPOSE} + "#}, + ) + .expect("failed to write project manifest"); + + ctx.docker_pull_network(); + let _guard = ctx.start_network_in(&project_dir, "compose-network").await; + ctx.ping_until_healthy(&project_dir, "compose-network"); + + // Verify the network descriptor has compose child locator + let descriptor_contents = ctx.read_network_descriptor(&project_dir, "compose-network"); + let descriptor: serde_json::Value = + serde_json::from_slice(&descriptor_contents).expect("descriptor should be valid JSON"); + + let child_locator = descriptor + .get("child-locator") + .expect("descriptor should have child-locator"); + + // Check compose-specific fields + assert!( + child_locator.get("project-name").is_some(), + "compose child locator should have project-name" + ); + assert!( + child_locator.get("compose-file").is_some(), + "compose child locator should have compose-file" + ); + assert!( + child_locator.get("socket").is_some(), + "compose child locator should have socket" + ); + + // Verify the project name follows the convention + let project_name = child_locator + .get("project-name") + .and_then(|p| p.as_str()) + .expect("project-name should be a string"); + assert_eq!( + project_name, "icp-compose-network", + "compose project name should be icp-" + ); + + // Verify network is healthy by checking balance + let balance = clients::ledger(&ctx) + .balance_of(Principal::anonymous(), None) + .await; + assert!(balance > 0_u128); +} + +#[tag(docker)] +#[tokio::test] +async fn network_compose_stop() { + let ctx = TestContext::new(); + + let project_dir = ctx.create_project_dir("compose-stop"); + + // Copy compose file to project + ctx.copy_asset_dir("compose", &project_dir.join("compose")); + + // Project manifest + write_string( + &project_dir.join("icp.yaml"), + &formatdoc! {r#" + {NETWORK_COMPOSE} + {ENVIRONMENT_COMPOSE} + "#}, + ) + .expect("failed to write project manifest"); + + ctx.docker_pull_network(); + + // Start network in background + ctx.icp() + .current_dir(&project_dir) + .args(["network", "start", "compose-network", "--background"]) + .assert() + .success(); + + let network = ctx.wait_for_network_descriptor(&project_dir, "compose-network"); + ctx.ping_until_healthy(&project_dir, "compose-network"); + + // Verify descriptor file exists + let descriptor_file = project_dir + .join(".icp") + .join("cache") + .join("networks") + .join("compose-network") + .join("descriptor.json"); + assert!( + descriptor_file.exists(), + "descriptor file should exist before stop" + ); + + // Stop the network + ctx.icp() + .current_dir(&project_dir) + .args(["network", "stop", "compose-network"]) + .assert() + .success() + .stdout(contains( + "Stopping compose network (project: icp-compose-network)", + )) + .stdout(contains("Network stopped successfully")); + + // Verify descriptor file is removed + assert!( + !descriptor_file.exists(), + "descriptor file should be removed after stop" + ); + + // Verify network is no longer reachable + let agent = ic_agent::Agent::builder() + .with_url(format!("http://127.0.0.1:{}", network.gateway_port)) + .build() + .expect("Failed to build agent"); + + let status_result = agent.status().await; + assert!( + status_result.is_err(), + "Network should not be reachable after stopping" + ); +} + +#[tokio::test] +async fn network_compose_file_not_found() { + let ctx = TestContext::new(); + + let project_dir = ctx.create_project_dir("compose-missing"); + + // Project manifest without copying the compose file + write_string( + &project_dir.join("icp.yaml"), + &formatdoc! {r#" + {NETWORK_COMPOSE} + {ENVIRONMENT_COMPOSE} + "#}, + ) + .expect("failed to write project manifest"); + + // Start should fail with file not found error + ctx.icp() + .current_dir(&project_dir) + .args(["network", "start", "compose-network", "--background"]) + .assert() + .failure() + .stderr(contains("docker compose file not found")); +} + #[tokio::test] #[file_serial(default_local_network)] async fn override_local_network_as_connected() { diff --git a/crates/icp/src/manifest/network.rs b/crates/icp/src/manifest/network.rs index ee35217f..dd61fdbe 100644 --- a/crates/icp/src/manifest/network.rs +++ b/crates/icp/src/manifest/network.rs @@ -56,6 +56,10 @@ pub enum ManagedMode { /// Bind mounts to add to the container in the format relative_host_path:container_path[:options] mounts: Option>, }, + Compose { + /// Docker Compose configuration + compose: ComposeConfig, + }, Launcher { /// HTTP gateway configuration gateway: Option, @@ -70,6 +74,21 @@ pub enum ManagedMode { }, } +/// Docker Compose network configuration +#[derive(Clone, Debug, Deserialize, PartialEq, JsonSchema)] +#[serde(rename_all = "kebab-case")] +pub struct ComposeConfig { + /// Path to the docker-compose.yml file (relative to project root) + pub file: String, + + /// Name of the service that runs the IC gateway + pub gateway_service: String, + + /// Additional environment variables to pass to docker compose + #[serde(default)] + pub environment: Vec, +} + impl Default for ManagedMode { fn default() -> Self { ManagedMode::Launcher { diff --git a/crates/icp/src/network/config.rs b/crates/icp/src/network/config.rs index 6ef56bf6..07692b21 100644 --- a/crates/icp/src/network/config.rs +++ b/crates/icp/src/network/config.rs @@ -88,6 +88,15 @@ pub enum ChildLocator { /// Whether to remove the container when it exits. rm_on_exit: bool, }, + /// A Docker Compose project (used for multi-container setups like Bitcoin). + Compose { + /// The compose project name (e.g., "icp-local-bitcoin"). + project_name: String, + /// Path to the docker-compose.yml file. + compose_file: String, + /// Docker socket path. + socket: String, + }, } impl ChildLocator { @@ -106,6 +115,11 @@ impl ChildLocator { ChildLocator::Container { id, socket, .. } => { crate::network::managed::docker::is_container_running(socket, id).await } + ChildLocator::Compose { + project_name, + socket, + .. + } => crate::network::managed::compose::is_compose_running(socket, project_name).await, } } } diff --git a/crates/icp/src/network/managed/compose.rs b/crates/icp/src/network/managed/compose.rs new file mode 100644 index 00000000..357fad55 --- /dev/null +++ b/crates/icp/src/network/managed/compose.rs @@ -0,0 +1,576 @@ +//! Docker Compose network management. +//! +//! This module handles starting, stopping, and monitoring Docker Compose-based networks. +//! Compose networks allow running multi-container setups like Bitcoin regtest alongside +//! the IC network launcher. + +use std::time::Duration; + +use async_dropper::{AsyncDrop, AsyncDropper}; +use async_trait::async_trait; +use bollard::{ + Docker, + query_parameters::{InspectContainerOptions, ListContainersOptions}, +}; +use snafu::prelude::*; +use tokio::select; + +use crate::{ + network::{ManagedComposeConfig, config::ChildLocator, managed::launcher::NetworkInstance}, + prelude::*, +}; + +use super::{docker::connect_docker, launcher::wait_for_launcher_status}; + +/// Check if a Docker Compose project is running by checking if its containers exist. +pub async fn is_compose_running(socket: &str, project_name: &str) -> bool { + let Ok(docker) = connect_docker(socket) else { + return false; + }; + + // List containers with the compose project label + let filters: std::collections::HashMap> = [( + "label".to_string(), + vec![format!("com.docker.compose.project={project_name}")], + )] + .into(); + + match docker + .list_containers(Some(ListContainersOptions { + filters: Some(filters), + ..Default::default() + })) + .await + { + Ok(containers) => { + // Check if any containers are running + containers.iter().any(|c| { + c.state + .as_ref() + .is_some_and(|s| s.to_string().to_lowercase() == "running") + }) + } + Err(_) => false, + } +} + +#[derive(Debug, Snafu)] +pub enum ComposeError { + #[snafu(display("docker compose file not found: {path}"))] + ComposeFileNotFound { path: PathBuf }, + + #[snafu(display("failed to start compose services: {message}"))] + StartServices { message: String }, + + #[snafu(display("failed to stop compose services: {message}"))] + StopServices { message: String }, + + #[snafu(display("gateway service '{service}' not found in compose project"))] + GatewayServiceNotFound { service: String }, + + #[snafu(display("timed out waiting for gateway status after {seconds} seconds"))] + StatusTimeout { seconds: u64 }, + + #[snafu(display( + "gateway service '{service}' exited unexpectedly. \ + Check logs with: docker compose -f {compose_file} -p {project_name} logs {service}" + ))] + GatewayServiceExited { + service: String, + compose_file: String, + project_name: String, + }, + + #[snafu(display("failed to read status from container: {message}"))] + ReadStatus { message: String }, + + #[snafu(transparent)] + ConnectDocker { + source: super::docker::ConnectDockerError, + }, + + #[snafu(transparent)] + WaitForLauncherStatus { + source: super::launcher::WaitForLauncherStatusError, + }, + + #[snafu(transparent)] + WatchStatusDir { + source: super::launcher::WaitForFileError, + }, + + #[snafu(display("failed to parse root key {key}"))] + ParseRootKey { + key: String, + source: hex::FromHexError, + }, + + #[snafu(display("failed to create status directory"))] + CreateStatusDir { source: std::io::Error }, + + #[snafu(display("failed to inspect container {container_id}"))] + InspectContainer { + source: bollard::errors::Error, + container_id: String, + }, + + #[snafu(display("required field {field} missing from docker API"))] + RequiredFieldMissing { field: String }, +} + +/// Manages a Docker Compose network lifecycle. +pub struct ComposeNetwork { + project_name: String, + compose_file: PathBuf, + gateway_service: String, + environment: Vec, +} + +impl ComposeNetwork { + pub fn new(network_name: &str, config: &ManagedComposeConfig, project_root: &Path) -> Self { + Self { + project_name: format!("icp-{network_name}"), + compose_file: project_root.join(&config.file), + gateway_service: config.gateway_service.clone(), + environment: config.environment.clone(), + } + } + + /// Start the Docker Compose services and wait for the gateway to be ready. + pub async fn start( + &self, + ) -> Result< + ( + AsyncDropper, + NetworkInstance, + ChildLocator, + ), + ComposeError, + > { + // Verify compose file exists + ensure!( + self.compose_file.exists(), + ComposeFileNotFoundSnafu { + path: &self.compose_file + } + ); + + // Get Docker socket + let socket = get_docker_socket()?; + + // Create a temporary directory for the status file on the host + let host_status_tmpdir = + camino_tempfile::Utf8TempDir::new().context(CreateStatusDirSnafu)?; + let host_status_dir = host_status_tmpdir.path().to_path_buf(); + + // Build environment variables for compose + let mut env_vars: Vec<(&str, &str)> = self + .environment + .iter() + .filter_map(|e| e.split_once('=')) + .collect(); + + // Add the status directory as an environment variable so compose can mount it + let status_dir_str = host_status_dir.to_string(); + env_vars.push(("ICP_STATUS_DIR", &status_dir_str)); + + // Set up the file watcher before starting compose + let watcher = wait_for_launcher_status(&host_status_dir)?; + + // Start compose services + let mut cmd = tokio::process::Command::new("docker"); + cmd.args([ + "compose", + "-f", + self.compose_file.as_str(), + "-p", + &self.project_name, + "up", + "-d", + "--wait", + ]); + + // Add environment variables + for (key, value) in &env_vars { + cmd.env(key, value); + } + + let output = cmd + .output() + .await + .map_err(|e| ComposeError::StartServices { + message: e.to_string(), + })?; + + if !output.status.success() { + return Err(ComposeError::StartServices { + message: String::from_utf8_lossy(&output.stderr).to_string(), + }); + } + + eprintln!("Started compose project '{}'", self.project_name); + + // Create drop guard + let guard = AsyncDropper::new(ComposeDropGuard { + project_name: Some(self.project_name.clone()), + compose_file: Some(self.compose_file.clone()), + }); + + // Connect to Docker for container monitoring and port retrieval + let docker = connect_docker(&socket)?; + let container_name = format!("{}-{}-1", self.project_name, self.gateway_service); + + // Wait for gateway status with timeout and container monitoring + let timeout_seconds = 120u64; + let launcher_status = select! { + status = watcher => status?, + _ = tokio::time::sleep(Duration::from_secs(timeout_seconds)) => { + return StatusTimeoutSnafu { seconds: timeout_seconds }.fail(); + } + _ = monitor_gateway_exit(&docker, &container_name) => { + return GatewayServiceExitedSnafu { + service: &self.gateway_service, + compose_file: self.compose_file.as_str(), + project_name: &self.project_name, + }.fail(); + } + }; + + // Get the gateway container's mapped port + let gateway_port = self + .get_gateway_port(&docker, launcher_status.gateway_port) + .await?; + + let locator = ChildLocator::Compose { + project_name: self.project_name.clone(), + compose_file: self.compose_file.to_string(), + socket, + }; + + Ok(( + guard, + NetworkInstance { + gateway_port, + root_key: hex::decode(&launcher_status.root_key).context(ParseRootKeySnafu { + key: &launcher_status.root_key, + })?, + pocketic_config_port: launcher_status.config_port, + pocketic_instance_id: launcher_status.instance_id, + }, + locator, + )) + } + + /// Get the host port mapped to the gateway container's port. + async fn get_gateway_port( + &self, + docker: &Docker, + container_port: u16, + ) -> Result { + let container_name = format!("{}-{}-1", self.project_name, self.gateway_service); + + let info = docker + .inspect_container(&container_name, None::) + .await + .context(InspectContainerSnafu { + container_id: &container_name, + })?; + + let port_bindings = info + .network_settings + .ok_or(ComposeError::RequiredFieldMissing { + field: "NetworkSettings".to_string(), + })? + .ports + .ok_or(ComposeError::RequiredFieldMissing { + field: "NetworkSettings.Ports".to_string(), + })?; + + let port_key = format!("{container_port}/tcp"); + let binding = port_bindings + .get(&port_key) + .and_then(|b| b.as_ref()) + .and_then(|b| b.first()) + .ok_or(ComposeError::RequiredFieldMissing { + field: format!("Port binding for {port_key}"), + })?; + + let host_port_str = + binding + .host_port + .as_ref() + .ok_or(ComposeError::RequiredFieldMissing { + field: "HostPort".to_string(), + })?; + + host_port_str + .parse::() + .map_err(|_| ComposeError::RequiredFieldMissing { + field: format!("Valid port number (got {host_port_str})"), + }) + } +} + +/// Resolves when the given container is no longer running. +/// Used to detect early gateway crashes during startup. +async fn monitor_gateway_exit(docker: &Docker, container_name: &str) { + loop { + tokio::time::sleep(Duration::from_secs(2)).await; + let running = match docker + .inspect_container(container_name, None::) + .await + { + Ok(info) => info.state.and_then(|s| s.running).unwrap_or(false), + Err(_) => false, + }; + if !running { + return; + } + } +} + +/// Stop a Docker Compose project. +pub async fn stop_compose(compose_file: &str, project_name: &str) -> Result<(), ComposeError> { + let output = tokio::process::Command::new("docker") + .args(["compose", "-f", compose_file, "-p", project_name, "down"]) + .output() + .await + .map_err(|e| ComposeError::StopServices { + message: e.to_string(), + })?; + + if !output.status.success() { + return Err(ComposeError::StopServices { + message: String::from_utf8_lossy(&output.stderr).to_string(), + }); + } + + eprintln!("Stopped compose project '{project_name}'"); + Ok(()) +} + +/// Get logs from a Docker Compose project. +pub async fn get_compose_logs( + compose_file: &str, + project_name: &str, + service: Option<&str>, + follow: bool, + tail: Option, +) -> Result { + let mut cmd = tokio::process::Command::new("docker"); + cmd.args(["compose", "-f", compose_file, "-p", project_name, "logs"]); + + if follow { + cmd.arg("-f"); + } + + if let Some(n) = tail { + cmd.args(["--tail", &n.to_string()]); + } + + if let Some(svc) = service { + cmd.arg(svc); + } + + cmd.stdout(std::process::Stdio::inherit()) + .stderr(std::process::Stdio::inherit()) + .spawn() + .map_err(|e| ComposeError::StartServices { + message: format!("Failed to get logs: {e}"), + }) +} + +/// Get the status of services in a Docker Compose project. +pub async fn get_compose_status( + compose_file: &str, + project_name: &str, +) -> Result, ComposeError> { + let output = tokio::process::Command::new("docker") + .args([ + "compose", + "-f", + compose_file, + "-p", + project_name, + "ps", + "--format", + "json", + ]) + .output() + .await + .map_err(|e| ComposeError::StartServices { + message: format!("Failed to get status: {e}"), + })?; + + if !output.status.success() { + return Err(ComposeError::StartServices { + message: String::from_utf8_lossy(&output.stderr).to_string(), + }); + } + + // Parse JSON output (docker compose ps --format json outputs one JSON object per line) + let stdout = String::from_utf8_lossy(&output.stdout); + let services: Vec = stdout + .lines() + .filter(|line| !line.trim().is_empty()) + .filter_map(|line| serde_json::from_str(line).ok()) + .collect(); + + Ok(services) +} + +/// Status of a single service in a Docker Compose project. +#[derive(Debug, Clone, serde::Deserialize, serde::Serialize)] +#[serde(rename_all = "PascalCase")] +pub struct ServiceStatus { + #[serde(alias = "Name")] + pub name: String, + #[serde(alias = "Service")] + pub service: String, + #[serde(alias = "State")] + pub state: String, + #[serde(alias = "Health", default)] + pub health: String, + #[serde(alias = "Image")] + pub image: String, +} + +/// Get the Docker socket path. +fn get_docker_socket() -> Result { + if let Ok(socket) = std::env::var("DOCKER_HOST") { + return Ok(socket); + } + + #[cfg(unix)] + { + let default_sock = "/var/run/docker.sock"; + if Path::new(default_sock).exists() { + return Ok(default_sock.to_string()); + } + + // Try to get socket from docker context + let output = std::process::Command::new("docker") + .args([ + "context", + "inspect", + "--format", + "{{.Endpoints.docker.Host}}", + ]) + .output() + .map_err(|e| ComposeError::StartServices { + message: format!("Failed to get docker context: {e}"), + })?; + + if output.status.success() { + return Ok(String::from_utf8_lossy(&output.stdout).trim().to_string()); + } + + Ok(default_sock.to_string()) + } + + #[cfg(windows)] + { + Ok(r"\\.\pipe\docker_engine".to_string()) + } +} + +/// Drop guard for Docker Compose projects. +#[derive(Default)] +pub struct ComposeDropGuard { + project_name: Option, + compose_file: Option, +} + +impl ComposeDropGuard { + /// Disarm the guard so it won't stop the compose project on drop. + pub fn defuse(&mut self) { + self.project_name = None; + self.compose_file = None; + } + + /// Stop the compose project. + pub async fn stop(&mut self) -> Result<(), ComposeError> { + if let (Some(project_name), Some(compose_file)) = + (self.project_name.take(), self.compose_file.take()) + { + stop_compose(compose_file.as_str(), &project_name).await?; + } + Ok(()) + } +} + +#[async_trait] +impl AsyncDrop for ComposeDropGuard { + async fn async_drop(&mut self) { + let _ = self.stop().await; + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::network::ManagedComposeConfig; + + #[test] + fn compose_network_generates_correct_project_name() { + let config = ManagedComposeConfig { + file: PathBuf::from("docker-compose.yml"), + gateway_service: "icp-network".to_string(), + environment: vec![], + }; + + let network = ComposeNetwork::new("local-bitcoin", &config, &PathBuf::from("/project")); + + assert_eq!(network.project_name, "icp-local-bitcoin"); + assert_eq!( + network.compose_file, + PathBuf::from("/project/docker-compose.yml") + ); + assert_eq!(network.gateway_service, "icp-network"); + } + + #[test] + fn compose_network_joins_relative_compose_file_with_project_root() { + let config = ManagedComposeConfig { + file: PathBuf::from("infra/docker-compose.bitcoin.yml"), + gateway_service: "gateway".to_string(), + environment: vec!["FOO=bar".to_string()], + }; + + let network = + ComposeNetwork::new("btc-test", &config, &PathBuf::from("/home/user/myproject")); + + assert_eq!( + network.compose_file, + PathBuf::from("/home/user/myproject/infra/docker-compose.bitcoin.yml") + ); + } + + #[test] + fn compose_drop_guard_defuse_clears_fields() { + let mut guard = ComposeDropGuard { + project_name: Some("test-project".to_string()), + compose_file: Some(PathBuf::from("/path/to/compose.yml")), + }; + + guard.defuse(); + + assert!(guard.project_name.is_none()); + assert!(guard.compose_file.is_none()); + } + + #[test] + fn service_status_deserializes_from_docker_compose_output() { + let json = r#"{"Name":"icp-local-1","Service":"icp-network","State":"running","Health":"","Image":"ghcr.io/dfinity/icp-cli-network-launcher:latest"}"#; + + let status: ServiceStatus = serde_json::from_str(json).unwrap(); + + assert_eq!(status.name, "icp-local-1"); + assert_eq!(status.service, "icp-network"); + assert_eq!(status.state, "running"); + assert_eq!( + status.image, + "ghcr.io/dfinity/icp-cli-network-launcher:latest" + ); + } +} diff --git a/crates/icp/src/network/managed/mod.rs b/crates/icp/src/network/managed/mod.rs index a139c413..b1e71bc8 100644 --- a/crates/icp/src/network/managed/mod.rs +++ b/crates/icp/src/network/managed/mod.rs @@ -1,4 +1,5 @@ pub mod cache; +pub mod compose; pub mod docker; pub mod launcher; pub mod run; diff --git a/crates/icp/src/network/managed/run.rs b/crates/icp/src/network/managed/run.rs index b27cdb44..535dc4bd 100644 --- a/crates/icp/src/network/managed/run.rs +++ b/crates/icp/src/network/managed/run.rs @@ -35,13 +35,14 @@ use uuid::Uuid; use crate::{ fs::{create_dir_all, lock::LockError, remove_dir_all}, network::{ - Managed, ManagedLauncherConfig, ManagedMode, NetworkDirectory, Port, + Managed, ManagedComposeConfig, ManagedLauncherConfig, ManagedMode, NetworkDirectory, Port, config::{ChildLocator, NetworkDescriptorGatewayPort, NetworkDescriptorModel}, directory::{ CheckPortInUseError, PortInUseError, SaveNetworkDescriptorError, save_network_descriptors, }, managed::{ + compose::{ComposeDropGuard, ComposeNetwork}, docker::{DockerDropGuard, ManagedImageOptions, spawn_docker_launcher}, launcher::{ChildSignalOnDrop, launcher_settings_flags, spawn_network_launcher}, }, @@ -87,6 +88,13 @@ pub async fn stop_network(locator: &ChildLocator) -> Result<(), StopNetworkError } => { super::docker::stop_docker_launcher(socket, id, *rm_on_exit).await?; } + ChildLocator::Compose { + project_name, + compose_file, + .. + } => { + super::compose::stop_compose(compose_file, project_name).await?; + } } Ok(()) } @@ -109,6 +117,11 @@ pub enum StopNetworkError { DockerLauncher { source: super::docker::StopContainerError, }, + + #[snafu(transparent)] + ComposeLauncher { + source: super::compose::ComposeError, + }, } #[allow(clippy::too_many_arguments)] @@ -129,6 +142,7 @@ async fn run_network_launcher( enum LaunchMode<'a> { Image(ManagedImageOptions), NativeLauncher(&'a ManagedLauncherConfig), + Compose(&'a ManagedComposeConfig), } let (launch_mode, fixed_ports) = match &config.mode { ManagedMode::Image(image_config) => { @@ -148,6 +162,11 @@ async fn run_network_launcher( }; (LaunchMode::NativeLauncher(launcher_config), fixed_ports) } + ManagedMode::Compose(compose_config) => { + // Compose networks don't have fixed ports configured in the manifest + // Port mapping is defined in the compose file + (LaunchMode::Compose(compose_config), vec![]) + } }; let (mut guard, instance, gateway, locator) = network_root @@ -208,6 +227,17 @@ async fn run_network_launcher( }; Ok((ShutdownGuard::Process(child), instance, gateway, locator)) } + LaunchMode::Compose(compose_config) => { + let compose_network = + ComposeNetwork::new(&nd.network_name, compose_config, project_root); + let (guard, instance, locator) = compose_network.start().await?; + let gateway = NetworkDescriptorGatewayPort { + port: instance.gateway_port, + // Compose networks manage their own port mapping + fixed: false, + }; + Ok((ShutdownGuard::Compose(guard), instance, gateway, locator)) + } } }) .await??; @@ -315,6 +345,7 @@ fn transform_native_launcher_to_container(config: &ManagedLauncherConfig) -> Man enum ShutdownGuard { Container(AsyncDropper), Process(AsyncDropper), + Compose(AsyncDropper), } impl ShutdownGuard { @@ -322,12 +353,14 @@ impl ShutdownGuard { match self { ShutdownGuard::Container(mut guard) => guard.async_drop().await, ShutdownGuard::Process(mut guard) => guard.async_drop().await, + ShutdownGuard::Compose(mut guard) => guard.async_drop().await, } } fn defuse(self) { match self { ShutdownGuard::Container(mut guard) => guard.defuse(), ShutdownGuard::Process(mut guard) => guard.defuse(), + ShutdownGuard::Compose(mut guard) => guard.defuse(), } } } @@ -383,6 +416,11 @@ pub enum RunNetworkLauncherError { SpawnDockerLauncher { source: crate::network::managed::docker::DockerLauncherError, }, + + #[snafu(transparent)] + SpawnComposeLauncher { + source: crate::network::managed::compose::ComposeError, + }, } #[derive(Debug)] @@ -400,9 +438,9 @@ fn safe_eprintln(msg: &str) { async fn wait_for_shutdown(guard: &mut ShutdownGuard) -> ShutdownReason { match guard { - ShutdownGuard::Container(_) => { + ShutdownGuard::Container(_) | ShutdownGuard::Compose(_) => { stop_signal().await; - safe_eprintln("Received Ctrl-C, shutting down PocketIC..."); + safe_eprintln("Received Ctrl-C, shutting down network..."); ShutdownReason::CtrlC } ShutdownGuard::Process(child) => { diff --git a/crates/icp/src/network/mod.rs b/crates/icp/src/network/mod.rs index d01d24cb..5b0f9d85 100644 --- a/crates/icp/src/network/mod.rs +++ b/crates/icp/src/network/mod.rs @@ -80,6 +80,7 @@ pub struct Managed { #[serde(untagged)] pub enum ManagedMode { Image(Box), + Compose(Box), Launcher(Box), } @@ -92,6 +93,19 @@ pub struct ManagedLauncherConfig { pub subnets: Option>, } +/// Configuration for Docker Compose managed networks +#[derive(Clone, Debug, Deserialize, PartialEq, JsonSchema, Serialize)] +#[serde(rename_all = "kebab-case")] +pub struct ManagedComposeConfig { + /// Path to the docker-compose.yml file (relative to project root) + #[schemars(with = "String")] + pub file: PathBuf, + /// Name of the service that runs the IC gateway + pub gateway_service: String, + /// Additional environment variables to pass to docker compose + pub environment: Vec, +} + #[derive( Clone, Debug, Deserialize, PartialEq, JsonSchema, Serialize, EnumString, strum::Display, )] @@ -263,6 +277,17 @@ impl From for Configuration { })), }, }, + crate::manifest::network::ManagedMode::Compose { compose } => { + Configuration::Managed { + managed: Managed { + mode: ManagedMode::Compose(Box::new(ManagedComposeConfig { + file: PathBuf::from(&compose.file), + gateway_service: compose.gateway_service, + environment: compose.environment, + })), + }, + } + } }, Mode::Connected(connected) => Configuration::Connected { connected: connected.into(), diff --git a/docs/guides/containerized-networks.md b/docs/guides/containerized-networks.md index c62c8a20..1a9a46aa 100644 --- a/docs/guides/containerized-networks.md +++ b/docs/guides/containerized-networks.md @@ -463,6 +463,152 @@ If you're on Windows and want to use a manually instantiated `dockerd` in a WSL2 - `ICP_CLI_DOCKER_WSL2_DISTRO=` — the WSL2 distribution name running dockerd - `DOCKER_HOST=tcp://:` — the TCP address where dockerd is listening +## Docker Compose Networks + +For more complex setups involving multiple services (like Bitcoin integration), you can use Docker Compose to manage your local network. + +### When to Use Compose + +Use Docker Compose when you need: +- Multiple services running together (e.g., Bitcoin node + IC replica) +- Complex service dependencies +- Custom networking between containers +- Services that must start in a specific order + +### Basic Configuration + +Instead of specifying an `image`, use the `compose` configuration: + +```yaml +networks: + - name: local + mode: managed + compose: + file: docker-compose.yml + gateway-service: icp-network +``` + +| Field | Type | Required | Description | +|-------------------|--------|----------|----------------------------------------------------| +| `file` | string | Yes | Path to docker-compose.yml (relative to project) | +| `gateway-service` | string | Yes | Name of the service running the IC gateway | + +### Example: Bitcoin Integration + +Here's a complete setup for local Bitcoin development: + +**docker-compose.bitcoin.yml:** +```yaml +services: + bitcoind: + image: lncm/bitcoind:v27.2 + command: + - -regtest + - -server + - -rpcbind=0.0.0.0 + - -rpcallowip=0.0.0.0/0 + - -rpcuser=ic-btc-integration + - -rpcpassword=ic-btc-integration + - -fallbackfee=0.00001 + ports: + - "18443:18443" + healthcheck: + test: ["CMD", "bitcoin-cli", "-regtest", "-rpcuser=ic-btc-integration", "-rpcpassword=ic-btc-integration", "getblockchaininfo"] + interval: 5s + timeout: 5s + retries: 20 + + icp-network: + image: ghcr.io/dfinity/icp-cli-network-launcher:latest + depends_on: + bitcoind: + condition: service_healthy + environment: + - ICP_CLI_NETWORK_LAUNCHER_INTERFACE_VERSION=1.0.0 + command: + - --bitcoind-addr=bitcoind:18444 + ports: + - "0:4943" + volumes: + - "${ICP_STATUS_DIR:-/tmp/icp-status}:/app/status" +``` + +> **Subnet configuration:** The network launcher defaults to creating an `application` subnet (plus the always-implied `nns` subnet). The `--bitcoind-addr` flag implicitly adds a `bitcoin` subnet. If you need additional subnets, use `--subnet` flags — but note that explicitly passing any `--subnet` flag **overrides the default** `application` subnet. To keep it, include `--subnet=application` explicitly. For example, to add a `system` subnet alongside bitcoin: `--subnet=application --subnet=system --bitcoind-addr=bitcoind:18444`. + +**icp.yaml:** +```yaml +networks: + - name: local + mode: managed + compose: + file: docker-compose.bitcoin.yml + gateway-service: icp-network + +environments: + - name: local + network: local + settings: + backend: + environment_variables: + BITCOIN_NETWORK: "regtest" +``` + +### How Compose Networks Work + +When you run `icp network start`: + +1. icp-cli creates a temporary status directory on the host +2. Sets the `ICP_STATUS_DIR` environment variable pointing to this directory +3. Runs `docker compose up -d` with the compose file +4. Monitors the status directory for the gateway's status file +5. Once ready, retrieves the mapped port from the gateway container + +The compose file must: +- Mount `${ICP_STATUS_DIR}` to the gateway service's status path (default `/app/status`) +- Have the gateway service write the status file when ready (same format as image-based networks) + +### Working with Bitcoin Regtest + +Generate blocks to create spendable Bitcoin: + +```bash +# Generate 101 blocks (coinbase maturity is 100 blocks) +docker compose -f docker-compose.bitcoin.yml exec bitcoind \ + bitcoin-cli -regtest -rpcuser=ic-btc-integration -rpcpassword=ic-btc-integration \ + generatetoaddress 101 "$(docker compose -f docker-compose.bitcoin.yml exec bitcoind \ + bitcoin-cli -regtest -rpcuser=ic-btc-integration -rpcpassword=ic-btc-integration getnewaddress)" +``` + +Create and fund addresses: + +```bash +# Get a new address +docker compose -f docker-compose.bitcoin.yml exec bitcoind \ + bitcoin-cli -regtest -rpcuser=ic-btc-integration -rpcpassword=ic-btc-integration \ + getnewaddress + +# Send Bitcoin to an address +docker compose -f docker-compose.bitcoin.yml exec bitcoind \ + bitcoin-cli -regtest -rpcuser=ic-btc-integration -rpcpassword=ic-btc-integration \ + sendtoaddress "bcrt1q..." 1.0 +``` + +### Bitcoin Template + +For a quick start with Bitcoin integration, use the bitcoin template: + +```bash +icp new my-bitcoin-project --template bitcoin +cd my-bitcoin-project +icp network start +icp build && icp deploy +``` + +The template includes: +- A backend canister with Bitcoin balance/UTXO query functions +- Docker Compose configuration for bitcoind + IC replica +- Environment variable configuration for network selection + ## Related Documentation - [Managing Environments](managing-environments.md) — Configure environments that use containerized networks diff --git a/docs/guides/index.md b/docs/guides/index.md index aebe9d1e..bde24930 100644 --- a/docs/guides/index.md +++ b/docs/guides/index.md @@ -20,7 +20,7 @@ Step-by-step instructions for common tasks. Each guide assumes you've completed ## Configuration -- [Containerized Networks](containerized-networks.md) — Run managed networks in Docker containers +- [Containerized Networks](containerized-networks.md) — Run managed networks in Docker containers, including Docker Compose and Bitcoin integration - [Using Recipes](using-recipes.md) — Reusable build templates for common patterns ## Advanced diff --git a/docs/reference/configuration.md b/docs/reference/configuration.md index d9da1ce5..c88b2119 100644 --- a/docs/reference/configuration.md +++ b/docs/reference/configuration.md @@ -256,6 +256,31 @@ networks: See [Containerized Networks](../guides/containerized-networks.md) for full options. +### Docker Compose Network + +For multi-container setups (like Bitcoin integration): + +```yaml +networks: + - name: local + mode: managed + compose: + file: docker-compose.yml + gateway-service: icp-network +``` + +| Property | Type | Required | Description | +|----------|------|----------|-------------| +| `file` | string | Yes | Path to docker-compose.yml (relative to project root) | +| `gateway-service` | string | Yes | Name of the service running the IC gateway | +| `environment` | array | No | Additional environment variables for docker compose | + +The compose file must: +- Have the gateway service mount `${ICP_STATUS_DIR}` to its status directory (default `/app/status`) +- Write a status file when the network is ready + +See [Docker Compose Networks](../guides/containerized-networks.md#docker-compose-networks) for examples. + ## Environments ```yaml diff --git a/docs/schemas/icp-yaml-schema.json b/docs/schemas/icp-yaml-schema.json index 08ba9d5b..321d4a68 100644 --- a/docs/schemas/icp-yaml-schema.json +++ b/docs/schemas/icp-yaml-schema.json @@ -214,6 +214,32 @@ ], "type": "object" }, + "ComposeConfig": { + "description": "Docker Compose network configuration", + "properties": { + "environment": { + "default": [], + "description": "Additional environment variables to pass to docker compose", + "items": { + "type": "string" + }, + "type": "array" + }, + "file": { + "description": "Path to the docker-compose.yml file (relative to project root)", + "type": "string" + }, + "gateway-service": { + "description": "Name of the service that runs the IC gateway", + "type": "string" + } + }, + "required": [ + "file", + "gateway-service" + ], + "type": "object" + }, "Connected": { "properties": { "root-key": { @@ -489,6 +515,18 @@ ], "type": "object" }, + { + "properties": { + "compose": { + "$ref": "#/$defs/ComposeConfig", + "description": "Docker Compose configuration" + } + }, + "required": [ + "compose" + ], + "type": "object" + }, { "properties": { "artificial-delay-ms": { diff --git a/docs/schemas/network-yaml-schema.json b/docs/schemas/network-yaml-schema.json index 9b80d66b..f13bfe0c 100644 --- a/docs/schemas/network-yaml-schema.json +++ b/docs/schemas/network-yaml-schema.json @@ -1,5 +1,31 @@ { "$defs": { + "ComposeConfig": { + "description": "Docker Compose network configuration", + "properties": { + "environment": { + "default": [], + "description": "Additional environment variables to pass to docker compose", + "items": { + "type": "string" + }, + "type": "array" + }, + "file": { + "description": "Path to the docker-compose.yml file (relative to project root)", + "type": "string" + }, + "gateway-service": { + "description": "Name of the service that runs the IC gateway", + "type": "string" + } + }, + "required": [ + "file", + "gateway-service" + ], + "type": "object" + }, "Connected": { "properties": { "root-key": { @@ -148,6 +174,18 @@ ], "type": "object" }, + { + "properties": { + "compose": { + "$ref": "#/$defs/ComposeConfig", + "description": "Docker Compose configuration" + } + }, + "required": [ + "compose" + ], + "type": "object" + }, { "properties": { "artificial-delay-ms": { From 80652a6fcaafef38dcec55d179c21789e0c22429 Mon Sep 17 00:00:00 2001 From: Marco Walz Date: Fri, 6 Feb 2026 23:04:11 +0100 Subject: [PATCH 2/6] feat: add bitcoind/dogecoind-addr config and unify launcher settings across modes --- crates/icp-cli/src/commands/network/stop.rs | 6 - .../assets/compose/docker-compose.test.yml | 11 - crates/icp-cli/tests/common/mod.rs | 16 - crates/icp-cli/tests/network_tests.rs | 167 +---- crates/icp/src/context/tests.rs | 8 + crates/icp/src/lib.rs | 4 + crates/icp/src/manifest/network.rs | 181 +++++- crates/icp/src/manifest/project.rs | 4 +- crates/icp/src/network/config.rs | 14 - crates/icp/src/network/managed/compose.rs | 576 ------------------ crates/icp/src/network/managed/docker.rs | 279 ++++++++- crates/icp/src/network/managed/launcher.rs | 130 ++++ crates/icp/src/network/managed/mod.rs | 1 - crates/icp/src/network/managed/run.rs | 180 ++++-- crates/icp/src/network/mod.rs | 150 ++++- crates/icp/src/project.rs | 2 + docs/guides/containerized-networks.md | 176 +----- docs/guides/index.md | 2 +- docs/reference/configuration.md | 74 ++- docs/schemas/icp-yaml-schema.json | 111 ++-- docs/schemas/network-yaml-schema.json | 111 ++-- 21 files changed, 1082 insertions(+), 1121 deletions(-) delete mode 100644 crates/icp-cli/tests/assets/compose/docker-compose.test.yml delete mode 100644 crates/icp/src/network/managed/compose.rs diff --git a/crates/icp-cli/src/commands/network/stop.rs b/crates/icp-cli/src/commands/network/stop.rs index a860a760..35220bea 100644 --- a/crates/icp-cli/src/commands/network/stop.rs +++ b/crates/icp-cli/src/commands/network/stop.rs @@ -65,12 +65,6 @@ pub async fn exec(ctx: &Context, cmd: &Cmd) -> Result<(), anyhow::Error> { &id[..12] )); } - ChildLocator::Compose { project_name, .. } => { - let _ = ctx.term.write_line(&format!( - "Stopping compose network (project: {})...", - project_name - )); - } } stop_network(&descriptor.child_locator).await?; diff --git a/crates/icp-cli/tests/assets/compose/docker-compose.test.yml b/crates/icp-cli/tests/assets/compose/docker-compose.test.yml deleted file mode 100644 index 767274da..00000000 --- a/crates/icp-cli/tests/assets/compose/docker-compose.test.yml +++ /dev/null @@ -1,11 +0,0 @@ -# Test compose file for icp-cli compose network testing -# This is a minimal configuration that starts only the IC network launcher -services: - icp-network: - image: ghcr.io/dfinity/icp-cli-network-launcher:v11.0.0 - ports: - - "0:4943" - environment: - - ICP_CLI_NETWORK_LAUNCHER_INTERFACE_VERSION=1.0.0 - volumes: - - "${ICP_STATUS_DIR:-/tmp/status}:/app/status" diff --git a/crates/icp-cli/tests/common/mod.rs b/crates/icp-cli/tests/common/mod.rs index 3786e5fd..7ec02d52 100644 --- a/crates/icp-cli/tests/common/mod.rs +++ b/crates/icp-cli/tests/common/mod.rs @@ -49,22 +49,6 @@ environments: network: docker-network "#; -/// A network manifest for a compose-based network -pub(crate) const NETWORK_COMPOSE: &str = r#" -networks: - - name: compose-network - mode: managed - compose: - file: compose/docker-compose.test.yml - gateway-service: icp-network -"#; - -pub(crate) const ENVIRONMENT_COMPOSE: &str = r#" -environments: - - name: compose-environment - network: compose-network -"#; - /// This ID is dependent on the toplogy being served by pocket-ic /// NOTE: If the topology is changed (another subnet is added, etc) the ID may change. /// References: diff --git a/crates/icp-cli/tests/network_tests.rs b/crates/icp-cli/tests/network_tests.rs index 9aeb9d24..b811c203 100644 --- a/crates/icp-cli/tests/network_tests.rs +++ b/crates/icp-cli/tests/network_tests.rs @@ -14,8 +14,8 @@ use serial_test::file_serial; use test_tag::tag; use crate::common::{ - ENVIRONMENT_COMPOSE, ENVIRONMENT_DOCKER, ENVIRONMENT_RANDOM_PORT, NETWORK_COMPOSE, - NETWORK_DOCKER, NETWORK_RANDOM_PORT, TestContext, TestNetwork, clients, + ENVIRONMENT_DOCKER, ENVIRONMENT_RANDOM_PORT, NETWORK_DOCKER, NETWORK_RANDOM_PORT, TestContext, + TestNetwork, clients, }; use icp::{ fs::{read_to_string, write_string}, @@ -579,169 +579,6 @@ async fn cannot_override_ic() { .stderr(contains("`ic` is a reserved network name")); } -#[tag(docker)] -#[tokio::test] -async fn network_compose() { - let ctx = TestContext::new(); - - let project_dir = ctx.create_project_dir("compose-network"); - - // Copy compose file to project - ctx.copy_asset_dir("compose", &project_dir.join("compose")); - - // Project manifest - write_string( - &project_dir.join("icp.yaml"), - &formatdoc! {r#" - {NETWORK_COMPOSE} - {ENVIRONMENT_COMPOSE} - "#}, - ) - .expect("failed to write project manifest"); - - ctx.docker_pull_network(); - let _guard = ctx.start_network_in(&project_dir, "compose-network").await; - ctx.ping_until_healthy(&project_dir, "compose-network"); - - // Verify the network descriptor has compose child locator - let descriptor_contents = ctx.read_network_descriptor(&project_dir, "compose-network"); - let descriptor: serde_json::Value = - serde_json::from_slice(&descriptor_contents).expect("descriptor should be valid JSON"); - - let child_locator = descriptor - .get("child-locator") - .expect("descriptor should have child-locator"); - - // Check compose-specific fields - assert!( - child_locator.get("project-name").is_some(), - "compose child locator should have project-name" - ); - assert!( - child_locator.get("compose-file").is_some(), - "compose child locator should have compose-file" - ); - assert!( - child_locator.get("socket").is_some(), - "compose child locator should have socket" - ); - - // Verify the project name follows the convention - let project_name = child_locator - .get("project-name") - .and_then(|p| p.as_str()) - .expect("project-name should be a string"); - assert_eq!( - project_name, "icp-compose-network", - "compose project name should be icp-" - ); - - // Verify network is healthy by checking balance - let balance = clients::ledger(&ctx) - .balance_of(Principal::anonymous(), None) - .await; - assert!(balance > 0_u128); -} - -#[tag(docker)] -#[tokio::test] -async fn network_compose_stop() { - let ctx = TestContext::new(); - - let project_dir = ctx.create_project_dir("compose-stop"); - - // Copy compose file to project - ctx.copy_asset_dir("compose", &project_dir.join("compose")); - - // Project manifest - write_string( - &project_dir.join("icp.yaml"), - &formatdoc! {r#" - {NETWORK_COMPOSE} - {ENVIRONMENT_COMPOSE} - "#}, - ) - .expect("failed to write project manifest"); - - ctx.docker_pull_network(); - - // Start network in background - ctx.icp() - .current_dir(&project_dir) - .args(["network", "start", "compose-network", "--background"]) - .assert() - .success(); - - let network = ctx.wait_for_network_descriptor(&project_dir, "compose-network"); - ctx.ping_until_healthy(&project_dir, "compose-network"); - - // Verify descriptor file exists - let descriptor_file = project_dir - .join(".icp") - .join("cache") - .join("networks") - .join("compose-network") - .join("descriptor.json"); - assert!( - descriptor_file.exists(), - "descriptor file should exist before stop" - ); - - // Stop the network - ctx.icp() - .current_dir(&project_dir) - .args(["network", "stop", "compose-network"]) - .assert() - .success() - .stdout(contains( - "Stopping compose network (project: icp-compose-network)", - )) - .stdout(contains("Network stopped successfully")); - - // Verify descriptor file is removed - assert!( - !descriptor_file.exists(), - "descriptor file should be removed after stop" - ); - - // Verify network is no longer reachable - let agent = ic_agent::Agent::builder() - .with_url(format!("http://127.0.0.1:{}", network.gateway_port)) - .build() - .expect("Failed to build agent"); - - let status_result = agent.status().await; - assert!( - status_result.is_err(), - "Network should not be reachable after stopping" - ); -} - -#[tokio::test] -async fn network_compose_file_not_found() { - let ctx = TestContext::new(); - - let project_dir = ctx.create_project_dir("compose-missing"); - - // Project manifest without copying the compose file - write_string( - &project_dir.join("icp.yaml"), - &formatdoc! {r#" - {NETWORK_COMPOSE} - {ENVIRONMENT_COMPOSE} - "#}, - ) - .expect("failed to write project manifest"); - - // Start should fail with file not found error - ctx.icp() - .current_dir(&project_dir) - .args(["network", "start", "compose-network", "--background"]) - .assert() - .failure() - .stderr(contains("docker compose file not found")); -} - #[tokio::test] #[file_serial(default_local_network)] async fn override_local_network_as_connected() { diff --git a/crates/icp/src/context/tests.rs b/crates/icp/src/context/tests.rs index f4e77d55..4e00cfa6 100644 --- a/crates/icp/src/context/tests.rs +++ b/crates/icp/src/context/tests.rs @@ -604,6 +604,8 @@ async fn test_get_agent_defaults_inside_project_with_default_local() { ii: false, nns: false, subnets: None, + bitcoind_addr: None, + dogecoind_addr: None, })), }, }, @@ -669,6 +671,8 @@ async fn test_get_agent_defaults_with_overridden_local_network() { ii: false, nns: false, subnets: None, + bitcoind_addr: None, + dogecoind_addr: None, })), }, }, @@ -736,6 +740,8 @@ async fn test_get_agent_defaults_with_overridden_local_environment() { ii: false, nns: false, subnets: None, + bitcoind_addr: None, + dogecoind_addr: None, })), }, }, @@ -754,6 +760,8 @@ async fn test_get_agent_defaults_with_overridden_local_environment() { ii: false, nns: false, subnets: None, + bitcoind_addr: None, + dogecoind_addr: None, })), }, }, diff --git a/crates/icp/src/lib.rs b/crates/icp/src/lib.rs index 1754dde5..018c01e7 100644 --- a/crates/icp/src/lib.rs +++ b/crates/icp/src/lib.rs @@ -365,6 +365,8 @@ impl MockProjectLoader { ii: false, nns: false, subnets: None, + bitcoind_addr: None, + dogecoind_addr: None, })), }, }, @@ -383,6 +385,8 @@ impl MockProjectLoader { ii: false, nns: false, subnets: None, + bitcoind_addr: None, + dogecoind_addr: None, })), }, }, diff --git a/crates/icp/src/manifest/network.rs b/crates/icp/src/manifest/network.rs index dd61fdbe..9a607e9a 100644 --- a/crates/icp/src/manifest/network.rs +++ b/crates/icp/src/manifest/network.rs @@ -55,10 +55,18 @@ pub enum ManagedMode { status_dir: Option, /// Bind mounts to add to the container in the format relative_host_path:container_path[:options] mounts: Option>, - }, - Compose { - /// Docker Compose configuration - compose: ComposeConfig, + /// Artificial delay to add to every update call + artificial_delay_ms: Option, + /// Set up the Internet Identity canister + ii: Option, + /// Set up the NNS + nns: Option, + /// Configure the list of subnets (one application subnet by default) + subnets: Option>, + /// Bitcoin P2P node addresses to connect to (e.g. "127.0.0.1:18444") + bitcoind_addr: Option>, + /// Dogecoin P2P node addresses to connect to + dogecoind_addr: Option>, }, Launcher { /// HTTP gateway configuration @@ -71,24 +79,13 @@ pub enum ManagedMode { nns: Option, /// Configure the list of subnets (one application subnet by default) subnets: Option>, + /// Bitcoin P2P node addresses to connect to (e.g. "127.0.0.1:18444") + bitcoind_addr: Option>, + /// Dogecoin P2P node addresses to connect to + dogecoind_addr: Option>, }, } -/// Docker Compose network configuration -#[derive(Clone, Debug, Deserialize, PartialEq, JsonSchema)] -#[serde(rename_all = "kebab-case")] -pub struct ComposeConfig { - /// Path to the docker-compose.yml file (relative to project root) - pub file: String, - - /// Name of the service that runs the IC gateway - pub gateway_service: String, - - /// Additional environment variables to pass to docker compose - #[serde(default)] - pub environment: Vec, -} - impl Default for ManagedMode { fn default() -> Self { ManagedMode::Launcher { @@ -97,6 +94,8 @@ impl Default for ManagedMode { ii: None, nns: None, subnets: None, + bitcoind_addr: None, + dogecoind_addr: None, } } } @@ -250,7 +249,9 @@ mod tests { artificial_delay_ms: None, ii: None, nns: None, - subnets: None + subnets: None, + bitcoind_addr: None, + dogecoind_addr: None, }) }) }, @@ -278,6 +279,8 @@ mod tests { ii: None, nns: None, subnets: None, + bitcoind_addr: None, + dogecoind_addr: None, }) }) }, @@ -306,6 +309,144 @@ mod tests { ii: None, nns: None, subnets: None, + bitcoind_addr: None, + dogecoind_addr: None, + }) + }) + }, + ); + } + + #[test] + fn managed_network_with_dogecoind_addr() { + assert_eq!( + validate_network_yaml(indoc! {r#" + name: my-network + mode: managed + dogecoind-addr: + - "127.0.0.1:22556" + "#}), + NetworkManifest { + name: "my-network".to_string(), + configuration: Mode::Managed(Managed { + mode: Box::new(ManagedMode::Launcher { + gateway: None, + artificial_delay_ms: None, + ii: None, + nns: None, + subnets: None, + bitcoind_addr: None, + dogecoind_addr: Some(vec!["127.0.0.1:22556".to_string()]), + }) + }) + }, + ); + } + + #[test] + fn managed_network_with_bitcoind_addr() { + assert_eq!( + validate_network_yaml(indoc! {r#" + name: my-network + mode: managed + bitcoind-addr: + - "127.0.0.1:18444" + "#}), + NetworkManifest { + name: "my-network".to_string(), + configuration: Mode::Managed(Managed { + mode: Box::new(ManagedMode::Launcher { + gateway: None, + artificial_delay_ms: None, + ii: None, + nns: None, + subnets: None, + bitcoind_addr: Some(vec!["127.0.0.1:18444".to_string()]), + dogecoind_addr: None, + }) + }) + }, + ); + } + + #[test] + fn image_network_with_launcher_settings() { + assert_eq!( + validate_network_yaml(indoc! {r#" + name: my-network + mode: managed + image: ghcr.io/dfinity/icp-cli-network-launcher + port-mapping: + - "8000:4943" + ii: true + nns: true + bitcoind-addr: + - "127.0.0.1:18444" + dogecoind-addr: + - "127.0.0.1:22556" + "#}), + NetworkManifest { + name: "my-network".to_string(), + configuration: Mode::Managed(Managed { + mode: Box::new(ManagedMode::Image { + image: "ghcr.io/dfinity/icp-cli-network-launcher".to_string(), + port_mapping: vec!["8000:4943".to_string()], + rm_on_exit: None, + args: None, + entrypoint: None, + environment: None, + volumes: None, + platform: None, + user: None, + shm_size: None, + status_dir: None, + mounts: None, + artificial_delay_ms: None, + ii: Some(true), + nns: Some(true), + subnets: None, + bitcoind_addr: Some(vec!["127.0.0.1:18444".to_string()]), + dogecoind_addr: Some(vec!["127.0.0.1:22556".to_string()]), + }) + }) + }, + ); + } + + #[test] + fn image_network_with_bitcoind_addr() { + assert_eq!( + validate_network_yaml(indoc! {r#" + name: my-network + mode: managed + image: ghcr.io/dfinity/icp-cli-network-launcher + port-mapping: + - "0:4943" + bitcoind-addr: + - "127.0.0.1:18444" + "#}), + NetworkManifest { + name: "my-network".to_string(), + configuration: Mode::Managed(Managed { + mode: Box::new(ManagedMode::Image { + image: "ghcr.io/dfinity/icp-cli-network-launcher".to_string(), + port_mapping: vec!["0:4943".to_string()], + rm_on_exit: None, + args: None, + entrypoint: None, + environment: None, + volumes: None, + platform: None, + user: None, + shm_size: None, + status_dir: None, + mounts: None, + artificial_delay_ms: None, + ii: None, + nns: None, + subnets: None, + bitcoind_addr: Some(vec!["127.0.0.1:18444".to_string()]), + dogecoind_addr: None, }) }) }, diff --git a/crates/icp/src/manifest/project.rs b/crates/icp/src/manifest/project.rs index bbfae484..caa146de 100644 --- a/crates/icp/src/manifest/project.rs +++ b/crates/icp/src/manifest/project.rs @@ -289,7 +289,9 @@ mod tests { artificial_delay_ms: None, ii: None, nns: None, - subnets: None + subnets: None, + bitcoind_addr: None, + dogecoind_addr: None, }), }), })], diff --git a/crates/icp/src/network/config.rs b/crates/icp/src/network/config.rs index 07692b21..6ef56bf6 100644 --- a/crates/icp/src/network/config.rs +++ b/crates/icp/src/network/config.rs @@ -88,15 +88,6 @@ pub enum ChildLocator { /// Whether to remove the container when it exits. rm_on_exit: bool, }, - /// A Docker Compose project (used for multi-container setups like Bitcoin). - Compose { - /// The compose project name (e.g., "icp-local-bitcoin"). - project_name: String, - /// Path to the docker-compose.yml file. - compose_file: String, - /// Docker socket path. - socket: String, - }, } impl ChildLocator { @@ -115,11 +106,6 @@ impl ChildLocator { ChildLocator::Container { id, socket, .. } => { crate::network::managed::docker::is_container_running(socket, id).await } - ChildLocator::Compose { - project_name, - socket, - .. - } => crate::network::managed::compose::is_compose_running(socket, project_name).await, } } } diff --git a/crates/icp/src/network/managed/compose.rs b/crates/icp/src/network/managed/compose.rs deleted file mode 100644 index 357fad55..00000000 --- a/crates/icp/src/network/managed/compose.rs +++ /dev/null @@ -1,576 +0,0 @@ -//! Docker Compose network management. -//! -//! This module handles starting, stopping, and monitoring Docker Compose-based networks. -//! Compose networks allow running multi-container setups like Bitcoin regtest alongside -//! the IC network launcher. - -use std::time::Duration; - -use async_dropper::{AsyncDrop, AsyncDropper}; -use async_trait::async_trait; -use bollard::{ - Docker, - query_parameters::{InspectContainerOptions, ListContainersOptions}, -}; -use snafu::prelude::*; -use tokio::select; - -use crate::{ - network::{ManagedComposeConfig, config::ChildLocator, managed::launcher::NetworkInstance}, - prelude::*, -}; - -use super::{docker::connect_docker, launcher::wait_for_launcher_status}; - -/// Check if a Docker Compose project is running by checking if its containers exist. -pub async fn is_compose_running(socket: &str, project_name: &str) -> bool { - let Ok(docker) = connect_docker(socket) else { - return false; - }; - - // List containers with the compose project label - let filters: std::collections::HashMap> = [( - "label".to_string(), - vec![format!("com.docker.compose.project={project_name}")], - )] - .into(); - - match docker - .list_containers(Some(ListContainersOptions { - filters: Some(filters), - ..Default::default() - })) - .await - { - Ok(containers) => { - // Check if any containers are running - containers.iter().any(|c| { - c.state - .as_ref() - .is_some_and(|s| s.to_string().to_lowercase() == "running") - }) - } - Err(_) => false, - } -} - -#[derive(Debug, Snafu)] -pub enum ComposeError { - #[snafu(display("docker compose file not found: {path}"))] - ComposeFileNotFound { path: PathBuf }, - - #[snafu(display("failed to start compose services: {message}"))] - StartServices { message: String }, - - #[snafu(display("failed to stop compose services: {message}"))] - StopServices { message: String }, - - #[snafu(display("gateway service '{service}' not found in compose project"))] - GatewayServiceNotFound { service: String }, - - #[snafu(display("timed out waiting for gateway status after {seconds} seconds"))] - StatusTimeout { seconds: u64 }, - - #[snafu(display( - "gateway service '{service}' exited unexpectedly. \ - Check logs with: docker compose -f {compose_file} -p {project_name} logs {service}" - ))] - GatewayServiceExited { - service: String, - compose_file: String, - project_name: String, - }, - - #[snafu(display("failed to read status from container: {message}"))] - ReadStatus { message: String }, - - #[snafu(transparent)] - ConnectDocker { - source: super::docker::ConnectDockerError, - }, - - #[snafu(transparent)] - WaitForLauncherStatus { - source: super::launcher::WaitForLauncherStatusError, - }, - - #[snafu(transparent)] - WatchStatusDir { - source: super::launcher::WaitForFileError, - }, - - #[snafu(display("failed to parse root key {key}"))] - ParseRootKey { - key: String, - source: hex::FromHexError, - }, - - #[snafu(display("failed to create status directory"))] - CreateStatusDir { source: std::io::Error }, - - #[snafu(display("failed to inspect container {container_id}"))] - InspectContainer { - source: bollard::errors::Error, - container_id: String, - }, - - #[snafu(display("required field {field} missing from docker API"))] - RequiredFieldMissing { field: String }, -} - -/// Manages a Docker Compose network lifecycle. -pub struct ComposeNetwork { - project_name: String, - compose_file: PathBuf, - gateway_service: String, - environment: Vec, -} - -impl ComposeNetwork { - pub fn new(network_name: &str, config: &ManagedComposeConfig, project_root: &Path) -> Self { - Self { - project_name: format!("icp-{network_name}"), - compose_file: project_root.join(&config.file), - gateway_service: config.gateway_service.clone(), - environment: config.environment.clone(), - } - } - - /// Start the Docker Compose services and wait for the gateway to be ready. - pub async fn start( - &self, - ) -> Result< - ( - AsyncDropper, - NetworkInstance, - ChildLocator, - ), - ComposeError, - > { - // Verify compose file exists - ensure!( - self.compose_file.exists(), - ComposeFileNotFoundSnafu { - path: &self.compose_file - } - ); - - // Get Docker socket - let socket = get_docker_socket()?; - - // Create a temporary directory for the status file on the host - let host_status_tmpdir = - camino_tempfile::Utf8TempDir::new().context(CreateStatusDirSnafu)?; - let host_status_dir = host_status_tmpdir.path().to_path_buf(); - - // Build environment variables for compose - let mut env_vars: Vec<(&str, &str)> = self - .environment - .iter() - .filter_map(|e| e.split_once('=')) - .collect(); - - // Add the status directory as an environment variable so compose can mount it - let status_dir_str = host_status_dir.to_string(); - env_vars.push(("ICP_STATUS_DIR", &status_dir_str)); - - // Set up the file watcher before starting compose - let watcher = wait_for_launcher_status(&host_status_dir)?; - - // Start compose services - let mut cmd = tokio::process::Command::new("docker"); - cmd.args([ - "compose", - "-f", - self.compose_file.as_str(), - "-p", - &self.project_name, - "up", - "-d", - "--wait", - ]); - - // Add environment variables - for (key, value) in &env_vars { - cmd.env(key, value); - } - - let output = cmd - .output() - .await - .map_err(|e| ComposeError::StartServices { - message: e.to_string(), - })?; - - if !output.status.success() { - return Err(ComposeError::StartServices { - message: String::from_utf8_lossy(&output.stderr).to_string(), - }); - } - - eprintln!("Started compose project '{}'", self.project_name); - - // Create drop guard - let guard = AsyncDropper::new(ComposeDropGuard { - project_name: Some(self.project_name.clone()), - compose_file: Some(self.compose_file.clone()), - }); - - // Connect to Docker for container monitoring and port retrieval - let docker = connect_docker(&socket)?; - let container_name = format!("{}-{}-1", self.project_name, self.gateway_service); - - // Wait for gateway status with timeout and container monitoring - let timeout_seconds = 120u64; - let launcher_status = select! { - status = watcher => status?, - _ = tokio::time::sleep(Duration::from_secs(timeout_seconds)) => { - return StatusTimeoutSnafu { seconds: timeout_seconds }.fail(); - } - _ = monitor_gateway_exit(&docker, &container_name) => { - return GatewayServiceExitedSnafu { - service: &self.gateway_service, - compose_file: self.compose_file.as_str(), - project_name: &self.project_name, - }.fail(); - } - }; - - // Get the gateway container's mapped port - let gateway_port = self - .get_gateway_port(&docker, launcher_status.gateway_port) - .await?; - - let locator = ChildLocator::Compose { - project_name: self.project_name.clone(), - compose_file: self.compose_file.to_string(), - socket, - }; - - Ok(( - guard, - NetworkInstance { - gateway_port, - root_key: hex::decode(&launcher_status.root_key).context(ParseRootKeySnafu { - key: &launcher_status.root_key, - })?, - pocketic_config_port: launcher_status.config_port, - pocketic_instance_id: launcher_status.instance_id, - }, - locator, - )) - } - - /// Get the host port mapped to the gateway container's port. - async fn get_gateway_port( - &self, - docker: &Docker, - container_port: u16, - ) -> Result { - let container_name = format!("{}-{}-1", self.project_name, self.gateway_service); - - let info = docker - .inspect_container(&container_name, None::) - .await - .context(InspectContainerSnafu { - container_id: &container_name, - })?; - - let port_bindings = info - .network_settings - .ok_or(ComposeError::RequiredFieldMissing { - field: "NetworkSettings".to_string(), - })? - .ports - .ok_or(ComposeError::RequiredFieldMissing { - field: "NetworkSettings.Ports".to_string(), - })?; - - let port_key = format!("{container_port}/tcp"); - let binding = port_bindings - .get(&port_key) - .and_then(|b| b.as_ref()) - .and_then(|b| b.first()) - .ok_or(ComposeError::RequiredFieldMissing { - field: format!("Port binding for {port_key}"), - })?; - - let host_port_str = - binding - .host_port - .as_ref() - .ok_or(ComposeError::RequiredFieldMissing { - field: "HostPort".to_string(), - })?; - - host_port_str - .parse::() - .map_err(|_| ComposeError::RequiredFieldMissing { - field: format!("Valid port number (got {host_port_str})"), - }) - } -} - -/// Resolves when the given container is no longer running. -/// Used to detect early gateway crashes during startup. -async fn monitor_gateway_exit(docker: &Docker, container_name: &str) { - loop { - tokio::time::sleep(Duration::from_secs(2)).await; - let running = match docker - .inspect_container(container_name, None::) - .await - { - Ok(info) => info.state.and_then(|s| s.running).unwrap_or(false), - Err(_) => false, - }; - if !running { - return; - } - } -} - -/// Stop a Docker Compose project. -pub async fn stop_compose(compose_file: &str, project_name: &str) -> Result<(), ComposeError> { - let output = tokio::process::Command::new("docker") - .args(["compose", "-f", compose_file, "-p", project_name, "down"]) - .output() - .await - .map_err(|e| ComposeError::StopServices { - message: e.to_string(), - })?; - - if !output.status.success() { - return Err(ComposeError::StopServices { - message: String::from_utf8_lossy(&output.stderr).to_string(), - }); - } - - eprintln!("Stopped compose project '{project_name}'"); - Ok(()) -} - -/// Get logs from a Docker Compose project. -pub async fn get_compose_logs( - compose_file: &str, - project_name: &str, - service: Option<&str>, - follow: bool, - tail: Option, -) -> Result { - let mut cmd = tokio::process::Command::new("docker"); - cmd.args(["compose", "-f", compose_file, "-p", project_name, "logs"]); - - if follow { - cmd.arg("-f"); - } - - if let Some(n) = tail { - cmd.args(["--tail", &n.to_string()]); - } - - if let Some(svc) = service { - cmd.arg(svc); - } - - cmd.stdout(std::process::Stdio::inherit()) - .stderr(std::process::Stdio::inherit()) - .spawn() - .map_err(|e| ComposeError::StartServices { - message: format!("Failed to get logs: {e}"), - }) -} - -/// Get the status of services in a Docker Compose project. -pub async fn get_compose_status( - compose_file: &str, - project_name: &str, -) -> Result, ComposeError> { - let output = tokio::process::Command::new("docker") - .args([ - "compose", - "-f", - compose_file, - "-p", - project_name, - "ps", - "--format", - "json", - ]) - .output() - .await - .map_err(|e| ComposeError::StartServices { - message: format!("Failed to get status: {e}"), - })?; - - if !output.status.success() { - return Err(ComposeError::StartServices { - message: String::from_utf8_lossy(&output.stderr).to_string(), - }); - } - - // Parse JSON output (docker compose ps --format json outputs one JSON object per line) - let stdout = String::from_utf8_lossy(&output.stdout); - let services: Vec = stdout - .lines() - .filter(|line| !line.trim().is_empty()) - .filter_map(|line| serde_json::from_str(line).ok()) - .collect(); - - Ok(services) -} - -/// Status of a single service in a Docker Compose project. -#[derive(Debug, Clone, serde::Deserialize, serde::Serialize)] -#[serde(rename_all = "PascalCase")] -pub struct ServiceStatus { - #[serde(alias = "Name")] - pub name: String, - #[serde(alias = "Service")] - pub service: String, - #[serde(alias = "State")] - pub state: String, - #[serde(alias = "Health", default)] - pub health: String, - #[serde(alias = "Image")] - pub image: String, -} - -/// Get the Docker socket path. -fn get_docker_socket() -> Result { - if let Ok(socket) = std::env::var("DOCKER_HOST") { - return Ok(socket); - } - - #[cfg(unix)] - { - let default_sock = "/var/run/docker.sock"; - if Path::new(default_sock).exists() { - return Ok(default_sock.to_string()); - } - - // Try to get socket from docker context - let output = std::process::Command::new("docker") - .args([ - "context", - "inspect", - "--format", - "{{.Endpoints.docker.Host}}", - ]) - .output() - .map_err(|e| ComposeError::StartServices { - message: format!("Failed to get docker context: {e}"), - })?; - - if output.status.success() { - return Ok(String::from_utf8_lossy(&output.stdout).trim().to_string()); - } - - Ok(default_sock.to_string()) - } - - #[cfg(windows)] - { - Ok(r"\\.\pipe\docker_engine".to_string()) - } -} - -/// Drop guard for Docker Compose projects. -#[derive(Default)] -pub struct ComposeDropGuard { - project_name: Option, - compose_file: Option, -} - -impl ComposeDropGuard { - /// Disarm the guard so it won't stop the compose project on drop. - pub fn defuse(&mut self) { - self.project_name = None; - self.compose_file = None; - } - - /// Stop the compose project. - pub async fn stop(&mut self) -> Result<(), ComposeError> { - if let (Some(project_name), Some(compose_file)) = - (self.project_name.take(), self.compose_file.take()) - { - stop_compose(compose_file.as_str(), &project_name).await?; - } - Ok(()) - } -} - -#[async_trait] -impl AsyncDrop for ComposeDropGuard { - async fn async_drop(&mut self) { - let _ = self.stop().await; - } -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::network::ManagedComposeConfig; - - #[test] - fn compose_network_generates_correct_project_name() { - let config = ManagedComposeConfig { - file: PathBuf::from("docker-compose.yml"), - gateway_service: "icp-network".to_string(), - environment: vec![], - }; - - let network = ComposeNetwork::new("local-bitcoin", &config, &PathBuf::from("/project")); - - assert_eq!(network.project_name, "icp-local-bitcoin"); - assert_eq!( - network.compose_file, - PathBuf::from("/project/docker-compose.yml") - ); - assert_eq!(network.gateway_service, "icp-network"); - } - - #[test] - fn compose_network_joins_relative_compose_file_with_project_root() { - let config = ManagedComposeConfig { - file: PathBuf::from("infra/docker-compose.bitcoin.yml"), - gateway_service: "gateway".to_string(), - environment: vec!["FOO=bar".to_string()], - }; - - let network = - ComposeNetwork::new("btc-test", &config, &PathBuf::from("/home/user/myproject")); - - assert_eq!( - network.compose_file, - PathBuf::from("/home/user/myproject/infra/docker-compose.bitcoin.yml") - ); - } - - #[test] - fn compose_drop_guard_defuse_clears_fields() { - let mut guard = ComposeDropGuard { - project_name: Some("test-project".to_string()), - compose_file: Some(PathBuf::from("/path/to/compose.yml")), - }; - - guard.defuse(); - - assert!(guard.project_name.is_none()); - assert!(guard.compose_file.is_none()); - } - - #[test] - fn service_status_deserializes_from_docker_compose_output() { - let json = r#"{"Name":"icp-local-1","Service":"icp-network","State":"running","Health":"","Image":"ghcr.io/dfinity/icp-cli-network-launcher:latest"}"#; - - let status: ServiceStatus = serde_json::from_str(json).unwrap(); - - assert_eq!(status.name, "icp-local-1"); - assert_eq!(status.service, "icp-network"); - assert_eq!(status.state, "running"); - assert_eq!( - status.image, - "ghcr.io/dfinity/icp-cli-network-launcher:latest" - ); - } -} diff --git a/crates/icp/src/network/managed/docker.rs b/crates/icp/src/network/managed/docker.rs index a73685cc..6cd54f2b 100644 --- a/crates/icp/src/network/managed/docker.rs +++ b/crates/icp/src/network/managed/docker.rs @@ -20,10 +20,13 @@ use tokio::select; use wslpath2::Conversion; use crate::network::{ - ManagedImageConfig, config::ChildLocator, managed::launcher::NetworkInstance, + Gateway, ManagedImageConfig, ManagedLauncherConfig, config::ChildLocator, + managed::launcher::NetworkInstance, }; use crate::prelude::*; +use super::launcher::launcher_settings_flags; + use super::launcher::wait_for_launcher_status; /// Error converting a path for WSL2. @@ -52,6 +55,8 @@ pub struct ManagedImageOptions { pub status_dir: String, /// Parsed mounts (excluding the status directory mount, which is added at runtime). pub mounts: Vec, + /// Extra hosts entries for Docker networking (e.g. "host.docker.internal:host-gateway"). + pub extra_hosts: Vec, } impl ManagedImageOptions { @@ -147,11 +152,38 @@ impl TryFrom<&ManagedImageConfig> for ManagedImageOptions { .map(|v| convert_volume(wsl2_convert, wsl2_distro, v)) .try_collect()?; + // Generate launcher settings flags from semantic fields + let launcher_config = ManagedLauncherConfig { + gateway: Gateway::default(), + artificial_delay_ms: config.artificial_delay_ms, + ii: config.ii, + nns: config.nns, + subnets: config.subnets.clone(), + bitcoind_addr: config.bitcoind_addr.clone(), + dogecoind_addr: config.dogecoind_addr.clone(), + }; + let launcher_flags = + translate_launcher_args_for_docker(launcher_settings_flags(&launcher_config)); + + // Append launcher flags to explicit user args + let mut all_args = config.args.clone(); + all_args.extend(launcher_flags); + + // Compute extra_hosts for Docker networking + let all_addrs: Vec = config + .bitcoind_addr + .iter() + .chain(config.dogecoind_addr.iter()) + .flatten() + .cloned() + .collect(); + let extra_hosts = docker_extra_hosts_for_addrs(&all_addrs); + Ok(ManagedImageOptions { image: config.image.clone(), port_bindings, rm_on_exit: config.rm_on_exit, - args: config.args.clone(), + args: all_args, entrypoint: config.entrypoint.clone(), environment: config.environment.clone(), volumes, @@ -160,6 +192,7 @@ impl TryFrom<&ManagedImageConfig> for ManagedImageOptions { shm_size: config.shm_size, status_dir: config.status_dir.clone(), mounts, + extra_hosts, }) } } @@ -187,6 +220,52 @@ pub enum ManagedImageConversionError { WslPathConvert { source: WslPathConversionError }, } +/// Translates a host:port address for use inside a Docker container. +/// Replaces `127.0.0.1`, `localhost`, and `::1` with `host.docker.internal` +/// so the container can reach services running on the host machine. +pub(super) fn translate_addr_for_docker(addr: &str) -> String { + if let Some((host, port)) = addr.rsplit_once(':') { + let translated = match host { + "127.0.0.1" | "localhost" | "::1" => "host.docker.internal", + _ => host, + }; + format!("{translated}:{port}") + } else { + addr.to_string() + } +} + +/// Returns extra_hosts entries needed for Docker to resolve `host.docker.internal`. +/// On Linux, Docker Engine does not provide `host.docker.internal` by default, +/// so we add `host.docker.internal:host-gateway` when any addresses reference localhost. +pub(super) fn docker_extra_hosts_for_addrs(addrs: &[String]) -> Vec { + let needs_host_gateway = addrs.iter().any(|addr| { + addr.rsplit_once(':') + .map(|(host, _)| matches!(host, "127.0.0.1" | "localhost" | "::1")) + .unwrap_or(false) + }); + if needs_host_gateway { + vec!["host.docker.internal:host-gateway".to_string()] + } else { + vec![] + } +} + +/// Translates localhost addresses in `--bitcoind-addr=` and `--dogecoind-addr=` flags +/// for use inside a Docker container. +pub(super) fn translate_launcher_args_for_docker(args: Vec) -> Vec { + args.into_iter() + .map(|arg| { + for prefix in ["--bitcoind-addr=", "--dogecoind-addr="] { + if let Some(addr) = arg.strip_prefix(prefix) { + return format!("{prefix}{}", translate_addr_for_docker(addr)); + } + } + arg + }) + .collect() +} + pub async fn spawn_docker_launcher( options: &ManagedImageOptions, ) -> Result< @@ -211,6 +290,7 @@ pub async fn spawn_docker_launcher( shm_size, status_dir, mounts, + extra_hosts, } = options; // Create status tmpdir and convert path for WSL2 if needed @@ -320,6 +400,11 @@ pub async fn spawn_docker_launcher( mounts: Some(all_mounts), binds: Some(volumes.clone()), shm_size: *shm_size, + extra_hosts: if extra_hosts.is_empty() { + None + } else { + Some(extra_hosts.clone()) + }, ..<_>::default() }), ..<_>::default() @@ -700,3 +785,193 @@ impl AsyncDrop for DockerDropGuard { _ = self.stop().await; } } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn translate_addr_localhost_ipv4() { + assert_eq!( + translate_addr_for_docker("127.0.0.1:18444"), + "host.docker.internal:18444" + ); + } + + #[test] + fn translate_addr_localhost_name() { + assert_eq!( + translate_addr_for_docker("localhost:18444"), + "host.docker.internal:18444" + ); + } + + #[test] + fn translate_addr_localhost_ipv6() { + assert_eq!( + translate_addr_for_docker("::1:18444"), + "host.docker.internal:18444" + ); + } + + #[test] + fn translate_addr_remote_unchanged() { + assert_eq!( + translate_addr_for_docker("192.168.1.5:18444"), + "192.168.1.5:18444" + ); + } + + #[test] + fn translate_addr_hostname_unchanged() { + assert_eq!( + translate_addr_for_docker("my-bitcoin-node:18444"), + "my-bitcoin-node:18444" + ); + } + + #[test] + fn translate_addr_no_port_unchanged() { + assert_eq!(translate_addr_for_docker("just-a-string"), "just-a-string"); + } + + #[test] + fn extra_hosts_for_localhost_addr() { + let addrs = vec!["127.0.0.1:18444".to_string()]; + assert_eq!( + docker_extra_hosts_for_addrs(&addrs), + vec!["host.docker.internal:host-gateway"] + ); + } + + #[test] + fn extra_hosts_for_remote_addr() { + let addrs = vec!["192.168.1.5:18444".to_string()]; + assert!(docker_extra_hosts_for_addrs(&addrs).is_empty()); + } + + #[test] + fn extra_hosts_for_mixed_addrs() { + let addrs = vec![ + "127.0.0.1:18444".to_string(), + "192.168.1.5:22556".to_string(), + ]; + assert_eq!( + docker_extra_hosts_for_addrs(&addrs), + vec!["host.docker.internal:host-gateway"] + ); + } + + #[test] + fn extra_hosts_for_empty_addrs() { + let addrs: Vec = vec![]; + assert!(docker_extra_hosts_for_addrs(&addrs).is_empty()); + } + + #[test] + fn translate_launcher_args_bitcoind() { + let args = vec![ + "--ii".to_string(), + "--bitcoind-addr=127.0.0.1:18444".to_string(), + ]; + assert_eq!( + translate_launcher_args_for_docker(args), + vec![ + "--ii".to_string(), + "--bitcoind-addr=host.docker.internal:18444".to_string(), + ] + ); + } + + #[test] + fn translate_launcher_args_dogecoind() { + let args = vec!["--dogecoind-addr=localhost:22556".to_string()]; + assert_eq!( + translate_launcher_args_for_docker(args), + vec!["--dogecoind-addr=host.docker.internal:22556".to_string()] + ); + } + + #[test] + fn translate_launcher_args_non_addr_unchanged() { + let args = vec![ + "--nns".to_string(), + "--artificial-delay-ms=100".to_string(), + "--subnet=bitcoin".to_string(), + ]; + let expected = args.clone(); + assert_eq!(translate_launcher_args_for_docker(args), expected); + } + + #[test] + fn image_config_conversion_with_launcher_settings() { + let config = ManagedImageConfig { + image: "ghcr.io/dfinity/icp-cli-network-launcher".to_string(), + port_mapping: vec!["8000:4943".to_string()], + rm_on_exit: false, + args: vec!["--custom-flag".to_string()], + entrypoint: None, + environment: vec![], + volumes: vec![], + platform: None, + user: None, + shm_size: None, + status_dir: "/app/status".to_string(), + mounts: vec![], + artificial_delay_ms: None, + ii: true, + nns: false, + subnets: None, + bitcoind_addr: Some(vec!["127.0.0.1:18444".to_string()]), + dogecoind_addr: None, + }; + let options = ManagedImageOptions::try_from(&config).unwrap(); + + // User args come first, then generated launcher flags + assert!(options.args.contains(&"--custom-flag".to_string())); + assert!(options.args.contains(&"--ii".to_string())); + assert!( + options + .args + .contains(&"--bitcoind-addr=host.docker.internal:18444".to_string()) + ); + + // extra_hosts for Linux Docker compatibility + assert_eq!( + options.extra_hosts, + vec!["host.docker.internal:host-gateway"] + ); + } + + #[test] + fn image_config_conversion_without_localhost_no_extra_hosts() { + let config = ManagedImageConfig { + image: "ghcr.io/dfinity/icp-cli-network-launcher".to_string(), + port_mapping: vec!["8000:4943".to_string()], + rm_on_exit: false, + args: vec![], + entrypoint: None, + environment: vec![], + volumes: vec![], + platform: None, + user: None, + shm_size: None, + status_dir: "/app/status".to_string(), + mounts: vec![], + artificial_delay_ms: None, + ii: false, + nns: false, + subnets: None, + bitcoind_addr: Some(vec!["192.168.1.5:18444".to_string()]), + dogecoind_addr: None, + }; + let options = ManagedImageOptions::try_from(&config).unwrap(); + + assert!( + options + .args + .contains(&"--bitcoind-addr=192.168.1.5:18444".to_string()) + ); + assert!(options.extra_hosts.is_empty()); + } +} diff --git a/crates/icp/src/network/managed/launcher.rs b/crates/icp/src/network/managed/launcher.rs index f69b3d4c..d7eb2cd6 100644 --- a/crates/icp/src/network/managed/launcher.rs +++ b/crates/icp/src/network/managed/launcher.rs @@ -174,6 +174,8 @@ pub fn launcher_settings_flags(config: &ManagedLauncherConfig) -> Vec { ii, nns, subnets, + bitcoind_addr, + dogecoind_addr, } = config; let mut flags = vec![]; if *ii { @@ -190,6 +192,16 @@ pub fn launcher_settings_flags(config: &ManagedLauncherConfig) -> Vec { flags.push(format!("--subnet={subnet}")); } } + if let Some(addrs) = &bitcoind_addr { + for addr in addrs { + flags.push(format!("--bitcoind-addr={addr}")); + } + } + if let Some(addrs) = &dogecoind_addr { + for addr in addrs { + flags.push(format!("--dogecoind-addr={addr}")); + } + } flags } @@ -325,6 +337,124 @@ pub fn wait_for_launcher_status( }) } +#[cfg(test)] +mod tests { + use super::*; + use crate::network::{Gateway, Port, SubnetKind}; + + fn default_config() -> ManagedLauncherConfig { + ManagedLauncherConfig { + gateway: Gateway::default(), + artificial_delay_ms: None, + ii: false, + nns: false, + subnets: None, + bitcoind_addr: None, + dogecoind_addr: None, + } + } + + #[test] + fn flags_default_config_empty() { + let config = default_config(); + assert!(launcher_settings_flags(&config).is_empty()); + } + + #[test] + fn flags_ii() { + let config = ManagedLauncherConfig { + ii: true, + ..default_config() + }; + assert_eq!(launcher_settings_flags(&config), vec!["--ii"]); + } + + #[test] + fn flags_nns() { + let config = ManagedLauncherConfig { + nns: true, + ..default_config() + }; + assert_eq!(launcher_settings_flags(&config), vec!["--nns"]); + } + + #[test] + fn flags_artificial_delay() { + let config = ManagedLauncherConfig { + artificial_delay_ms: Some(100), + ..default_config() + }; + assert_eq!( + launcher_settings_flags(&config), + vec!["--artificial-delay-ms=100"] + ); + } + + #[test] + fn flags_subnets() { + let config = ManagedLauncherConfig { + subnets: Some(vec![SubnetKind::Application, SubnetKind::Bitcoin]), + ..default_config() + }; + assert_eq!( + launcher_settings_flags(&config), + vec!["--subnet=application", "--subnet=bitcoin"] + ); + } + + #[test] + fn flags_bitcoind_addr() { + let config = ManagedLauncherConfig { + bitcoind_addr: Some(vec!["127.0.0.1:18444".to_string()]), + ..default_config() + }; + assert_eq!( + launcher_settings_flags(&config), + vec!["--bitcoind-addr=127.0.0.1:18444"] + ); + } + + #[test] + fn flags_dogecoind_addr() { + let config = ManagedLauncherConfig { + dogecoind_addr: Some(vec!["127.0.0.1:22556".to_string()]), + ..default_config() + }; + assert_eq!( + launcher_settings_flags(&config), + vec!["--dogecoind-addr=127.0.0.1:22556"] + ); + } + + #[test] + fn flags_full_config() { + let config = ManagedLauncherConfig { + gateway: Gateway { + host: "localhost".to_string(), + port: Port::Fixed(8000), + }, + artificial_delay_ms: Some(50), + ii: true, + nns: true, + subnets: Some(vec![SubnetKind::Application]), + bitcoind_addr: Some(vec!["127.0.0.1:18444".to_string()]), + dogecoind_addr: Some(vec!["127.0.0.1:22556".to_string()]), + }; + let flags = launcher_settings_flags(&config); + assert_eq!( + flags, + vec![ + "--ii", + "--nns", + "--artificial-delay-ms=50", + "--subnet=application", + "--bitcoind-addr=127.0.0.1:18444", + "--dogecoind-addr=127.0.0.1:22556", + ] + ); + } +} + #[derive(Debug, Snafu)] pub enum WaitForLauncherStatusError { WaitForFile { source: WaitForFileError }, diff --git a/crates/icp/src/network/managed/mod.rs b/crates/icp/src/network/managed/mod.rs index b1e71bc8..a139c413 100644 --- a/crates/icp/src/network/managed/mod.rs +++ b/crates/icp/src/network/managed/mod.rs @@ -1,5 +1,4 @@ pub mod cache; -pub mod compose; pub mod docker; pub mod launcher; pub mod run; diff --git a/crates/icp/src/network/managed/run.rs b/crates/icp/src/network/managed/run.rs index 535dc4bd..bb792976 100644 --- a/crates/icp/src/network/managed/run.rs +++ b/crates/icp/src/network/managed/run.rs @@ -35,14 +35,13 @@ use uuid::Uuid; use crate::{ fs::{create_dir_all, lock::LockError, remove_dir_all}, network::{ - Managed, ManagedComposeConfig, ManagedLauncherConfig, ManagedMode, NetworkDirectory, Port, + Managed, ManagedLauncherConfig, ManagedMode, NetworkDirectory, Port, config::{ChildLocator, NetworkDescriptorGatewayPort, NetworkDescriptorModel}, directory::{ CheckPortInUseError, PortInUseError, SaveNetworkDescriptorError, save_network_descriptors, }, managed::{ - compose::{ComposeDropGuard, ComposeNetwork}, docker::{DockerDropGuard, ManagedImageOptions, spawn_docker_launcher}, launcher::{ChildSignalOnDrop, launcher_settings_flags, spawn_network_launcher}, }, @@ -88,13 +87,6 @@ pub async fn stop_network(locator: &ChildLocator) -> Result<(), StopNetworkError } => { super::docker::stop_docker_launcher(socket, id, *rm_on_exit).await?; } - ChildLocator::Compose { - project_name, - compose_file, - .. - } => { - super::compose::stop_compose(compose_file, project_name).await?; - } } Ok(()) } @@ -117,11 +109,6 @@ pub enum StopNetworkError { DockerLauncher { source: super::docker::StopContainerError, }, - - #[snafu(transparent)] - ComposeLauncher { - source: super::compose::ComposeError, - }, } #[allow(clippy::too_many_arguments)] @@ -142,7 +129,6 @@ async fn run_network_launcher( enum LaunchMode<'a> { Image(ManagedImageOptions), NativeLauncher(&'a ManagedLauncherConfig), - Compose(&'a ManagedComposeConfig), } let (launch_mode, fixed_ports) = match &config.mode { ManagedMode::Image(image_config) => { @@ -162,11 +148,6 @@ async fn run_network_launcher( }; (LaunchMode::NativeLauncher(launcher_config), fixed_ports) } - ManagedMode::Compose(compose_config) => { - // Compose networks don't have fixed ports configured in the manifest - // Port mapping is defined in the compose file - (LaunchMode::Compose(compose_config), vec![]) - } }; let (mut guard, instance, gateway, locator) = network_root @@ -227,17 +208,6 @@ async fn run_network_launcher( }; Ok((ShutdownGuard::Process(child), instance, gateway, locator)) } - LaunchMode::Compose(compose_config) => { - let compose_network = - ComposeNetwork::new(&nd.network_name, compose_config, project_root); - let (guard, instance, locator) = compose_network.start().await?; - let gateway = NetworkDescriptorGatewayPort { - port: instance.gateway_port, - // Compose networks manage their own port mapping - fixed: false, - }; - Ok((ShutdownGuard::Compose(guard), instance, gateway, locator)) - } } }) .await??; @@ -305,11 +275,23 @@ fn transform_native_launcher_to_container(config: &ManagedLauncherConfig) -> Man use bollard::secret::PortBinding; use std::collections::HashMap; + use super::docker::{docker_extra_hosts_for_addrs, translate_launcher_args_for_docker}; + let port = match config.gateway.port { Port::Fixed(port) => port, Port::Random => 0, }; let args = launcher_settings_flags(config); + let args = translate_launcher_args_for_docker(args); + + let all_addrs: Vec = config + .bitcoind_addr + .iter() + .chain(config.dogecoind_addr.iter()) + .flatten() + .cloned() + .collect(); + let extra_hosts = docker_extra_hosts_for_addrs(&all_addrs); let platform = if cfg!(target_arch = "aarch64") { "linux/arm64".to_string() @@ -339,13 +321,13 @@ fn transform_native_launcher_to_container(config: &ManagedLauncherConfig) -> Man shm_size: None, status_dir: "/app/status".to_string(), mounts: vec![], + extra_hosts, } } enum ShutdownGuard { Container(AsyncDropper), Process(AsyncDropper), - Compose(AsyncDropper), } impl ShutdownGuard { @@ -353,14 +335,12 @@ impl ShutdownGuard { match self { ShutdownGuard::Container(mut guard) => guard.async_drop().await, ShutdownGuard::Process(mut guard) => guard.async_drop().await, - ShutdownGuard::Compose(mut guard) => guard.async_drop().await, } } fn defuse(self) { match self { ShutdownGuard::Container(mut guard) => guard.defuse(), ShutdownGuard::Process(mut guard) => guard.defuse(), - ShutdownGuard::Compose(mut guard) => guard.defuse(), } } } @@ -416,11 +396,6 @@ pub enum RunNetworkLauncherError { SpawnDockerLauncher { source: crate::network::managed::docker::DockerLauncherError, }, - - #[snafu(transparent)] - SpawnComposeLauncher { - source: crate::network::managed::compose::ComposeError, - }, } #[derive(Debug)] @@ -438,9 +413,9 @@ fn safe_eprintln(msg: &str) { async fn wait_for_shutdown(guard: &mut ShutdownGuard) -> ShutdownReason { match guard { - ShutdownGuard::Container(_) | ShutdownGuard::Compose(_) => { + ShutdownGuard::Container(_) => { stop_signal().await; - safe_eprintln("Received Ctrl-C, shutting down network..."); + safe_eprintln("Received Ctrl-C, shutting down PocketIC..."); ShutdownReason::CtrlC } ShutdownGuard::Process(child) => { @@ -785,3 +760,126 @@ async fn install_candid_ui( Ok(canister_id) } + +#[cfg(test)] +mod tests { + use super::*; + use crate::network::{Gateway, ManagedLauncherConfig, Port}; + + #[test] + fn transform_native_launcher_default_config() { + let config = ManagedLauncherConfig { + gateway: Gateway { + host: "localhost".to_string(), + port: Port::Fixed(8000), + }, + artificial_delay_ms: None, + ii: false, + nns: false, + subnets: None, + bitcoind_addr: None, + dogecoind_addr: None, + }; + let opts = transform_native_launcher_to_container(&config); + assert_eq!( + opts.image, + "ghcr.io/dfinity/icp-cli-network-launcher:latest" + ); + assert!(opts.args.is_empty()); + assert!(opts.extra_hosts.is_empty()); + assert!(opts.rm_on_exit); + assert_eq!(opts.status_dir, "/app/status"); + // Port binding maps host 8000 to container 4943 + let binding = opts.port_bindings.get("4943/tcp").unwrap().as_ref().unwrap(); + assert_eq!(binding[0].host_port.as_deref(), Some("8000")); + } + + #[test] + fn transform_native_launcher_random_port() { + let config = ManagedLauncherConfig { + gateway: Gateway { + host: "localhost".to_string(), + port: Port::Random, + }, + artificial_delay_ms: None, + ii: false, + nns: false, + subnets: None, + bitcoind_addr: None, + dogecoind_addr: None, + }; + let opts = transform_native_launcher_to_container(&config); + let binding = opts.port_bindings.get("4943/tcp").unwrap().as_ref().unwrap(); + assert_eq!(binding[0].host_port.as_deref(), Some("0")); + } + + #[test] + fn transform_native_launcher_with_bitcoind_addr() { + let config = ManagedLauncherConfig { + gateway: Gateway::default(), + artificial_delay_ms: None, + ii: true, + nns: false, + subnets: None, + bitcoind_addr: Some(vec!["127.0.0.1:18444".to_string()]), + dogecoind_addr: None, + }; + let opts = transform_native_launcher_to_container(&config); + // --ii flag should be present + assert!(opts.args.contains(&"--ii".to_string())); + // bitcoind-addr should be translated for Docker + assert!(opts + .args + .contains(&"--bitcoind-addr=host.docker.internal:18444".to_string())); + // extra_hosts should include host.docker.internal mapping + assert_eq!( + opts.extra_hosts, + vec!["host.docker.internal:host-gateway".to_string()] + ); + } + + #[test] + fn transform_native_launcher_with_dogecoind_addr() { + let config = ManagedLauncherConfig { + gateway: Gateway::default(), + artificial_delay_ms: Some(50), + ii: false, + nns: true, + subnets: None, + bitcoind_addr: None, + dogecoind_addr: Some(vec!["localhost:22556".to_string()]), + }; + let opts = transform_native_launcher_to_container(&config); + assert!(opts.args.contains(&"--nns".to_string())); + assert!(opts + .args + .contains(&"--artificial-delay-ms=50".to_string())); + assert!(opts + .args + .contains(&"--dogecoind-addr=host.docker.internal:22556".to_string())); + assert_eq!( + opts.extra_hosts, + vec!["host.docker.internal:host-gateway".to_string()] + ); + } + + #[test] + fn transform_native_launcher_external_addr_no_extra_hosts() { + let config = ManagedLauncherConfig { + gateway: Gateway::default(), + artificial_delay_ms: None, + ii: false, + nns: false, + subnets: None, + bitcoind_addr: Some(vec!["192.168.1.5:18444".to_string()]), + dogecoind_addr: None, + }; + let opts = transform_native_launcher_to_container(&config); + // External address should pass through unchanged + assert!(opts + .args + .contains(&"--bitcoind-addr=192.168.1.5:18444".to_string())); + // No extra_hosts needed for external addresses + assert!(opts.extra_hosts.is_empty()); + } +} diff --git a/crates/icp/src/network/mod.rs b/crates/icp/src/network/mod.rs index 5b0f9d85..cce1300d 100644 --- a/crates/icp/src/network/mod.rs +++ b/crates/icp/src/network/mod.rs @@ -80,7 +80,6 @@ pub struct Managed { #[serde(untagged)] pub enum ManagedMode { Image(Box), - Compose(Box), Launcher(Box), } @@ -91,19 +90,8 @@ pub struct ManagedLauncherConfig { pub ii: bool, pub nns: bool, pub subnets: Option>, -} - -/// Configuration for Docker Compose managed networks -#[derive(Clone, Debug, Deserialize, PartialEq, JsonSchema, Serialize)] -#[serde(rename_all = "kebab-case")] -pub struct ManagedComposeConfig { - /// Path to the docker-compose.yml file (relative to project root) - #[schemars(with = "String")] - pub file: PathBuf, - /// Name of the service that runs the IC gateway - pub gateway_service: String, - /// Additional environment variables to pass to docker compose - pub environment: Vec, + pub bitcoind_addr: Option>, + pub dogecoind_addr: Option>, } #[derive( @@ -142,6 +130,8 @@ impl ManagedMode { ii: false, nns: false, subnets: None, + bitcoind_addr: None, + dogecoind_addr: None, })) } } @@ -160,6 +150,12 @@ pub struct ManagedImageConfig { pub shm_size: Option, pub status_dir: String, pub mounts: Vec, + pub artificial_delay_ms: Option, + pub ii: bool, + pub nns: bool, + pub subnets: Option>, + pub bitcoind_addr: Option>, + pub dogecoind_addr: Option>, } #[derive(Clone, Debug, Deserialize, PartialEq, JsonSchema, Serialize)] @@ -229,6 +225,8 @@ impl From for Configuration { ii, nns, subnets, + bitcoind_addr, + dogecoind_addr, } => { let gateway: Gateway = match gateway { Some(g) => g.into(), @@ -242,6 +240,8 @@ impl From for Configuration { ii: ii.unwrap_or(false), nns: nns.unwrap_or(false), subnets, + bitcoind_addr, + dogecoind_addr, })), }, } @@ -259,6 +259,12 @@ impl From for Configuration { shm_size, status_dir, mounts: mount, + artificial_delay_ms, + ii, + nns, + subnets, + bitcoind_addr, + dogecoind_addr, } => Configuration::Managed { managed: Managed { mode: ManagedMode::Image(Box::new(ManagedImageConfig { @@ -274,20 +280,15 @@ impl From for Configuration { shm_size, status_dir: status_dir.unwrap_or_else(|| "/app/status".to_string()), mounts: mount.unwrap_or_default(), + artificial_delay_ms, + ii: ii.unwrap_or(false), + nns: nns.unwrap_or(false), + subnets, + bitcoind_addr, + dogecoind_addr, })), }, }, - crate::manifest::network::ManagedMode::Compose { compose } => { - Configuration::Managed { - managed: Managed { - mode: ManagedMode::Compose(Box::new(ManagedComposeConfig { - file: PathBuf::from(&compose.file), - gateway_service: compose.gateway_service, - environment: compose.environment, - })), - }, - } - } }, Mode::Connected(connected) => Configuration::Connected { connected: connected.into(), @@ -402,3 +403,102 @@ impl Access for MockNetworkAccessor { }) } } + +#[cfg(test)] +mod tests { + use super::*; + use crate::manifest::network::{ + Gateway as ManifestGateway, Managed as ManifestManaged, ManagedMode as ManifestManagedMode, + Mode, + }; + + #[test] + fn from_mode_image_with_launcher_settings() { + let mode = Mode::Managed(ManifestManaged { + mode: Box::new(ManifestManagedMode::Image { + image: "ghcr.io/dfinity/icp-cli-network-launcher".to_string(), + port_mapping: vec!["8000:4943".to_string()], + rm_on_exit: None, + args: None, + entrypoint: None, + environment: None, + volumes: None, + platform: None, + user: None, + shm_size: None, + status_dir: None, + mounts: None, + artificial_delay_ms: Some(50), + ii: Some(true), + nns: None, + subnets: None, + bitcoind_addr: Some(vec!["127.0.0.1:18444".to_string()]), + dogecoind_addr: Some(vec!["127.0.0.1:22556".to_string()]), + }), + }); + + let config: Configuration = mode.into(); + match config { + Configuration::Managed { + managed: + Managed { + mode: ManagedMode::Image(image_config), + }, + } => { + assert_eq!( + image_config.image, + "ghcr.io/dfinity/icp-cli-network-launcher" + ); + assert!(image_config.ii); + assert!(!image_config.nns); // defaults to false + assert_eq!(image_config.artificial_delay_ms, Some(50)); + assert_eq!( + image_config.bitcoind_addr, + Some(vec!["127.0.0.1:18444".to_string()]) + ); + assert_eq!( + image_config.dogecoind_addr, + Some(vec!["127.0.0.1:22556".to_string()]) + ); + } + _ => panic!("expected ManagedMode::Image"), + } + } + + #[test] + fn from_mode_launcher_with_bitcoind_addr() { + let mode = Mode::Managed(ManifestManaged { + mode: Box::new(ManifestManagedMode::Launcher { + gateway: Some(ManifestGateway { + host: None, + port: Some(8000), + }), + artificial_delay_ms: None, + ii: None, + nns: None, + subnets: None, + bitcoind_addr: Some(vec!["127.0.0.1:18444".to_string()]), + dogecoind_addr: None, + }), + }); + + let config: Configuration = mode.into(); + match config { + Configuration::Managed { + managed: + Managed { + mode: ManagedMode::Launcher(launcher_config), + }, + } => { + assert_eq!( + launcher_config.bitcoind_addr, + Some(vec!["127.0.0.1:18444".to_string()]) + ); + assert_eq!(launcher_config.dogecoind_addr, None); + assert!(!launcher_config.ii); + assert!(!launcher_config.nns); + } + _ => panic!("expected ManagedMode::Launcher"), + } + } +} diff --git a/crates/icp/src/project.rs b/crates/icp/src/project.rs index a10352bd..fa55f41a 100644 --- a/crates/icp/src/project.rs +++ b/crates/icp/src/project.rs @@ -318,6 +318,8 @@ pub async fn consolidate_manifest( ii: false, nns: false, subnets: None, + bitcoind_addr: None, + dogecoind_addr: None, })), }, }, diff --git a/docs/guides/containerized-networks.md b/docs/guides/containerized-networks.md index 1a9a46aa..d813827d 100644 --- a/docs/guides/containerized-networks.md +++ b/docs/guides/containerized-networks.md @@ -158,6 +158,30 @@ networks: - POCKET_IC_MUTE_SERVER=false ``` +### Launcher Settings + +You can use the same launcher settings as native managed networks. These are automatically translated into container command arguments: + +```yaml +networks: + - name: docker-local + mode: managed + image: ghcr.io/dfinity/icp-cli-network-launcher + port-mapping: + - "8000:4943" + ii: true + nns: true + subnets: + - application + - bitcoin + bitcoind-addr: + - "127.0.0.1:18444" +``` + +Available launcher settings: `ii`, `nns`, `subnets`, `artificial-delay-ms`, `bitcoind-addr`, `dogecoind-addr`. + +**Docker networking note:** When `bitcoind-addr` or `dogecoind-addr` addresses reference `127.0.0.1`, `localhost`, or `::1`, they are automatically translated to `host.docker.internal` so the container can reach services running on the host machine. On Linux Docker Engine, `host.docker.internal:host-gateway` is added automatically to ensure compatibility. + ### Remove Container on Exit Automatically delete the container when stopped: @@ -295,6 +319,12 @@ All available configuration options for containerized networks: | `user` | string | No | User to run as in `user[:group]` format (group is optional) | | `shm-size` | number | No | Size of `/dev/shm` in bytes | | `status-dir` | string | No | Status directory path (default: `/app/status`) | +| `ii` | bool | No | Set up Internet Identity canister (default: `false`) | +| `nns` | bool | No | Set up NNS canisters (default: `false`) | +| `subnets` | string[] | No | Configure subnet types (default: one application subnet) | +| `artificial-delay-ms` | number | No | Artificial delay for update calls (ms) | +| `bitcoind-addr` | string[] | No | Bitcoin P2P node addresses (auto-translated for Docker) | +| `dogecoind-addr` | string[] | No | Dogecoin P2P node addresses (auto-translated for Docker) | Example with multiple options: @@ -463,152 +493,6 @@ If you're on Windows and want to use a manually instantiated `dockerd` in a WSL2 - `ICP_CLI_DOCKER_WSL2_DISTRO=` — the WSL2 distribution name running dockerd - `DOCKER_HOST=tcp://:` — the TCP address where dockerd is listening -## Docker Compose Networks - -For more complex setups involving multiple services (like Bitcoin integration), you can use Docker Compose to manage your local network. - -### When to Use Compose - -Use Docker Compose when you need: -- Multiple services running together (e.g., Bitcoin node + IC replica) -- Complex service dependencies -- Custom networking between containers -- Services that must start in a specific order - -### Basic Configuration - -Instead of specifying an `image`, use the `compose` configuration: - -```yaml -networks: - - name: local - mode: managed - compose: - file: docker-compose.yml - gateway-service: icp-network -``` - -| Field | Type | Required | Description | -|-------------------|--------|----------|----------------------------------------------------| -| `file` | string | Yes | Path to docker-compose.yml (relative to project) | -| `gateway-service` | string | Yes | Name of the service running the IC gateway | - -### Example: Bitcoin Integration - -Here's a complete setup for local Bitcoin development: - -**docker-compose.bitcoin.yml:** -```yaml -services: - bitcoind: - image: lncm/bitcoind:v27.2 - command: - - -regtest - - -server - - -rpcbind=0.0.0.0 - - -rpcallowip=0.0.0.0/0 - - -rpcuser=ic-btc-integration - - -rpcpassword=ic-btc-integration - - -fallbackfee=0.00001 - ports: - - "18443:18443" - healthcheck: - test: ["CMD", "bitcoin-cli", "-regtest", "-rpcuser=ic-btc-integration", "-rpcpassword=ic-btc-integration", "getblockchaininfo"] - interval: 5s - timeout: 5s - retries: 20 - - icp-network: - image: ghcr.io/dfinity/icp-cli-network-launcher:latest - depends_on: - bitcoind: - condition: service_healthy - environment: - - ICP_CLI_NETWORK_LAUNCHER_INTERFACE_VERSION=1.0.0 - command: - - --bitcoind-addr=bitcoind:18444 - ports: - - "0:4943" - volumes: - - "${ICP_STATUS_DIR:-/tmp/icp-status}:/app/status" -``` - -> **Subnet configuration:** The network launcher defaults to creating an `application` subnet (plus the always-implied `nns` subnet). The `--bitcoind-addr` flag implicitly adds a `bitcoin` subnet. If you need additional subnets, use `--subnet` flags — but note that explicitly passing any `--subnet` flag **overrides the default** `application` subnet. To keep it, include `--subnet=application` explicitly. For example, to add a `system` subnet alongside bitcoin: `--subnet=application --subnet=system --bitcoind-addr=bitcoind:18444`. - -**icp.yaml:** -```yaml -networks: - - name: local - mode: managed - compose: - file: docker-compose.bitcoin.yml - gateway-service: icp-network - -environments: - - name: local - network: local - settings: - backend: - environment_variables: - BITCOIN_NETWORK: "regtest" -``` - -### How Compose Networks Work - -When you run `icp network start`: - -1. icp-cli creates a temporary status directory on the host -2. Sets the `ICP_STATUS_DIR` environment variable pointing to this directory -3. Runs `docker compose up -d` with the compose file -4. Monitors the status directory for the gateway's status file -5. Once ready, retrieves the mapped port from the gateway container - -The compose file must: -- Mount `${ICP_STATUS_DIR}` to the gateway service's status path (default `/app/status`) -- Have the gateway service write the status file when ready (same format as image-based networks) - -### Working with Bitcoin Regtest - -Generate blocks to create spendable Bitcoin: - -```bash -# Generate 101 blocks (coinbase maturity is 100 blocks) -docker compose -f docker-compose.bitcoin.yml exec bitcoind \ - bitcoin-cli -regtest -rpcuser=ic-btc-integration -rpcpassword=ic-btc-integration \ - generatetoaddress 101 "$(docker compose -f docker-compose.bitcoin.yml exec bitcoind \ - bitcoin-cli -regtest -rpcuser=ic-btc-integration -rpcpassword=ic-btc-integration getnewaddress)" -``` - -Create and fund addresses: - -```bash -# Get a new address -docker compose -f docker-compose.bitcoin.yml exec bitcoind \ - bitcoin-cli -regtest -rpcuser=ic-btc-integration -rpcpassword=ic-btc-integration \ - getnewaddress - -# Send Bitcoin to an address -docker compose -f docker-compose.bitcoin.yml exec bitcoind \ - bitcoin-cli -regtest -rpcuser=ic-btc-integration -rpcpassword=ic-btc-integration \ - sendtoaddress "bcrt1q..." 1.0 -``` - -### Bitcoin Template - -For a quick start with Bitcoin integration, use the bitcoin template: - -```bash -icp new my-bitcoin-project --template bitcoin -cd my-bitcoin-project -icp network start -icp build && icp deploy -``` - -The template includes: -- A backend canister with Bitcoin balance/UTXO query functions -- Docker Compose configuration for bitcoind + IC replica -- Environment variable configuration for network selection - ## Related Documentation - [Managing Environments](managing-environments.md) — Configure environments that use containerized networks diff --git a/docs/guides/index.md b/docs/guides/index.md index bde24930..aebe9d1e 100644 --- a/docs/guides/index.md +++ b/docs/guides/index.md @@ -20,7 +20,7 @@ Step-by-step instructions for common tasks. Each guide assumes you've completed ## Configuration -- [Containerized Networks](containerized-networks.md) — Run managed networks in Docker containers, including Docker Compose and Bitcoin integration +- [Containerized Networks](containerized-networks.md) — Run managed networks in Docker containers - [Using Recipes](using-recipes.md) — Reusable build templates for common patterns ## Advanced diff --git a/docs/reference/configuration.md b/docs/reference/configuration.md index c88b2119..fa7c6dc5 100644 --- a/docs/reference/configuration.md +++ b/docs/reference/configuration.md @@ -203,10 +203,14 @@ networks: | `mode` | string | Yes | `managed` | | `gateway.host` | string | No | Host address (default: localhost) | | `gateway.port` | integer | No | Port number (default: 8000, use 0 for random) | -| `artificial_delay_ms` | integer | No | Artificial delay to add to every update call (ms) | +| `artificial-delay-ms` | integer | No | Artificial delay to add to every update call (ms) | | `ii` | boolean | No | Set up Internet Identity canister (default: false) | | `nns` | boolean | No | Set up NNS canisters (default: false) | | `subnets` | array | No | Configure subnet types (default: one application subnet) | +| `bitcoind-addr` | array | No | Bitcoin P2P node addresses to connect to (e.g. `127.0.0.1:18444`) | +| `dogecoind-addr` | array | No | Dogecoin P2P node addresses to connect to | + +**Note:** The settings `artificial-delay-ms`, `ii`, `nns`, `subnets`, `bitcoind-addr`, and `dogecoind-addr` also work with [Docker image mode](#docker-network). When using Docker, `bitcoind-addr` and `dogecoind-addr` addresses referencing `127.0.0.1` or `localhost` are automatically translated to `host.docker.internal`. #### Subnet Configuration @@ -226,6 +230,45 @@ Available subnet types: `application`, `system`, `verified-application`, `bitcoi **Note:** Subnet type support depends on the network launcher version. The `application` type is commonly used for testing. +#### Bitcoin and Dogecoin Integration + +Connect the local network to a Bitcoin or Dogecoin node for testing chain integration: + +```yaml +networks: + - name: local + mode: managed + bitcoind-addr: + - "127.0.0.1:18444" +``` + +The `bitcoind-addr` field specifies the P2P address (not RPC) of the Bitcoin node. The network launcher will connect to the node and enable Bitcoin integration on the local network. Multiple addresses can be specified. + +Dogecoin integration works the same way via `dogecoind-addr`: + +```yaml +networks: + - name: local + mode: managed + dogecoind-addr: + - "127.0.0.1:22556" +``` + +Both can be configured simultaneously. + +**Note:** When `bitcoind-addr` or `dogecoind-addr` is configured, the network launcher automatically adds a bitcoin subnet. If you also explicitly specify `subnets`, you must include `application` to keep the default application subnet: + +```yaml +networks: + - name: local + mode: managed + bitcoind-addr: + - "127.0.0.1:18444" + subnets: + - application + - system +``` + ### Connected Network ```yaml @@ -254,32 +297,23 @@ networks: - "0:4943" ``` -See [Containerized Networks](../guides/containerized-networks.md) for full options. - -### Docker Compose Network - -For multi-container setups (like Bitcoin integration): +Docker networks support all the same launcher settings as native managed networks (`ii`, `nns`, `subnets`, `artificial-delay-ms`, `bitcoind-addr`, `dogecoind-addr`). These are translated into the appropriate container command arguments automatically: ```yaml networks: - - name: local + - name: docker-local mode: managed - compose: - file: docker-compose.yml - gateway-service: icp-network + image: ghcr.io/dfinity/icp-cli-network-launcher + port-mapping: + - "8000:4943" + ii: true + bitcoind-addr: + - "127.0.0.1:18444" ``` -| Property | Type | Required | Description | -|----------|------|----------|-------------| -| `file` | string | Yes | Path to docker-compose.yml (relative to project root) | -| `gateway-service` | string | Yes | Name of the service running the IC gateway | -| `environment` | array | No | Additional environment variables for docker compose | +**Docker networking:** When `bitcoind-addr` or `dogecoind-addr` addresses reference `127.0.0.1`, `localhost`, or `::1`, they are automatically translated to `host.docker.internal` so the container can reach services on the host. On Linux, `host.docker.internal:host-gateway` is added to ensure compatibility. -The compose file must: -- Have the gateway service mount `${ICP_STATUS_DIR}` to its status directory (default `/app/status`) -- Write a status file when the network is ready - -See [Docker Compose Networks](../guides/containerized-networks.md#docker-compose-networks) for examples. +See [Containerized Networks](../guides/containerized-networks.md) for full options. ## Environments diff --git a/docs/schemas/icp-yaml-schema.json b/docs/schemas/icp-yaml-schema.json index 321d4a68..0440daee 100644 --- a/docs/schemas/icp-yaml-schema.json +++ b/docs/schemas/icp-yaml-schema.json @@ -214,32 +214,6 @@ ], "type": "object" }, - "ComposeConfig": { - "description": "Docker Compose network configuration", - "properties": { - "environment": { - "default": [], - "description": "Additional environment variables to pass to docker compose", - "items": { - "type": "string" - }, - "type": "array" - }, - "file": { - "description": "Path to the docker-compose.yml file (relative to project root)", - "type": "string" - }, - "gateway-service": { - "description": "Name of the service that runs the IC gateway", - "type": "string" - } - }, - "required": [ - "file", - "gateway-service" - ], - "type": "object" - }, "Connected": { "properties": { "root-key": { @@ -421,6 +395,35 @@ "null" ] }, + "artificial-delay-ms": { + "description": "Artificial delay to add to every update call", + "format": "uint64", + "minimum": 0, + "type": [ + "integer", + "null" + ] + }, + "bitcoind-addr": { + "description": "Bitcoin P2P node addresses to connect to (e.g. \"127.0.0.1:18444\")", + "items": { + "type": "string" + }, + "type": [ + "array", + "null" + ] + }, + "dogecoind-addr": { + "description": "Dogecoin P2P node addresses to connect to", + "items": { + "type": "string" + }, + "type": [ + "array", + "null" + ] + }, "entrypoint": { "description": "Entrypoint to use for the container", "items": { @@ -441,6 +444,13 @@ "null" ] }, + "ii": { + "description": "Set up the Internet Identity canister", + "type": [ + "boolean", + "null" + ] + }, "image": { "description": "The docker image to use for the network", "type": "string" @@ -455,6 +465,13 @@ "null" ] }, + "nns": { + "description": "Set up the NNS", + "type": [ + "boolean", + "null" + ] + }, "platform": { "description": "The platform to use for the container (e.g. linux/amd64)", "type": [ @@ -491,6 +508,16 @@ "null" ] }, + "subnets": { + "description": "Configure the list of subnets (one application subnet by default)", + "items": { + "$ref": "#/$defs/SubnetKind" + }, + "type": [ + "array", + "null" + ] + }, "user": { "description": "The user to run the container as in the format user[:group]", "type": [ @@ -515,18 +542,6 @@ ], "type": "object" }, - { - "properties": { - "compose": { - "$ref": "#/$defs/ComposeConfig", - "description": "Docker Compose configuration" - } - }, - "required": [ - "compose" - ], - "type": "object" - }, { "properties": { "artificial-delay-ms": { @@ -538,6 +553,26 @@ "null" ] }, + "bitcoind-addr": { + "description": "Bitcoin P2P node addresses to connect to (e.g. \"127.0.0.1:18444\")", + "items": { + "type": "string" + }, + "type": [ + "array", + "null" + ] + }, + "dogecoind-addr": { + "description": "Dogecoin P2P node addresses to connect to", + "items": { + "type": "string" + }, + "type": [ + "array", + "null" + ] + }, "gateway": { "anyOf": [ { diff --git a/docs/schemas/network-yaml-schema.json b/docs/schemas/network-yaml-schema.json index f13bfe0c..d84e23c1 100644 --- a/docs/schemas/network-yaml-schema.json +++ b/docs/schemas/network-yaml-schema.json @@ -1,31 +1,5 @@ { "$defs": { - "ComposeConfig": { - "description": "Docker Compose network configuration", - "properties": { - "environment": { - "default": [], - "description": "Additional environment variables to pass to docker compose", - "items": { - "type": "string" - }, - "type": "array" - }, - "file": { - "description": "Path to the docker-compose.yml file (relative to project root)", - "type": "string" - }, - "gateway-service": { - "description": "Name of the service that runs the IC gateway", - "type": "string" - } - }, - "required": [ - "file", - "gateway-service" - ], - "type": "object" - }, "Connected": { "properties": { "root-key": { @@ -80,6 +54,35 @@ "null" ] }, + "artificial-delay-ms": { + "description": "Artificial delay to add to every update call", + "format": "uint64", + "minimum": 0, + "type": [ + "integer", + "null" + ] + }, + "bitcoind-addr": { + "description": "Bitcoin P2P node addresses to connect to (e.g. \"127.0.0.1:18444\")", + "items": { + "type": "string" + }, + "type": [ + "array", + "null" + ] + }, + "dogecoind-addr": { + "description": "Dogecoin P2P node addresses to connect to", + "items": { + "type": "string" + }, + "type": [ + "array", + "null" + ] + }, "entrypoint": { "description": "Entrypoint to use for the container", "items": { @@ -100,6 +103,13 @@ "null" ] }, + "ii": { + "description": "Set up the Internet Identity canister", + "type": [ + "boolean", + "null" + ] + }, "image": { "description": "The docker image to use for the network", "type": "string" @@ -114,6 +124,13 @@ "null" ] }, + "nns": { + "description": "Set up the NNS", + "type": [ + "boolean", + "null" + ] + }, "platform": { "description": "The platform to use for the container (e.g. linux/amd64)", "type": [ @@ -150,6 +167,16 @@ "null" ] }, + "subnets": { + "description": "Configure the list of subnets (one application subnet by default)", + "items": { + "$ref": "#/$defs/SubnetKind" + }, + "type": [ + "array", + "null" + ] + }, "user": { "description": "The user to run the container as in the format user[:group]", "type": [ @@ -174,18 +201,6 @@ ], "type": "object" }, - { - "properties": { - "compose": { - "$ref": "#/$defs/ComposeConfig", - "description": "Docker Compose configuration" - } - }, - "required": [ - "compose" - ], - "type": "object" - }, { "properties": { "artificial-delay-ms": { @@ -197,6 +212,26 @@ "null" ] }, + "bitcoind-addr": { + "description": "Bitcoin P2P node addresses to connect to (e.g. \"127.0.0.1:18444\")", + "items": { + "type": "string" + }, + "type": [ + "array", + "null" + ] + }, + "dogecoind-addr": { + "description": "Dogecoin P2P node addresses to connect to", + "items": { + "type": "string" + }, + "type": [ + "array", + "null" + ] + }, "gateway": { "anyOf": [ { From b84539eb747a7b2e3751611eb1d04ce914161c69 Mon Sep 17 00:00:00 2001 From: Marco Walz Date: Fri, 6 Feb 2026 23:19:51 +0100 Subject: [PATCH 3/6] docs: document unified launcher settings and args overlap for Docker image mode --- docs/guides/containerized-networks.md | 23 ++++++++++++++++------- docs/reference/configuration.md | 4 +++- 2 files changed, 19 insertions(+), 8 deletions(-) diff --git a/docs/guides/containerized-networks.md b/docs/guides/containerized-networks.md index d813827d..51fefba6 100644 --- a/docs/guides/containerized-networks.md +++ b/docs/guides/containerized-networks.md @@ -160,28 +160,37 @@ networks: ### Launcher Settings -You can use the same launcher settings as native managed networks. These are automatically translated into container command arguments: +Docker image networks support all the same launcher settings as native managed networks. This means you can switch between native and Docker modes by simply adding or removing the `image` and `port-mapping` fields — all other settings stay the same: +**Native launcher:** ```yaml networks: - - name: docker-local + - name: local + mode: managed + bitcoind-addr: + - "127.0.0.1:18444" +``` + +**Docker image (same settings, just add `image` and `port-mapping`):** +```yaml +networks: + - name: local mode: managed image: ghcr.io/dfinity/icp-cli-network-launcher port-mapping: - "8000:4943" - ii: true - nns: true - subnets: - - application - - bitcoin bitcoind-addr: - "127.0.0.1:18444" ``` +All launcher settings are automatically translated into the appropriate container command arguments. + Available launcher settings: `ii`, `nns`, `subnets`, `artificial-delay-ms`, `bitcoind-addr`, `dogecoind-addr`. **Docker networking note:** When `bitcoind-addr` or `dogecoind-addr` addresses reference `127.0.0.1`, `localhost`, or `::1`, they are automatically translated to `host.docker.internal` so the container can reach services running on the host machine. On Linux Docker Engine, `host.docker.internal:host-gateway` is added automatically to ensure compatibility. +> **Note:** Do not pass launcher settings (like `--bitcoind-addr` or `--ii`) via `args` when using the corresponding top-level fields. The `args` field is intended for additional flags not covered by the semantic settings. If the same setting is specified in both places, it will be passed to the launcher twice, and addresses in `args` will **not** be automatically translated for Docker networking. + ### Remove Container on Exit Automatically delete the container when stopped: diff --git a/docs/reference/configuration.md b/docs/reference/configuration.md index fa7c6dc5..314de8e6 100644 --- a/docs/reference/configuration.md +++ b/docs/reference/configuration.md @@ -297,7 +297,7 @@ networks: - "0:4943" ``` -Docker networks support all the same launcher settings as native managed networks (`ii`, `nns`, `subnets`, `artificial-delay-ms`, `bitcoind-addr`, `dogecoind-addr`). These are translated into the appropriate container command arguments automatically: +Docker networks support all the same launcher settings as native managed networks (`ii`, `nns`, `subnets`, `artificial-delay-ms`, `bitcoind-addr`, `dogecoind-addr`). This makes it easy to switch between native and Docker modes — just add or remove `image` and `port-mapping`, and all other settings stay the same. These are translated into the appropriate container command arguments automatically: ```yaml networks: @@ -313,6 +313,8 @@ networks: **Docker networking:** When `bitcoind-addr` or `dogecoind-addr` addresses reference `127.0.0.1`, `localhost`, or `::1`, they are automatically translated to `host.docker.internal` so the container can reach services on the host. On Linux, `host.docker.internal:host-gateway` is added to ensure compatibility. +> **Note:** Use the top-level fields (`bitcoind-addr`, `ii`, etc.) instead of passing these as raw flags via `args`. The `args` field is for additional flags not covered by the semantic settings. If specified in both places, flags will be duplicated and addresses in `args` will not be auto-translated for Docker networking. + See [Containerized Networks](../guides/containerized-networks.md) for full options. ## Environments From d233bfd57b1e345a8f6ba5c82b5d6748f5c9dca3 Mon Sep 17 00:00:00 2001 From: Marco Walz Date: Fri, 6 Feb 2026 23:34:28 +0100 Subject: [PATCH 4/6] fix: formatting --- crates/icp/src/network/managed/run.rs | 39 +++++++++++++++++---------- 1 file changed, 25 insertions(+), 14 deletions(-) diff --git a/crates/icp/src/network/managed/run.rs b/crates/icp/src/network/managed/run.rs index bb792976..9d6903e0 100644 --- a/crates/icp/src/network/managed/run.rs +++ b/crates/icp/src/network/managed/run.rs @@ -790,7 +790,12 @@ mod tests { assert!(opts.rm_on_exit); assert_eq!(opts.status_dir, "/app/status"); // Port binding maps host 8000 to container 4943 - let binding = opts.port_bindings.get("4943/tcp").unwrap().as_ref().unwrap(); + let binding = opts + .port_bindings + .get("4943/tcp") + .unwrap() + .as_ref() + .unwrap(); assert_eq!(binding[0].host_port.as_deref(), Some("8000")); } @@ -809,7 +814,12 @@ mod tests { dogecoind_addr: None, }; let opts = transform_native_launcher_to_container(&config); - let binding = opts.port_bindings.get("4943/tcp").unwrap().as_ref().unwrap(); + let binding = opts + .port_bindings + .get("4943/tcp") + .unwrap() + .as_ref() + .unwrap(); assert_eq!(binding[0].host_port.as_deref(), Some("0")); } @@ -828,9 +838,10 @@ mod tests { // --ii flag should be present assert!(opts.args.contains(&"--ii".to_string())); // bitcoind-addr should be translated for Docker - assert!(opts - .args - .contains(&"--bitcoind-addr=host.docker.internal:18444".to_string())); + assert!( + opts.args + .contains(&"--bitcoind-addr=host.docker.internal:18444".to_string()) + ); // extra_hosts should include host.docker.internal mapping assert_eq!( opts.extra_hosts, @@ -851,12 +862,11 @@ mod tests { }; let opts = transform_native_launcher_to_container(&config); assert!(opts.args.contains(&"--nns".to_string())); - assert!(opts - .args - .contains(&"--artificial-delay-ms=50".to_string())); - assert!(opts - .args - .contains(&"--dogecoind-addr=host.docker.internal:22556".to_string())); + assert!(opts.args.contains(&"--artificial-delay-ms=50".to_string())); + assert!( + opts.args + .contains(&"--dogecoind-addr=host.docker.internal:22556".to_string()) + ); assert_eq!( opts.extra_hosts, vec!["host.docker.internal:host-gateway".to_string()] @@ -876,9 +886,10 @@ mod tests { }; let opts = transform_native_launcher_to_container(&config); // External address should pass through unchanged - assert!(opts - .args - .contains(&"--bitcoind-addr=192.168.1.5:18444".to_string())); + assert!( + opts.args + .contains(&"--bitcoind-addr=192.168.1.5:18444".to_string()) + ); // No extra_hosts needed for external addresses assert!(opts.extra_hosts.is_empty()); } From eeb2cc26d10b25c08535d04e4d4b8e01a1401c87 Mon Sep 17 00:00:00 2001 From: Marco Walz Date: Mon, 9 Feb 2026 13:04:09 +0100 Subject: [PATCH 5/6] refactor: remove launcher settings from Docker image variant --- crates/icp/src/manifest/network.rs | 96 -------------------- crates/icp/src/network/managed/docker.rs | 108 +---------------------- crates/icp/src/network/mod.rs | 71 --------------- docs/guides/containerized-networks.md | 34 ++----- docs/reference/configuration.md | 62 ++++++------- docs/schemas/icp-yaml-schema.json | 53 ----------- docs/schemas/network-yaml-schema.json | 53 ----------- 7 files changed, 42 insertions(+), 435 deletions(-) diff --git a/crates/icp/src/manifest/network.rs b/crates/icp/src/manifest/network.rs index 9a607e9a..02aa8ceb 100644 --- a/crates/icp/src/manifest/network.rs +++ b/crates/icp/src/manifest/network.rs @@ -55,18 +55,6 @@ pub enum ManagedMode { status_dir: Option, /// Bind mounts to add to the container in the format relative_host_path:container_path[:options] mounts: Option>, - /// Artificial delay to add to every update call - artificial_delay_ms: Option, - /// Set up the Internet Identity canister - ii: Option, - /// Set up the NNS - nns: Option, - /// Configure the list of subnets (one application subnet by default) - subnets: Option>, - /// Bitcoin P2P node addresses to connect to (e.g. "127.0.0.1:18444") - bitcoind_addr: Option>, - /// Dogecoin P2P node addresses to connect to - dogecoind_addr: Option>, }, Launcher { /// HTTP gateway configuration @@ -368,88 +356,4 @@ mod tests { }, ); } - - #[test] - fn image_network_with_launcher_settings() { - assert_eq!( - validate_network_yaml(indoc! {r#" - name: my-network - mode: managed - image: ghcr.io/dfinity/icp-cli-network-launcher - port-mapping: - - "8000:4943" - ii: true - nns: true - bitcoind-addr: - - "127.0.0.1:18444" - dogecoind-addr: - - "127.0.0.1:22556" - "#}), - NetworkManifest { - name: "my-network".to_string(), - configuration: Mode::Managed(Managed { - mode: Box::new(ManagedMode::Image { - image: "ghcr.io/dfinity/icp-cli-network-launcher".to_string(), - port_mapping: vec!["8000:4943".to_string()], - rm_on_exit: None, - args: None, - entrypoint: None, - environment: None, - volumes: None, - platform: None, - user: None, - shm_size: None, - status_dir: None, - mounts: None, - artificial_delay_ms: None, - ii: Some(true), - nns: Some(true), - subnets: None, - bitcoind_addr: Some(vec!["127.0.0.1:18444".to_string()]), - dogecoind_addr: Some(vec!["127.0.0.1:22556".to_string()]), - }) - }) - }, - ); - } - - #[test] - fn image_network_with_bitcoind_addr() { - assert_eq!( - validate_network_yaml(indoc! {r#" - name: my-network - mode: managed - image: ghcr.io/dfinity/icp-cli-network-launcher - port-mapping: - - "0:4943" - bitcoind-addr: - - "127.0.0.1:18444" - "#}), - NetworkManifest { - name: "my-network".to_string(), - configuration: Mode::Managed(Managed { - mode: Box::new(ManagedMode::Image { - image: "ghcr.io/dfinity/icp-cli-network-launcher".to_string(), - port_mapping: vec!["0:4943".to_string()], - rm_on_exit: None, - args: None, - entrypoint: None, - environment: None, - volumes: None, - platform: None, - user: None, - shm_size: None, - status_dir: None, - mounts: None, - artificial_delay_ms: None, - ii: None, - nns: None, - subnets: None, - bitcoind_addr: Some(vec!["127.0.0.1:18444".to_string()]), - dogecoind_addr: None, - }) - }) - }, - ); - } } diff --git a/crates/icp/src/network/managed/docker.rs b/crates/icp/src/network/managed/docker.rs index 6cd54f2b..97fd866d 100644 --- a/crates/icp/src/network/managed/docker.rs +++ b/crates/icp/src/network/managed/docker.rs @@ -20,13 +20,10 @@ use tokio::select; use wslpath2::Conversion; use crate::network::{ - Gateway, ManagedImageConfig, ManagedLauncherConfig, config::ChildLocator, - managed::launcher::NetworkInstance, + ManagedImageConfig, config::ChildLocator, managed::launcher::NetworkInstance, }; use crate::prelude::*; -use super::launcher::launcher_settings_flags; - use super::launcher::wait_for_launcher_status; /// Error converting a path for WSL2. @@ -152,38 +149,11 @@ impl TryFrom<&ManagedImageConfig> for ManagedImageOptions { .map(|v| convert_volume(wsl2_convert, wsl2_distro, v)) .try_collect()?; - // Generate launcher settings flags from semantic fields - let launcher_config = ManagedLauncherConfig { - gateway: Gateway::default(), - artificial_delay_ms: config.artificial_delay_ms, - ii: config.ii, - nns: config.nns, - subnets: config.subnets.clone(), - bitcoind_addr: config.bitcoind_addr.clone(), - dogecoind_addr: config.dogecoind_addr.clone(), - }; - let launcher_flags = - translate_launcher_args_for_docker(launcher_settings_flags(&launcher_config)); - - // Append launcher flags to explicit user args - let mut all_args = config.args.clone(); - all_args.extend(launcher_flags); - - // Compute extra_hosts for Docker networking - let all_addrs: Vec = config - .bitcoind_addr - .iter() - .chain(config.dogecoind_addr.iter()) - .flatten() - .cloned() - .collect(); - let extra_hosts = docker_extra_hosts_for_addrs(&all_addrs); - Ok(ManagedImageOptions { image: config.image.clone(), port_bindings, rm_on_exit: config.rm_on_exit, - args: all_args, + args: config.args.clone(), entrypoint: config.entrypoint.clone(), environment: config.environment.clone(), volumes, @@ -192,7 +162,7 @@ impl TryFrom<&ManagedImageConfig> for ManagedImageOptions { shm_size: config.shm_size, status_dir: config.status_dir.clone(), mounts, - extra_hosts, + extra_hosts: vec![], }) } } @@ -902,76 +872,4 @@ mod tests { let expected = args.clone(); assert_eq!(translate_launcher_args_for_docker(args), expected); } - - #[test] - fn image_config_conversion_with_launcher_settings() { - let config = ManagedImageConfig { - image: "ghcr.io/dfinity/icp-cli-network-launcher".to_string(), - port_mapping: vec!["8000:4943".to_string()], - rm_on_exit: false, - args: vec!["--custom-flag".to_string()], - entrypoint: None, - environment: vec![], - volumes: vec![], - platform: None, - user: None, - shm_size: None, - status_dir: "/app/status".to_string(), - mounts: vec![], - artificial_delay_ms: None, - ii: true, - nns: false, - subnets: None, - bitcoind_addr: Some(vec!["127.0.0.1:18444".to_string()]), - dogecoind_addr: None, - }; - let options = ManagedImageOptions::try_from(&config).unwrap(); - - // User args come first, then generated launcher flags - assert!(options.args.contains(&"--custom-flag".to_string())); - assert!(options.args.contains(&"--ii".to_string())); - assert!( - options - .args - .contains(&"--bitcoind-addr=host.docker.internal:18444".to_string()) - ); - - // extra_hosts for Linux Docker compatibility - assert_eq!( - options.extra_hosts, - vec!["host.docker.internal:host-gateway"] - ); - } - - #[test] - fn image_config_conversion_without_localhost_no_extra_hosts() { - let config = ManagedImageConfig { - image: "ghcr.io/dfinity/icp-cli-network-launcher".to_string(), - port_mapping: vec!["8000:4943".to_string()], - rm_on_exit: false, - args: vec![], - entrypoint: None, - environment: vec![], - volumes: vec![], - platform: None, - user: None, - shm_size: None, - status_dir: "/app/status".to_string(), - mounts: vec![], - artificial_delay_ms: None, - ii: false, - nns: false, - subnets: None, - bitcoind_addr: Some(vec!["192.168.1.5:18444".to_string()]), - dogecoind_addr: None, - }; - let options = ManagedImageOptions::try_from(&config).unwrap(); - - assert!( - options - .args - .contains(&"--bitcoind-addr=192.168.1.5:18444".to_string()) - ); - assert!(options.extra_hosts.is_empty()); - } } diff --git a/crates/icp/src/network/mod.rs b/crates/icp/src/network/mod.rs index cce1300d..afcecafa 100644 --- a/crates/icp/src/network/mod.rs +++ b/crates/icp/src/network/mod.rs @@ -150,12 +150,6 @@ pub struct ManagedImageConfig { pub shm_size: Option, pub status_dir: String, pub mounts: Vec, - pub artificial_delay_ms: Option, - pub ii: bool, - pub nns: bool, - pub subnets: Option>, - pub bitcoind_addr: Option>, - pub dogecoind_addr: Option>, } #[derive(Clone, Debug, Deserialize, PartialEq, JsonSchema, Serialize)] @@ -259,12 +253,6 @@ impl From for Configuration { shm_size, status_dir, mounts: mount, - artificial_delay_ms, - ii, - nns, - subnets, - bitcoind_addr, - dogecoind_addr, } => Configuration::Managed { managed: Managed { mode: ManagedMode::Image(Box::new(ManagedImageConfig { @@ -280,12 +268,6 @@ impl From for Configuration { shm_size, status_dir: status_dir.unwrap_or_else(|| "/app/status".to_string()), mounts: mount.unwrap_or_default(), - artificial_delay_ms, - ii: ii.unwrap_or(false), - nns: nns.unwrap_or(false), - subnets, - bitcoind_addr, - dogecoind_addr, })), }, }, @@ -412,59 +394,6 @@ mod tests { Mode, }; - #[test] - fn from_mode_image_with_launcher_settings() { - let mode = Mode::Managed(ManifestManaged { - mode: Box::new(ManifestManagedMode::Image { - image: "ghcr.io/dfinity/icp-cli-network-launcher".to_string(), - port_mapping: vec!["8000:4943".to_string()], - rm_on_exit: None, - args: None, - entrypoint: None, - environment: None, - volumes: None, - platform: None, - user: None, - shm_size: None, - status_dir: None, - mounts: None, - artificial_delay_ms: Some(50), - ii: Some(true), - nns: None, - subnets: None, - bitcoind_addr: Some(vec!["127.0.0.1:18444".to_string()]), - dogecoind_addr: Some(vec!["127.0.0.1:22556".to_string()]), - }), - }); - - let config: Configuration = mode.into(); - match config { - Configuration::Managed { - managed: - Managed { - mode: ManagedMode::Image(image_config), - }, - } => { - assert_eq!( - image_config.image, - "ghcr.io/dfinity/icp-cli-network-launcher" - ); - assert!(image_config.ii); - assert!(!image_config.nns); // defaults to false - assert_eq!(image_config.artificial_delay_ms, Some(50)); - assert_eq!( - image_config.bitcoind_addr, - Some(vec!["127.0.0.1:18444".to_string()]) - ); - assert_eq!( - image_config.dogecoind_addr, - Some(vec!["127.0.0.1:22556".to_string()]) - ); - } - _ => panic!("expected ManagedMode::Image"), - } - } - #[test] fn from_mode_launcher_with_bitcoind_addr() { let mode = Mode::Managed(ManifestManaged { diff --git a/docs/guides/containerized-networks.md b/docs/guides/containerized-networks.md index 51fefba6..28dceb5c 100644 --- a/docs/guides/containerized-networks.md +++ b/docs/guides/containerized-networks.md @@ -158,38 +158,26 @@ networks: - POCKET_IC_MUTE_SERVER=false ``` -### Launcher Settings +### Passing Arguments to the Container -Docker image networks support all the same launcher settings as native managed networks. This means you can switch between native and Docker modes by simply adding or removing the `image` and `port-mapping` fields — all other settings stay the same: +Use the `args` field to pass command-line arguments to the container's entrypoint. This is how you configure image-specific behavior such as enabling Internet Identity, NNS, Bitcoin integration, or other flags supported by the image: -**Native launcher:** ```yaml networks: - - name: local - mode: managed - bitcoind-addr: - - "127.0.0.1:18444" -``` - -**Docker image (same settings, just add `image` and `port-mapping`):** -```yaml -networks: - - name: local + - name: docker-local mode: managed image: ghcr.io/dfinity/icp-cli-network-launcher port-mapping: - "8000:4943" - bitcoind-addr: - - "127.0.0.1:18444" + args: + - "--ii" ``` -All launcher settings are automatically translated into the appropriate container command arguments. - -Available launcher settings: `ii`, `nns`, `subnets`, `artificial-delay-ms`, `bitcoind-addr`, `dogecoind-addr`. +The `args` field passes values directly to the container entrypoint with no processing. The Docker image determines what arguments it accepts — see the image's documentation for available options. -**Docker networking note:** When `bitcoind-addr` or `dogecoind-addr` addresses reference `127.0.0.1`, `localhost`, or `::1`, they are automatically translated to `host.docker.internal` so the container can reach services running on the host machine. On Linux Docker Engine, `host.docker.internal:host-gateway` is added automatically to ensure compatibility. +**Comparison with native launcher mode:** When using native managed networks (without `image`), settings like `bitcoind-addr`, `ii`, `nns`, and `subnets` are configured as top-level YAML fields. In Docker image mode, these are passed via `args` instead, since the image could be any Docker image — not necessarily the official network launcher. -> **Note:** Do not pass launcher settings (like `--bitcoind-addr` or `--ii`) via `args` when using the corresponding top-level fields. The `args` field is intended for additional flags not covered by the semantic settings. If the same setting is specified in both places, it will be passed to the launcher twice, and addresses in `args` will **not** be automatically translated for Docker networking. +> **Docker networking note:** When referencing services running on the host machine from inside a container (e.g., a local Bitcoin node), use `host.docker.internal` instead of `127.0.0.1` or `localhost`. Inside a container, `127.0.0.1` refers to the container's own loopback, not the host. For example: `--bitcoind-addr=host.docker.internal:18444`. Docker Desktop (macOS/Windows) resolves `host.docker.internal` automatically. On Linux Docker Engine, you may need to pass `--add-host=host.docker.internal:host-gateway` or equivalent to ensure it resolves. ### Remove Container on Exit @@ -328,12 +316,6 @@ All available configuration options for containerized networks: | `user` | string | No | User to run as in `user[:group]` format (group is optional) | | `shm-size` | number | No | Size of `/dev/shm` in bytes | | `status-dir` | string | No | Status directory path (default: `/app/status`) | -| `ii` | bool | No | Set up Internet Identity canister (default: `false`) | -| `nns` | bool | No | Set up NNS canisters (default: `false`) | -| `subnets` | string[] | No | Configure subnet types (default: one application subnet) | -| `artificial-delay-ms` | number | No | Artificial delay for update calls (ms) | -| `bitcoind-addr` | string[] | No | Bitcoin P2P node addresses (auto-translated for Docker) | -| `dogecoind-addr` | string[] | No | Dogecoin P2P node addresses (auto-translated for Docker) | Example with multiple options: diff --git a/docs/reference/configuration.md b/docs/reference/configuration.md index 314de8e6..9d317108 100644 --- a/docs/reference/configuration.md +++ b/docs/reference/configuration.md @@ -186,8 +186,15 @@ type: https://example.com/recipe.hb.yaml ## Networks +Networks define where canisters are deployed. There are two modes: + +- **Managed** (`mode: managed`): A local test network launched and controlled by icp-cli. Can run [natively](#managed-network) or in a [Docker container](#docker-network). +- **Connected** (`mode: connected`): A remote network accessed by URL. + ### Managed Network +A managed network runs the [network launcher](https://github.com/dfinity/icp-cli-network-launcher) natively on your machine. To run it in a Docker container instead, see [Docker Network](#docker-network). + ```yaml networks: - name: local-dev @@ -203,18 +210,24 @@ networks: | `mode` | string | Yes | `managed` | | `gateway.host` | string | No | Host address (default: localhost) | | `gateway.port` | integer | No | Port number (default: 8000, use 0 for random) | -| `artificial-delay-ms` | integer | No | Artificial delay to add to every update call (ms) | -| `ii` | boolean | No | Set up Internet Identity canister (default: false) | -| `nns` | boolean | No | Set up NNS canisters (default: false) | -| `subnets` | array | No | Configure subnet types (default: one application subnet) | -| `bitcoind-addr` | array | No | Bitcoin P2P node addresses to connect to (e.g. `127.0.0.1:18444`) | -| `dogecoind-addr` | array | No | Dogecoin P2P node addresses to connect to | +| `artificial-delay-ms` | integer | No | Artificial delay for update calls (ms) | +| `ii` | boolean | No | Install Internet Identity canister (default: false). Also implicitly enabled by `nns`, `bitcoind-addr`, and `dogecoind-addr`. | +| `nns` | boolean | No | Install NNS and SNS canisters (default: false). Implies `ii` and adds an SNS subnet. | +| `subnets` | array | No | Configure subnet types. See [Subnet Configuration](#subnet-configuration). | +| `bitcoind-addr` | array | No | Bitcoin P2P node addresses (e.g. `127.0.0.1:18444`). Adds a bitcoin and II subnet. | +| `dogecoind-addr` | array | No | Dogecoin P2P node addresses. Adds a bitcoin and II subnet. | -**Note:** The settings `artificial-delay-ms`, `ii`, `nns`, `subnets`, `bitcoind-addr`, and `dogecoind-addr` also work with [Docker image mode](#docker-network). When using Docker, `bitcoind-addr` and `dogecoind-addr` addresses referencing `127.0.0.1` or `localhost` are automatically translated to `host.docker.internal`. +For full details on how these settings interact, see the [network launcher CLI reference](https://github.com/dfinity/icp-cli-network-launcher#cli-reference). + +> **Note:** These settings apply to native managed networks only. For [Docker image mode](#docker-network), pass equivalent flags via the `args` field instead. #### Subnet Configuration -Configure the local network's subnet layout. By default, a single application subnet is created. Use multiple subnets to test cross-subnet (Xnet) calls: +Configure the local network's subnet layout: + +- **Default** (no `subnets` field): one application subnet is created. +- **With `subnets`**: only the listed subnets are created — the default application subnet is **replaced**, not extended. Add `application` explicitly if you still need it. +- An **NNS subnet** is always created regardless of configuration (required for system operations). ```yaml networks: @@ -228,8 +241,6 @@ networks: Available subnet types: `application`, `system`, `verified-application`, `bitcoin`, `fiduciary`, `nns`, `sns` -**Note:** Subnet type support depends on the network launcher version. The `application` type is commonly used for testing. - #### Bitcoin and Dogecoin Integration Connect the local network to a Bitcoin or Dogecoin node for testing chain integration: @@ -242,21 +253,9 @@ networks: - "127.0.0.1:18444" ``` -The `bitcoind-addr` field specifies the P2P address (not RPC) of the Bitcoin node. The network launcher will connect to the node and enable Bitcoin integration on the local network. Multiple addresses can be specified. +The `bitcoind-addr` field specifies the P2P address (not RPC) of the Bitcoin node. Multiple addresses can be specified. Dogecoin integration works the same way via `dogecoind-addr`. Both can be configured simultaneously. -Dogecoin integration works the same way via `dogecoind-addr`: - -```yaml -networks: - - name: local - mode: managed - dogecoind-addr: - - "127.0.0.1:22556" -``` - -Both can be configured simultaneously. - -**Note:** When `bitcoind-addr` or `dogecoind-addr` is configured, the network launcher automatically adds a bitcoin subnet. If you also explicitly specify `subnets`, you must include `application` to keep the default application subnet: +**Implicit effects:** When `bitcoind-addr` or `dogecoind-addr` is configured, the network launcher automatically adds a **bitcoin** subnet and an **II** subnet (provides threshold signing keys required for chain operations). If you also explicitly specify `subnets`, you must include `application` to keep the default application subnet: ```yaml networks: @@ -288,6 +287,8 @@ networks: ### Docker Network +A managed network can also run inside a Docker container. Adding the `image` field switches from native to Docker mode: + ```yaml networks: - name: docker-local @@ -297,7 +298,7 @@ networks: - "0:4943" ``` -Docker networks support all the same launcher settings as native managed networks (`ii`, `nns`, `subnets`, `artificial-delay-ms`, `bitcoind-addr`, `dogecoind-addr`). This makes it easy to switch between native and Docker modes — just add or remove `image` and `port-mapping`, and all other settings stay the same. These are translated into the appropriate container command arguments automatically: +To configure image-specific behavior (e.g., enabling Internet Identity, NNS, or Bitcoin integration), use the `args` field to pass command-line arguments to the container entrypoint: ```yaml networks: @@ -306,16 +307,15 @@ networks: image: ghcr.io/dfinity/icp-cli-network-launcher port-mapping: - "8000:4943" - ii: true - bitcoind-addr: - - "127.0.0.1:18444" + args: + - "--ii" ``` -**Docker networking:** When `bitcoind-addr` or `dogecoind-addr` addresses reference `127.0.0.1`, `localhost`, or `::1`, they are automatically translated to `host.docker.internal` so the container can reach services on the host. On Linux, `host.docker.internal:host-gateway` is added to ensure compatibility. +The available arguments depend on the Docker image — see the image's documentation for details. -> **Note:** Use the top-level fields (`bitcoind-addr`, `ii`, etc.) instead of passing these as raw flags via `args`. The `args` field is for additional flags not covered by the semantic settings. If specified in both places, flags will be duplicated and addresses in `args` will not be auto-translated for Docker networking. +> **Docker networking note:** When referencing services running on the host machine from inside a container (e.g., a local Bitcoin node), use `host.docker.internal` instead of `127.0.0.1` or `localhost`. Inside a container, `127.0.0.1` refers to the container's own loopback, not the host. For example: `--bitcoind-addr=host.docker.internal:18444`. Docker Desktop (macOS/Windows) resolves `host.docker.internal` automatically. On Linux Docker Engine, you may need to pass `--add-host=host.docker.internal:host-gateway` or equivalent to ensure it resolves. -See [Containerized Networks](../guides/containerized-networks.md) for full options. +See [Containerized Networks](../guides/containerized-networks.md) for full configuration options. ## Environments diff --git a/docs/schemas/icp-yaml-schema.json b/docs/schemas/icp-yaml-schema.json index 0440daee..94f21985 100644 --- a/docs/schemas/icp-yaml-schema.json +++ b/docs/schemas/icp-yaml-schema.json @@ -395,35 +395,6 @@ "null" ] }, - "artificial-delay-ms": { - "description": "Artificial delay to add to every update call", - "format": "uint64", - "minimum": 0, - "type": [ - "integer", - "null" - ] - }, - "bitcoind-addr": { - "description": "Bitcoin P2P node addresses to connect to (e.g. \"127.0.0.1:18444\")", - "items": { - "type": "string" - }, - "type": [ - "array", - "null" - ] - }, - "dogecoind-addr": { - "description": "Dogecoin P2P node addresses to connect to", - "items": { - "type": "string" - }, - "type": [ - "array", - "null" - ] - }, "entrypoint": { "description": "Entrypoint to use for the container", "items": { @@ -444,13 +415,6 @@ "null" ] }, - "ii": { - "description": "Set up the Internet Identity canister", - "type": [ - "boolean", - "null" - ] - }, "image": { "description": "The docker image to use for the network", "type": "string" @@ -465,13 +429,6 @@ "null" ] }, - "nns": { - "description": "Set up the NNS", - "type": [ - "boolean", - "null" - ] - }, "platform": { "description": "The platform to use for the container (e.g. linux/amd64)", "type": [ @@ -508,16 +465,6 @@ "null" ] }, - "subnets": { - "description": "Configure the list of subnets (one application subnet by default)", - "items": { - "$ref": "#/$defs/SubnetKind" - }, - "type": [ - "array", - "null" - ] - }, "user": { "description": "The user to run the container as in the format user[:group]", "type": [ diff --git a/docs/schemas/network-yaml-schema.json b/docs/schemas/network-yaml-schema.json index d84e23c1..6f4655b5 100644 --- a/docs/schemas/network-yaml-schema.json +++ b/docs/schemas/network-yaml-schema.json @@ -54,35 +54,6 @@ "null" ] }, - "artificial-delay-ms": { - "description": "Artificial delay to add to every update call", - "format": "uint64", - "minimum": 0, - "type": [ - "integer", - "null" - ] - }, - "bitcoind-addr": { - "description": "Bitcoin P2P node addresses to connect to (e.g. \"127.0.0.1:18444\")", - "items": { - "type": "string" - }, - "type": [ - "array", - "null" - ] - }, - "dogecoind-addr": { - "description": "Dogecoin P2P node addresses to connect to", - "items": { - "type": "string" - }, - "type": [ - "array", - "null" - ] - }, "entrypoint": { "description": "Entrypoint to use for the container", "items": { @@ -103,13 +74,6 @@ "null" ] }, - "ii": { - "description": "Set up the Internet Identity canister", - "type": [ - "boolean", - "null" - ] - }, "image": { "description": "The docker image to use for the network", "type": "string" @@ -124,13 +88,6 @@ "null" ] }, - "nns": { - "description": "Set up the NNS", - "type": [ - "boolean", - "null" - ] - }, "platform": { "description": "The platform to use for the container (e.g. linux/amd64)", "type": [ @@ -167,16 +124,6 @@ "null" ] }, - "subnets": { - "description": "Configure the list of subnets (one application subnet by default)", - "items": { - "$ref": "#/$defs/SubnetKind" - }, - "type": [ - "array", - "null" - ] - }, "user": { "description": "The user to run the container as in the format user[:group]", "type": [ From 61a22fe72e7df32525ab805f37eb7a57ad405647 Mon Sep 17 00:00:00 2001 From: Marco Walz Date: Mon, 9 Feb 2026 13:15:20 +0100 Subject: [PATCH 6/6] chore: remove redundant tests and comments --- crates/icp/src/network/managed/docker.rs | 118 --------------------- crates/icp/src/network/managed/launcher.rs | 118 --------------------- crates/icp/src/network/managed/run.rs | 6 -- 3 files changed, 242 deletions(-) diff --git a/crates/icp/src/network/managed/docker.rs b/crates/icp/src/network/managed/docker.rs index 97fd866d..74f86872 100644 --- a/crates/icp/src/network/managed/docker.rs +++ b/crates/icp/src/network/managed/docker.rs @@ -755,121 +755,3 @@ impl AsyncDrop for DockerDropGuard { _ = self.stop().await; } } - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn translate_addr_localhost_ipv4() { - assert_eq!( - translate_addr_for_docker("127.0.0.1:18444"), - "host.docker.internal:18444" - ); - } - - #[test] - fn translate_addr_localhost_name() { - assert_eq!( - translate_addr_for_docker("localhost:18444"), - "host.docker.internal:18444" - ); - } - - #[test] - fn translate_addr_localhost_ipv6() { - assert_eq!( - translate_addr_for_docker("::1:18444"), - "host.docker.internal:18444" - ); - } - - #[test] - fn translate_addr_remote_unchanged() { - assert_eq!( - translate_addr_for_docker("192.168.1.5:18444"), - "192.168.1.5:18444" - ); - } - - #[test] - fn translate_addr_hostname_unchanged() { - assert_eq!( - translate_addr_for_docker("my-bitcoin-node:18444"), - "my-bitcoin-node:18444" - ); - } - - #[test] - fn translate_addr_no_port_unchanged() { - assert_eq!(translate_addr_for_docker("just-a-string"), "just-a-string"); - } - - #[test] - fn extra_hosts_for_localhost_addr() { - let addrs = vec!["127.0.0.1:18444".to_string()]; - assert_eq!( - docker_extra_hosts_for_addrs(&addrs), - vec!["host.docker.internal:host-gateway"] - ); - } - - #[test] - fn extra_hosts_for_remote_addr() { - let addrs = vec!["192.168.1.5:18444".to_string()]; - assert!(docker_extra_hosts_for_addrs(&addrs).is_empty()); - } - - #[test] - fn extra_hosts_for_mixed_addrs() { - let addrs = vec![ - "127.0.0.1:18444".to_string(), - "192.168.1.5:22556".to_string(), - ]; - assert_eq!( - docker_extra_hosts_for_addrs(&addrs), - vec!["host.docker.internal:host-gateway"] - ); - } - - #[test] - fn extra_hosts_for_empty_addrs() { - let addrs: Vec = vec![]; - assert!(docker_extra_hosts_for_addrs(&addrs).is_empty()); - } - - #[test] - fn translate_launcher_args_bitcoind() { - let args = vec![ - "--ii".to_string(), - "--bitcoind-addr=127.0.0.1:18444".to_string(), - ]; - assert_eq!( - translate_launcher_args_for_docker(args), - vec![ - "--ii".to_string(), - "--bitcoind-addr=host.docker.internal:18444".to_string(), - ] - ); - } - - #[test] - fn translate_launcher_args_dogecoind() { - let args = vec!["--dogecoind-addr=localhost:22556".to_string()]; - assert_eq!( - translate_launcher_args_for_docker(args), - vec!["--dogecoind-addr=host.docker.internal:22556".to_string()] - ); - } - - #[test] - fn translate_launcher_args_non_addr_unchanged() { - let args = vec![ - "--nns".to_string(), - "--artificial-delay-ms=100".to_string(), - "--subnet=bitcoin".to_string(), - ]; - let expected = args.clone(); - assert_eq!(translate_launcher_args_for_docker(args), expected); - } -} diff --git a/crates/icp/src/network/managed/launcher.rs b/crates/icp/src/network/managed/launcher.rs index d7eb2cd6..3d0e65cc 100644 --- a/crates/icp/src/network/managed/launcher.rs +++ b/crates/icp/src/network/managed/launcher.rs @@ -337,124 +337,6 @@ pub fn wait_for_launcher_status( }) } -#[cfg(test)] -mod tests { - use super::*; - use crate::network::{Gateway, Port, SubnetKind}; - - fn default_config() -> ManagedLauncherConfig { - ManagedLauncherConfig { - gateway: Gateway::default(), - artificial_delay_ms: None, - ii: false, - nns: false, - subnets: None, - bitcoind_addr: None, - dogecoind_addr: None, - } - } - - #[test] - fn flags_default_config_empty() { - let config = default_config(); - assert!(launcher_settings_flags(&config).is_empty()); - } - - #[test] - fn flags_ii() { - let config = ManagedLauncherConfig { - ii: true, - ..default_config() - }; - assert_eq!(launcher_settings_flags(&config), vec!["--ii"]); - } - - #[test] - fn flags_nns() { - let config = ManagedLauncherConfig { - nns: true, - ..default_config() - }; - assert_eq!(launcher_settings_flags(&config), vec!["--nns"]); - } - - #[test] - fn flags_artificial_delay() { - let config = ManagedLauncherConfig { - artificial_delay_ms: Some(100), - ..default_config() - }; - assert_eq!( - launcher_settings_flags(&config), - vec!["--artificial-delay-ms=100"] - ); - } - - #[test] - fn flags_subnets() { - let config = ManagedLauncherConfig { - subnets: Some(vec![SubnetKind::Application, SubnetKind::Bitcoin]), - ..default_config() - }; - assert_eq!( - launcher_settings_flags(&config), - vec!["--subnet=application", "--subnet=bitcoin"] - ); - } - - #[test] - fn flags_bitcoind_addr() { - let config = ManagedLauncherConfig { - bitcoind_addr: Some(vec!["127.0.0.1:18444".to_string()]), - ..default_config() - }; - assert_eq!( - launcher_settings_flags(&config), - vec!["--bitcoind-addr=127.0.0.1:18444"] - ); - } - - #[test] - fn flags_dogecoind_addr() { - let config = ManagedLauncherConfig { - dogecoind_addr: Some(vec!["127.0.0.1:22556".to_string()]), - ..default_config() - }; - assert_eq!( - launcher_settings_flags(&config), - vec!["--dogecoind-addr=127.0.0.1:22556"] - ); - } - - #[test] - fn flags_full_config() { - let config = ManagedLauncherConfig { - gateway: Gateway { - host: "localhost".to_string(), - port: Port::Fixed(8000), - }, - artificial_delay_ms: Some(50), - ii: true, - nns: true, - subnets: Some(vec![SubnetKind::Application]), - bitcoind_addr: Some(vec!["127.0.0.1:18444".to_string()]), - dogecoind_addr: Some(vec!["127.0.0.1:22556".to_string()]), - }; - let flags = launcher_settings_flags(&config); - assert_eq!( - flags, - vec![ - "--ii", - "--nns", - "--artificial-delay-ms=50", - "--subnet=application", - "--bitcoind-addr=127.0.0.1:18444", - "--dogecoind-addr=127.0.0.1:22556", - ] - ); - } -} - #[derive(Debug, Snafu)] pub enum WaitForLauncherStatusError { WaitForFile { source: WaitForFileError }, diff --git a/crates/icp/src/network/managed/run.rs b/crates/icp/src/network/managed/run.rs index 9d6903e0..5ca6b621 100644 --- a/crates/icp/src/network/managed/run.rs +++ b/crates/icp/src/network/managed/run.rs @@ -789,7 +789,6 @@ mod tests { assert!(opts.extra_hosts.is_empty()); assert!(opts.rm_on_exit); assert_eq!(opts.status_dir, "/app/status"); - // Port binding maps host 8000 to container 4943 let binding = opts .port_bindings .get("4943/tcp") @@ -835,14 +834,11 @@ mod tests { dogecoind_addr: None, }; let opts = transform_native_launcher_to_container(&config); - // --ii flag should be present assert!(opts.args.contains(&"--ii".to_string())); - // bitcoind-addr should be translated for Docker assert!( opts.args .contains(&"--bitcoind-addr=host.docker.internal:18444".to_string()) ); - // extra_hosts should include host.docker.internal mapping assert_eq!( opts.extra_hosts, vec!["host.docker.internal:host-gateway".to_string()] @@ -885,12 +881,10 @@ mod tests { dogecoind_addr: None, }; let opts = transform_native_launcher_to_container(&config); - // External address should pass through unchanged assert!( opts.args .contains(&"--bitcoind-addr=192.168.1.5:18444".to_string()) ); - // No extra_hosts needed for external addresses assert!(opts.extra_hosts.is_empty()); } }