diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index baffa7b6..99b62a5c 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -121,6 +121,16 @@ jobs: which ic-wasm ic-wasm --version + - name: Install wasm-tools + uses: taiki-e/install-action@3ae7038495f79cd771208b38319ff728a8b24538 # v2.67.3 + with: + tool: wasm-tools + + - name: Verify wasm-tools installation + run: | + which wasm-tools + wasm-tools --version + - name: install network launcher env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/CHANGELOG.md b/CHANGELOG.md index 449b98b5..8a29c3ca 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,7 @@ * The corresponding environment "ic" is defined implicitly which can be overwritten by user configuration. * The `--mainnet` and `--ic` flags are removed. Use `-n/--network ic`, `-e/--environment ic` instead. * feat: Allow overriding the implicit `local` network and environment. +* feat: Allow installing WASMs that are larger than 2MB # v0.1.0-beta.3 diff --git a/crates/icp-cli/Cargo.toml b/crates/icp-cli/Cargo.toml index b52e087e..aa5bd7b9 100644 --- a/crates/icp-cli/Cargo.toml +++ b/crates/icp-cli/Cargo.toml @@ -53,6 +53,7 @@ sec1.workspace = true serde_json.workspace = true serde_yaml.workspace = true serde.workspace = true +sha2.workspace = true shellwords.workspace = true snafu.workspace = true sysinfo.workspace = true diff --git a/crates/icp-cli/src/operations/install.rs b/crates/icp-cli/src/operations/install.rs index 91e0bbc0..e307ccd0 100644 --- a/crates/icp-cli/src/operations/install.rs +++ b/crates/icp-cli/src/operations/install.rs @@ -1,9 +1,12 @@ use futures::{StreamExt, stream::FuturesOrdered}; use ic_agent::{Agent, AgentError, export::Principal}; -use ic_management_canister_types::{UpgradeFlags, WasmMemoryPersistence}; +use ic_management_canister_types::{ + CanisterId, ChunkHash, UpgradeFlags, UploadChunkArgs, WasmMemoryPersistence, +}; use ic_utils::interfaces::{ ManagementCanister, management_canister::builders::CanisterInstallMode, }; +use sha2::{Digest, Sha256}; use snafu::Snafu; use std::sync::Arc; use tracing::debug; @@ -76,15 +79,114 @@ pub(crate) async fn install_canister( canister_name, install_mode ); - let mut builder = mgmt.install_code(canister_id, wasm).with_mode(install_mode); + do_install_operation( + agent, + canister_id, + canister_name, + wasm, + install_mode, + init_args, + ) + .await +} - if let Some(args) = init_args { - builder = builder.with_raw_arg(args.into()); - } +async fn do_install_operation( + agent: &Agent, + canister_id: &Principal, + canister_name: &str, + wasm: &[u8], + mode: CanisterInstallMode, + init_args: Option<&[u8]>, +) -> Result<(), InstallOperationError> { + let mgmt = ManagementCanister::create(agent); + + // Threshold for chunked installation: 2 MB + // Raw install_code messages are limited to 2 MiB + const CHUNK_THRESHOLD: usize = 2 * 1024 * 1024; + + // Chunk size: 1 MB (spec limit is 1 MiB per chunk) + const CHUNK_SIZE: usize = 1024 * 1024; + + // Generous overhead for encoding, target canister ID, install mode, etc. + const ENCODING_OVERHEAD: usize = 500; + + // Calculate total install message size + let init_args_len = init_args.map_or(0, |args| args.len()); + let total_install_size = wasm.len() + init_args_len + ENCODING_OVERHEAD; + + if total_install_size <= CHUNK_THRESHOLD { + // Small wasm: use regular install_code + debug!("Installing wasm for {canister_name} using install_code"); + + let mut builder = mgmt.install_code(canister_id, wasm).with_mode(mode); + + if let Some(args) = init_args { + builder = builder.with_raw_arg(args.into()); + } - builder - .await - .map_err(|source| InstallOperationError::Agent { source })?; + builder + .await + .map_err(|source| InstallOperationError::Agent { source })?; + } else { + // Large wasm: use chunked installation + debug!("Installing wasm for {canister_name} using chunked installation"); + + // Clear any existing chunks to ensure a clean state + mgmt.clear_chunk_store(canister_id) + .await + .map_err(|source| InstallOperationError::Agent { source })?; + + // Split wasm into chunks and upload them + let chunks: Vec<&[u8]> = wasm.chunks(CHUNK_SIZE).collect(); + let mut chunk_hashes: Vec = Vec::new(); + + for (i, chunk) in chunks.iter().enumerate() { + debug!( + "Uploading chunk {}/{} ({} bytes)", + i + 1, + chunks.len(), + chunk.len() + ); + + let upload_args = UploadChunkArgs { + canister_id: CanisterId::from(*canister_id), + chunk: chunk.to_vec(), + }; + + let (chunk_hash,) = mgmt + .upload_chunk(canister_id, &upload_args) + .await + .map_err(|source| InstallOperationError::Agent { source })?; + + chunk_hashes.push(chunk_hash); + } + + // Compute SHA-256 hash of the entire wasm module + let mut hasher = Sha256::new(); + hasher.update(wasm); + let wasm_module_hash = hasher.finalize().to_vec(); + + debug!("Installing chunked code with {} chunks", chunk_hashes.len()); + + // Build and execute install_chunked_code + let mut builder = mgmt + .install_chunked_code(canister_id, &wasm_module_hash) + .with_chunk_hashes(chunk_hashes) + .with_install_mode(mode); + + if let Some(args) = init_args { + builder = builder.with_raw_arg(args.to_vec()); + } + + builder + .await + .map_err(|source| InstallOperationError::Agent { source })?; + + // Clear chunk store after successful installation to free up storage + mgmt.clear_chunk_store(canister_id) + .await + .map_err(|source| InstallOperationError::Agent { source })?; + } Ok(()) } diff --git a/crates/icp-cli/tests/assets/.gitignore b/crates/icp-cli/tests/assets/.gitignore new file mode 100644 index 00000000..2ad46e17 --- /dev/null +++ b/crates/icp-cli/tests/assets/.gitignore @@ -0,0 +1,2 @@ +large.wat +large.wasm \ No newline at end of file diff --git a/crates/icp-cli/tests/assets/generate_large_wasm.sh b/crates/icp-cli/tests/assets/generate_large_wasm.sh new file mode 100755 index 00000000..55d5d2a5 --- /dev/null +++ b/crates/icp-cli/tests/assets/generate_large_wasm.sh @@ -0,0 +1,20 @@ +#!/usr/bin/env bash + +# Skip if large.wasm already exists and is larger than 3MB (3000000 bytes) +if [[ -f large.wasm ]] && [[ $(stat -f%z large.wasm 2>/dev/null || stat -c%s large.wasm 2>/dev/null) -gt 3000000 ]]; then + echo "large.wasm already exists and is >3MB, skipping generation" + exit 0 +fi + +# Create a WAT file with a large data segment +{ + echo '(module' + echo ' (memory 64)' + printf ' (data (i32.const 0) "' + dd if=/dev/zero bs=1 count=3000000 2>/dev/null | tr '\0' 'A' + echo '")' + echo ')' +} > large.wat + +# Convert to Wasm +wasm-tools parse large.wat -o large.wasm diff --git a/crates/icp-cli/tests/canister_install_tests.rs b/crates/icp-cli/tests/canister_install_tests.rs index f4783e97..7e4dbdd6 100644 --- a/crates/icp-cli/tests/canister_install_tests.rs +++ b/crates/icp-cli/tests/canister_install_tests.rs @@ -533,3 +533,94 @@ async fn canister_install_with_environment_settings_override() { output_str ); } + +#[cfg(unix)] // requires bash and wasm-tools in PATH +#[tokio::test] +async fn canister_install_large_wasm_chunked() { + // Generate large.wasm which is greater than 3MB + let assets_dir = Path::new(env!("CARGO_MANIFEST_DIR")) + .join("tests") + .join("assets"); + std::process::Command::new("bash") + .current_dir(&assets_dir) + .arg("generate_large_wasm.sh") + .status() + .expect("failed to run generate_large_wasm.sh"); + + let ctx = TestContext::new(); + + // Setup project + let project_dir = ctx.create_project_dir("icp"); + + // Use the 3MB wasm file to test chunked installation + let wasm = ctx.make_asset("large.wasm"); + + // Project manifest + let pm = formatdoc! {r#" + canisters: + - name: large-canister + build: + steps: + - type: script + command: cp {wasm} "$ICP_WASM_OUTPUT_PATH" + + {NETWORK_RANDOM_PORT} + {ENVIRONMENT_RANDOM_PORT} + "#}; + + write_string( + &project_dir.join("icp.yaml"), // path + &pm, // contents + ) + .expect("failed to write project manifest"); + + // Start network + let _g = ctx.start_network_in(&project_dir, "random-network").await; + ctx.ping_until_healthy(&project_dir, "random-network"); + + // Build canister + ctx.icp() + .current_dir(&project_dir) + .args(["build", "large-canister"]) + .assert() + .success(); + + ctx.icp() + .current_dir(&project_dir) + .args([ + "canister", + "create", + "large-canister", + "--environment", + "random-environment", + ]) + .assert() + .success(); + + // Install large canister (should use chunked installation) + ctx.icp() + .current_dir(&project_dir) + .args([ + "canister", + "install", + "large-canister", + "--environment", + "random-environment", + ]) + .assert() + .success(); + + // Verify the installation by checking the canister status + ctx.icp() + .current_dir(&project_dir) + .args([ + "canister", + "status", + "--environment", + "random-environment", + "large-canister", + ]) + .assert() + .success() + .stdout(contains("Status: Running")); +}