feat: support large wasms

.github/workflows/test.yml

@@ -121,6 +121,16 @@ jobs:
           which ic-wasm
           ic-wasm --version
 
+      - name: Install wasm-tools
+        uses: taiki-e/install-action@3ae7038495f79cd771208b38319ff728a8b24538 # v2.67.3
+        with:
+          tool: wasm-tools
+
+      - name: Verify wasm-tools installation
+        run: |
+          which wasm-tools
+          wasm-tools --version
+
       - name: install network launcher
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

CHANGELOG.md

@@ -11,6 +11,7 @@
   * The corresponding environment "ic" is defined implicitly which can be overwritten by user configuration.
   * The `--mainnet` and `--ic` flags are removed. Use `-n/--network ic`, `-e/--environment ic` instead.
 * feat: Allow overriding the implicit `local` network and environment.
+* feat: Allow installing WASMs that are larger than 2MB
 
 # v0.1.0-beta.3
 

crates/icp-cli/Cargo.toml

@@ -53,6 +53,7 @@ sec1.workspace = true
 serde_json.workspace = true
 serde_yaml.workspace = true
 serde.workspace = true
+sha2.workspace = true
 shellwords.workspace = true
 snafu.workspace = true
 sysinfo.workspace = true

crates/icp-cli/src/operations/install.rs

@@ -1,9 +1,12 @@
 use futures::{StreamExt, stream::FuturesOrdered};
 use ic_agent::{Agent, AgentError, export::Principal};
-use ic_management_canister_types::{UpgradeFlags, WasmMemoryPersistence};
+use ic_management_canister_types::{
+    CanisterId, ChunkHash, UpgradeFlags, UploadChunkArgs, WasmMemoryPersistence,
+};
 use ic_utils::interfaces::{
     ManagementCanister, management_canister::builders::CanisterInstallMode,
 };
+use sha2::{Digest, Sha256};
 use snafu::Snafu;
 use std::sync::Arc;
 use tracing::debug;
@@ -76,15 +79,114 @@ pub(crate) async fn install_canister(
         canister_name, install_mode
     );
 
-    let mut builder = mgmt.install_code(canister_id, wasm).with_mode(install_mode);
+    do_install_operation(
+        agent,
+        canister_id,
+        canister_name,
+        wasm,
+        install_mode,
+        init_args,
+    )
+    .await
+}
 
-    if let Some(args) = init_args {
-        builder = builder.with_raw_arg(args.into());
-    }
+async fn do_install_operation(
+    agent: &Agent,
+    canister_id: &Principal,
+    canister_name: &str,
+    wasm: &[u8],
+    mode: CanisterInstallMode,
+    init_args: Option<&[u8]>,
+) -> Result<(), InstallOperationError> {
+    let mgmt = ManagementCanister::create(agent);
+
+    // Threshold for chunked installation: 2 MB
+    // Raw install_code messages are limited to 2 MiB
+    const CHUNK_THRESHOLD: usize = 2 * 1024 * 1024;
+
+    // Chunk size: 1 MB (spec limit is 1 MiB per chunk)
+    const CHUNK_SIZE: usize = 1024 * 1024;
+
+    // Generous overhead for encoding, target canister ID, install mode, etc.
+    const ENCODING_OVERHEAD: usize = 500;
+
+    // Calculate total install message size
+    let init_args_len = init_args.map_or(0, |args| args.len());
+    let total_install_size = wasm.len() + init_args_len + ENCODING_OVERHEAD;
+
+    if total_install_size <= CHUNK_THRESHOLD {
+        // Small wasm: use regular install_code
+        debug!("Installing wasm for {canister_name} using install_code");
+
+        let mut builder = mgmt.install_code(canister_id, wasm).with_mode(mode);
+
+        if let Some(args) = init_args {
+            builder = builder.with_raw_arg(args.into());
+        }
 
-    builder
-        .await
-        .map_err(|source| InstallOperationError::Agent { source })?;
+        builder
+            .await
+            .map_err(|source| InstallOperationError::Agent { source })?;
+    } else {
+        // Large wasm: use chunked installation
+        debug!("Installing wasm for {canister_name} using chunked installation");
+
+        // Clear any existing chunks to ensure a clean state
+        mgmt.clear_chunk_store(canister_id)
+            .await
+            .map_err(|source| InstallOperationError::Agent { source })?;
+
+        // Split wasm into chunks and upload them
+        let chunks: Vec<&[u8]> = wasm.chunks(CHUNK_SIZE).collect();
+        let mut chunk_hashes: Vec<ChunkHash> = Vec::new();
+
+        for (i, chunk) in chunks.iter().enumerate() {
+            debug!(
+                "Uploading chunk {}/{} ({} bytes)",
+                i + 1,
+                chunks.len(),
+                chunk.len()
+            );
+
+            let upload_args = UploadChunkArgs {
+                canister_id: CanisterId::from(*canister_id),
+                chunk: chunk.to_vec(),
+            };
+
+            let (chunk_hash,) = mgmt
+                .upload_chunk(canister_id, &upload_args)
+                .await
+                .map_err(|source| InstallOperationError::Agent { source })?;
+
+            chunk_hashes.push(chunk_hash);
+        }
+
+        // Compute SHA-256 hash of the entire wasm module
+        let mut hasher = Sha256::new();
+        hasher.update(wasm);
+        let wasm_module_hash = hasher.finalize().to_vec();
+
+        debug!("Installing chunked code with {} chunks", chunk_hashes.len());
+
+        // Build and execute install_chunked_code
+        let mut builder = mgmt
+            .install_chunked_code(canister_id, &wasm_module_hash)
+            .with_chunk_hashes(chunk_hashes)
+            .with_install_mode(mode);
+
+        if let Some(args) = init_args {
+            builder = builder.with_raw_arg(args.to_vec());
+        }
+
+        builder
+            .await
+            .map_err(|source| InstallOperationError::Agent { source })?;
+
+        // Clear chunk store after successful installation to free up storage
+        mgmt.clear_chunk_store(canister_id)
+            .await
+            .map_err(|source| InstallOperationError::Agent { source })?;
+    }
 
     Ok(())
 }

crates/icp-cli/tests/assets/.gitignore

@@ -0,0 +1,2 @@
+large.wat
+large.wasm

crates/icp-cli/tests/assets/generate_large_wasm.sh

@@ -0,0 +1,20 @@
+#!/usr/bin/env bash
+
+# Skip if large.wasm already exists and is larger than 3MB (3000000 bytes)
+if [[ -f large.wasm ]] && [[ $(stat -f%z large.wasm 2>/dev/null || stat -c%s large.wasm 2>/dev/null) -gt 3000000 ]]; then
+  echo "large.wasm already exists and is >3MB, skipping generation"
+  exit 0
+fi
+
+# Create a WAT file with a large data segment
+{
+  echo '(module'
+  echo '  (memory 64)'
+  printf '  (data (i32.const 0) "'
+  dd if=/dev/zero bs=1 count=3000000 2>/dev/null | tr '\0' 'A'
+  echo '")'
+  echo ')'
+} > large.wat
+
+# Convert to Wasm
+wasm-tools parse large.wat -o large.wasm

crates/icp-cli/tests/canister_install_tests.rs

@@ -533,3 +533,94 @@ async fn canister_install_with_environment_settings_override() {
         output_str
     );
 }
+
+#[cfg(unix)] // requires bash and wasm-tools in PATH
+#[tokio::test]
+async fn canister_install_large_wasm_chunked() {
+    // Generate large.wasm which is greater than 3MB
+    let assets_dir = Path::new(env!("CARGO_MANIFEST_DIR"))
+        .join("tests")
+        .join("assets");
+    std::process::Command::new("bash")
+        .current_dir(&assets_dir)
+        .arg("generate_large_wasm.sh")
+        .status()
+        .expect("failed to run generate_large_wasm.sh");
+
+    let ctx = TestContext::new();
+
+    // Setup project
+    let project_dir = ctx.create_project_dir("icp");
+
+    // Use the 3MB wasm file to test chunked installation
+    let wasm = ctx.make_asset("large.wasm");
+
+    // Project manifest
+    let pm = formatdoc! {r#"
+        canisters:
+          - name: large-canister
+            build:
+              steps:
+                - type: script
+                  command: cp {wasm} "$ICP_WASM_OUTPUT_PATH"
+
+        {NETWORK_RANDOM_PORT}
+        {ENVIRONMENT_RANDOM_PORT}
+    "#};
+
+    write_string(
+        &project_dir.join("icp.yaml"), // path
+        &pm,                           // contents
+    )
+    .expect("failed to write project manifest");
+
+    // Start network
+    let _g = ctx.start_network_in(&project_dir, "random-network").await;
+    ctx.ping_until_healthy(&project_dir, "random-network");
+
+    // Build canister
+    ctx.icp()
+        .current_dir(&project_dir)
+        .args(["build", "large-canister"])
+        .assert()
+        .success();
+
+    ctx.icp()
+        .current_dir(&project_dir)
+        .args([
+            "canister",
+            "create",
+            "large-canister",
+            "--environment",
+            "random-environment",
+        ])
+        .assert()
+        .success();
+
+    // Install large canister (should use chunked installation)
+    ctx.icp()
+        .current_dir(&project_dir)
+        .args([
+            "canister",
+            "install",
+            "large-canister",
+            "--environment",
+            "random-environment",
+        ])
+        .assert()
+        .success();
+
+    // Verify the installation by checking the canister status
+    ctx.icp()
+        .current_dir(&project_dir)
+        .args([
+            "canister",
+            "status",
+            "--environment",
+            "random-environment",
+            "large-canister",
+        ])
+        .assert()
+        .success()
+        .stdout(contains("Status: Running"));
+}