Adding compact subcommand

docs/commands/compact.md

@@ -0,0 +1,20 @@
+# compact
+
+Compacts an MPQ archive. It performs a complete archive rebuild, effectively defragmenting the MPQ archive, removing all gaps that have been created by adding, replacing, renaming or deleting files within the archive. To succeed, the function requires all files in MPQ archive to be accessible and their filenames to be known.
+
+This may take several minutes to complete for large archives.
+
+```bash
+$ mpqcli compact wow-patch.mpq
+[*] Compacting archive. This may take some time...
+```
+
+
+## Use an external listfile
+
+The compact command requires all files inside the archive to be known. Older MPQ archives do not contain (complete) file paths of their content. By using the `-l` or `--listfile` argument, one can provide an external listfile that lists the content of the MPQ archive. Listfiles can be downloaded on [Ladislav Zezula's site](http://www.zezula.net/en/mpq/download.html).
+
+```bash
+$ mpqcli compact -l /path/to/listfile DIABDAT.MPQ
+[*] Compacting archive. This may take some time...
+```

src/commands.cpp

@@ -338,3 +338,14 @@ int HandleVerify(const std::string &target, bool printSignature) {
     CloseMpqArchive(hArchive);
     return result;
 }
+
+int HandleCompact(const std::string &target, const std::optional<std::string> &listfileName) {
+    HANDLE hArchive;
+    if (!OpenMpqArchive(target, &hArchive, 0)) {
+        return 1;
+    }
+
+    const int result = CompactMpqArchive(hArchive, listfileName);
+    CloseMpqArchive(hArchive);
+    return result;
+}

src/commands.h

@@ -32,5 +32,6 @@ int HandleExtract(const std::string &target, const std::optional<std::string> &o
 int HandleRead(const std::string &file, const std::string &target,
                const std::optional<std::string> &locale);
 int HandleVerify(const std::string &target, bool printSignature);
+int HandleCompact(const std::string &target, const std::optional<std::string> &listfileName);
 
 #endif  // COMMANDS_H

src/main.cpp

@@ -25,7 +25,7 @@ int main(int argc, char **argv) {
     std::optional<std::string> baseLocale;         // add, create, extract, read, remove
     std::optional<std::string> basePath;           // add, create
     std::optional<std::string> baseOutput;         // create, extract
-    std::optional<std::string> baseListfileName;   // extract, list
+    std::optional<std::string> baseListfileName;   // extract, list, compact
     std::optional<std::string> baseGameProfile;    // add, create
     int64_t fileDwFlags = -1;                      // add, create
     int64_t fileDwCompression = -1;                // add, create
@@ -213,6 +213,14 @@ int main(int argc, char **argv) {
         ->check(CLI::ExistingFile);
     verify->add_flag("-p,--print", verifyPrintSignature, "Print the digital signature (in hex)");
 
+    // Subcommand: compact
+    CLI::App *compact = app.add_subcommand("compact", "Compact the MPQ archive");
+    compact->add_option("target", baseTarget, "Target MPQ archive")
+        ->required()
+        ->check(CLI::ExistingFile);
+    compact->add_option("-l,--listfile", baseListfileName, "File listing content of an MPQ archive")
+        ->check(CLI::ExistingFile);
+
     try {
         app.parse(argc, argv);
     } catch (const CLI::ParseError &e) {
@@ -295,5 +303,9 @@ int main(int argc, char **argv) {
         return HandleVerify(baseTarget, verifyPrintSignature);
     }
 
+    if (app.got_subcommand(compact)) {
+        return HandleCompact(baseTarget, baseListfileName);
+    }
+
     return 0;
 }

src/mpq.cpp

@@ -651,6 +651,20 @@ uint32_t VerifyMpqArchive(HANDLE hArchive) {
     return SFileVerifyArchive(hArchive);
 }
 
+int CompactMpqArchive(HANDLE hArchive, const std::optional<std::string> &listfileName) {
+    std::cout << "[*] Compacting archive. This may take some time..." << std::endl;
+    // Check if the user provided a listfile input
+    const char *listfile = listfileName.has_value() ? listfileName->c_str() : nullptr;
+
+    if (!SFileCompactArchive(hArchive, listfile, false)) {
+        const auto error = SErrGetLastError();
+        std::cerr << "[!] Failed to compact archive: (" << error << ") " << StormErrorString(error)
+                  << std::endl;
+        return 1;
+    }
+    return 0;
+}
+
 int32_t PrintMpqSignature(HANDLE hArchive, const std::string &target) {
     // Determine if we have a strong or weak digital signature
     int32_t signatureType = GetFileInfo<int32_t>(hArchive, SFileMpqSignatures);

src/mpq.h

@@ -36,6 +36,7 @@ std::unique_ptr<char[]> ReadFile(HANDLE hArchive, const char *szFileName, unsign
                                  LCID preferredLocale);
 void PrintMpqInfo(HANDLE hArchive, const std::optional<std::string> &infoProperty);
 uint32_t VerifyMpqArchive(HANDLE hArchive);
+int CompactMpqArchive(HANDLE hArchive, const std::optional<std::string> &listfileName);
 int32_t PrintMpqSignature(HANDLE hArchive, const std::string &target);
 
 template <typename T>

test/test_compact.py

@@ -0,0 +1,169 @@
+import random
+import shutil
+from pathlib import Path
+import subprocess
+
+
+def test_compact_nonexistent_file(binary_path):
+    """
+    Test compacting non-existent MPQ file.
+
+    This test checks:
+    - If the application exits correctly.
+    """
+    script_dir = Path(__file__).parent
+    test_file = script_dir / "data" / "non_existent_file.mpq"
+
+    result = subprocess.run(
+        [str(binary_path), "compact", str(test_file)],
+        stdout=subprocess.PIPE,
+        stderr=subprocess.PIPE,
+        text=True
+    )
+
+    assert result.returncode != 1, f"mpqcli failed with error: {result.stderr}"
+
+
+def test_compact_nonmpq_file(binary_path):
+    """
+    Test compacting illegal MPQ file.
+
+    This test checks:
+    - If the application exits correctly.
+    """
+    script_dir = Path(__file__).parent
+    test_file = script_dir / "data" / "cats.txt"
+
+    expected_prefix = "[!] Failed to open MPQ archive"
+
+    result = subprocess.run(
+        [str(binary_path), "compact", str(test_file)],
+        stdout=subprocess.PIPE,
+        stderr=subprocess.PIPE,
+        text=True
+    )
+
+    output_lines = result.stderr.splitlines()
+
+    assert result.returncode == 1, f"mpqcli failed with error: {result.stderr}"
+    assert len(output_lines) == 1, f"Unexpected output: {output_lines}"
+    assert output_lines[0].startswith(expected_prefix), f"Unexpected output: {output_lines}"
+
+
+def test_compact_mpq_without_listfile(binary_path, generate_mpq_without_internal_listfile):
+    """
+    Test compacting MPQ file with no internal listfile,
+    and no externally provided one.
+
+    This test checks:
+    - That the application exits correctly and prints error message.
+    """
+    _ = generate_mpq_without_internal_listfile
+    script_dir = Path(__file__).parent
+    test_file = script_dir / "data" / "mpq_without_internal_listfile.mpq"
+
+    expected_prefix = "[!] Failed to compact archive: (10007) At least one file name is unknown (listfile is incomplete)"
+
+    result = subprocess.run(
+        [str(binary_path), "compact", str(test_file)],
+        stdout=subprocess.PIPE,
+        stderr=subprocess.PIPE,
+        text=True
+    )
+
+    output_lines = result.stderr.splitlines()
+
+    assert result.returncode == 1, f"mpqcli failed with error: {result.stderr}"
+    assert len(output_lines) == 1, f"Unexpected output: {output_lines}"
+    assert output_lines[0].startswith(expected_prefix), f"Unexpected output: {output_lines}"
+
+
+def test_compact_mpq_with_listfile(binary_path, generate_mpq_without_internal_listfile):
+    """
+    Test compacting MPQ file with no internal listfile,
+    instead providing an external one.
+
+    This test checks:
+    - That compaction works when the user provides an external listfile.
+    """
+    _ = generate_mpq_without_internal_listfile
+    script_dir = Path(__file__).parent
+    test_file = script_dir / "data" / "mpq_without_internal_listfile.mpq"
+    listfile = script_dir / "data" / "listfile.txt"
+    listfile.write_text("cats.txt\ndogs.txt\ncapybaras.txt")
+
+    expected_output = ["[*] Compacting archive. This may take some time..."]
+
+    result = subprocess.run(
+        [str(binary_path), "compact", str(test_file), "--listfile", str(listfile)],
+        stdout=subprocess.PIPE,
+        stderr=subprocess.PIPE,
+        text=True
+    )
+
+    output_lines = result.stdout.splitlines()
+
+    assert result.returncode == 0, f"mpqcli failed with error: {result.stderr}"
+    assert output_lines == expected_output, f"Unexpected output: {output_lines}"
+
+
+def test_compact_file(binary_path):
+    """
+    Test compacting an MPQ archive.
+
+    This test checks:
+    - Creating an archive with a small file and a ~512 KB incompressible file
+    - Removing the larger file from the archive
+    - Compacting the archive
+    - Verifying that the archive size shrinks significantly afterwards
+    """
+    script_dir = Path(__file__).parent
+    data_dir = script_dir / "data"
+
+    compaction_files_dir = data_dir / "compaction_files"
+    shutil.rmtree(compaction_files_dir, ignore_errors=True)
+    compaction_files_dir.mkdir(parents=True, exist_ok=True)
+
+    mpq_file = data_dir / "mpq_for_compaction.mpq"
+    mpq_file.unlink(missing_ok=True)
+
+    small_file = compaction_files_dir / "small.txt"
+    small_file.write_text("This is a small text file.\n", newline="\n")
+
+    large_file_name = "large.bin"
+    large_file = compaction_files_dir / large_file_name
+    large_file_size = 512 * 1024
+    large_file.write_bytes(random.Random(0).randbytes(large_file_size))
+
+    result = subprocess.run(
+        [str(binary_path), "create", "-o", str(mpq_file), str(compaction_files_dir)],
+        stdout=subprocess.PIPE,
+        stderr=subprocess.PIPE,
+        text=True
+    )
+    assert result.returncode == 0, f"create failed: {result.stderr}"
+
+    result = subprocess.run(
+        [str(binary_path), "remove", str(mpq_file), large_file_name],
+        stdout=subprocess.PIPE,
+        stderr=subprocess.PIPE,
+        text=True
+    )
+    assert result.returncode == 0, f"remove failed: {result.stderr}"
+
+    size_before = mpq_file.stat().st_size
+
+    result = subprocess.run(
+        [str(binary_path), "compact", str(mpq_file)],
+        stdout=subprocess.PIPE,
+        stderr=subprocess.PIPE,
+        text=True
+    )
+    assert result.returncode == 0, f"compact failed: {result.stderr}"
+
+    size_after = mpq_file.stat().st_size
+
+    assert size_before - size_after > size_before / 2, (
+        f"Expected compaction to shrink the archive significantly, "
+        f"but size went from {size_before} bytes to {size_after} bytes"
+    )