Merge branch 'master' into dev-1.2

Author: cracksalad

.github/workflows/ci.yml

@@ -15,7 +15,7 @@ jobs:
     steps:
       - uses: actions/checkout@v4
       - name: Cache Composer dependencies
-        uses: actions/cache@v3
+        uses: actions/cache@v4
         with:
           path: /tmp/composer-cache
           key: ${{ runner.os }}-${{ hashFiles('**/composer.lock') }}
@@ -34,7 +34,7 @@ jobs:
         uses: actions/checkout@v4
 
       - name: Psalm
-        uses: docker://ghcr.io/psalm/psalm-github-actions
+        uses: docker://ghcr.io/psalm/psalm-github-actions:6.4.1
         with:
           args: --shepherd
 
@@ -57,7 +57,7 @@ jobs:
     steps:
       - uses: actions/checkout@v4
       - name: Cache Composer dependencies
-        uses: actions/cache@v3
+        uses: actions/cache@v4
         with:
           path: /tmp/composer-cache
           key: ${{ runner.os }}-${{ hashFiles('**/composer.lock') }}

.phive/phars.xml

@@ -1,6 +1,6 @@
 <?xml version="1.0" encoding="UTF-8"?>
 <phive xmlns="https://phar.io/phive">
-  <phar name="phpunit" version="^10.0.0" installed="10.5.3" location="./tools/phpunit" copy="true"/>
-  <phar name="psalm" version="^5.12.0" installed="5.17.0" location="./tools/psalm" copy="true"/>
+  <phar name="phpunit" version="^10.0.0" installed="10.5.45" location="./tools/phpunit" copy="true"/>
+  <phar name="psalm" version="~6.4.0" installed="6.4.1" location="./tools/psalm" copy="true"/>
   <phar name="phpcpd" version="^6.0.0" installed="6.0.3" location="./tools/phpcpd" copy="true"/>
 </phive>

README.md

@@ -33,6 +33,7 @@ There are pretty much no dependencies with some exceptions:
 
 - If you want to use the `toJSONFile()` method, you need to install `ext-json` (PHP's PECL JSON extension) as well.
 - If you want to use the igbinary serializer, `ext-igbinary` is required. See [php-ext-igbinary](https://github.com/igbinary/igbinary).
+- If you want to use the msgpack serializer, `ext-msgpack` is required. See [php-ext-msgpack](https://github.com/msgpack/msgpack-php).
 - If you want to use LZ4 compression, `ext-lz4` is required. See [php-ext-lz4](https://github.com/kjdev/php-ext-lz4).
 
 ## Usage
@@ -62,9 +63,9 @@ The constructor of `LargeArrayBuffer` provides some options:
 
 1. You can set the threshold when to move the data to disk. When pushing data to the buffer, it is stored in memory until it gets too large.
     E.g.: `new LargeArrayBuffer(512);` to set a 512 MiB threshold. 
-1. You can choose either the PHP serializer or the [igbinary](https://github.com/igbinary/igbinary) serializer (PHP serializer is default).
+1. You can choose either the PHP serializer, the [igbinary](https://github.com/igbinary/igbinary) serializer or the [msgpack](https://github.com/msgpack/msgpack-php) serializer (PHP serializer is default).
     E.g.: `new LargeArrayBuffer(serializer: LargeArrayBuffer::COMPRESSION_IGBINARY);`
-1. You can enable GZIP or LZ4 compression for the serialized items. Although this is recommended only if your items are pretty big like > 1 KiB each. E.g.: `new LargeArrayBuffer(compression: LargeArrayBuffer::COMPRESSION_GZIP);`. Note, that LZ4 compression requires [ext-lz4](https://github.com/kjdev/php-ext-lz4) to be installed.
+1. You can enable GZIP or LZ4 compression for the serialized items. Although this is recommended only if your items are pretty big like > 1 KiB each. E.g.: `new LargeArrayBuffer(compression: LargeArrayBuffer::COMPRESSION_GZIP);`. Note, that LZ4 compression requires [ext-lz4](https://github.com/kjdev/php-ext-lz4) to be loaded.
 
 ### Read from the buffer
 

bench/benchmark.php

@@ -97,7 +97,7 @@ function printResult(string $label, array $metrics, string $key, int $tabs = 1,
   unset($buf);
 
   // buffer with LZ4
-  if(function_exists('lz4_compress')){
+  if(extension_loaded('lz4')){
     $start = microtime(true);
     $memBefore = memory_get_usage(true);
     $buf = new LargeArrayBuffer(128, compression: LargeArrayBuffer::COMPRESSION_LZ4);
@@ -118,7 +118,7 @@ function printResult(string $label, array $metrics, string $key, int $tabs = 1,
     unset($buf);
   }
 
-  if(function_exists('igbinary_serialize')){
+  if(extension_loaded('igbinary')){
     // normal buffer with igbinary
     $start = microtime(true);
     $memBefore = memory_get_usage(true);
@@ -160,7 +160,7 @@ function printResult(string $label, array $metrics, string $key, int $tabs = 1,
     unset($buf);
 
     // buffer with LZ4 and igbinary
-    if(function_exists('lz4_compress')){
+    if(extension_loaded('lz4')){
       $start = microtime(true);
       $memBefore = memory_get_usage(true);
       $buf = new LargeArrayBuffer(128, serializer: LargeArrayBuffer::SERIALIZER_IGBINARY, compression: LargeArrayBuffer::COMPRESSION_LZ4);
@@ -182,6 +182,70 @@ function printResult(string $label, array $metrics, string $key, int $tabs = 1,
     }
   }
 
+  if(extension_loaded('msgpack')){
+    // normal buffer with msgpack
+    $start = microtime(true);
+    $memBefore = memory_get_usage(true);
+    $buf = new LargeArrayBuffer(128, serializer: LargeArrayBuffer::SERIALIZER_MSGPACK);
+    $bench->bufferMeasurementsFill($buf);
+    $metrics['fill_buffer_mp'][] = [
+      'time' => microtime(true) - $start,
+      'mem' => memory_get_usage(true) - $memBefore,
+      'size' => $buf->getSize()
+    ];
+    
+    $start = microtime(true);
+    $bench->bufferMeasurementsIterate($buf);
+    $metrics['iterate_buffer_mp'][] = [
+      'time' => microtime(true) - $start,
+      'mem' => memory_get_usage(true) - $memBefore,
+      'size' => $buf->getSize()
+    ];
+    unset($buf);
+    
+    // buffer with GZIP and msgpack
+    $start = microtime(true);
+    $memBefore = memory_get_usage(true);
+    $buf = new LargeArrayBuffer(128, serializer: LargeArrayBuffer::SERIALIZER_MSGPACK, compression: LargeArrayBuffer::COMPRESSION_GZIP);
+    $bench->bufferMeasurementsFill($buf);
+    $metrics['fill_buffer_gz_mp'][] = [
+      'time' => microtime(true) - $start,
+      'mem' => memory_get_usage(true) - $memBefore,
+      'size' => $buf->getSize()
+    ];
+    
+    $start = microtime(true);
+    $bench->bufferMeasurementsIterate($buf);
+    $metrics['iterate_buffer_gz_mp'][] = [
+      'time' => microtime(true) - $start,
+      'mem' => memory_get_usage(true) - $memBefore,
+      'size' => $buf->getSize()
+    ];
+    unset($buf);
+    
+    // buffer with LZ4 and msgpack
+    if(extension_loaded('lz4')){
+      $start = microtime(true);
+      $memBefore = memory_get_usage(true);
+      $buf = new LargeArrayBuffer(128, serializer: LargeArrayBuffer::SERIALIZER_MSGPACK, compression: LargeArrayBuffer::COMPRESSION_LZ4);
+      $bench->bufferMeasurementsFill($buf);
+      $metrics['fill_buffer_lz4_mp'][] = [
+        'time' => microtime(true) - $start,
+        'mem' => memory_get_usage(true) - $memBefore,
+        'size' => $buf->getSize()
+      ];
+      
+      $start = microtime(true);
+      $bench->bufferMeasurementsIterate($buf);
+      $metrics['iterate_buffer_lz4_mp'][] = [
+        'time' => microtime(true) - $start,
+        'mem' => memory_get_usage(true) - $memBefore,
+        'size' => $buf->getSize()
+      ];
+      unset($buf);
+    }
+  }
+  
   unset($bench);
 }
 
@@ -191,17 +255,27 @@ function printResult(string $label, array $metrics, string $key, int $tabs = 1,
 printResult('Iterate over buffer', $metrics, 'iterate_buffer', 3, true);
 printResult('Fill buffer (GZIP)', $metrics, 'fill_buffer_gz', 3, true);
 printResult('Iterate over buffer (GZIP)', $metrics, 'iterate_buffer_gz', 2, true);
-if(function_exists('lz4_compress')){
+if(extension_loaded('lz4')){
   printResult('Fill buffer (LZ4)', $metrics, 'fill_buffer_lz4', 3, true);
   printResult('Iterate over buffer (LZ4)', $metrics, 'iterate_buffer_lz4', 2, true);
 }
-if(function_exists('igbinary_serialize')){
+if(extension_loaded('igbinary')){
   printResult('Fill buffer (igbinary)', $metrics, 'fill_buffer_ig', 2, true);
   printResult('Iterate over buffer (igbinary)', $metrics, 'iterate_buffer_ig', 1, true);
   printResult('Fill buffer (GZIP, igbinary)', $metrics, 'fill_buffer_gz_ig', 2, true);
   printResult('Iterate over buffer (GZIP, igbinary)', $metrics, 'iterate_buffer_gz_ig', 1, true);
-  if(function_exists('lz4_compress')){
+  if(extension_loaded('lz4')){
     printResult('Fill buffer (LZ4, igbinary)', $metrics, 'fill_buffer_lz4_ig', 2, true);
     printResult('Iterate over buffer (LZ4, igbinary)', $metrics, 'iterate_buffer_lz4_ig', 1, true);
   }
 }
+if(extension_loaded('msgpack')){
+  printResult('Fill buffer (msgpack)', $metrics, 'fill_buffer_mp', 2, true);
+  printResult('Iterate over buffer (msgpack)', $metrics, 'iterate_buffer_mp', 1, true);
+  printResult('Fill buffer (GZIP, msgpack)', $metrics, 'fill_buffer_gz_mp', 2, true);
+  printResult('Iterate over buffer (GZIP, msgpack)', $metrics, 'iterate_buffer_gz_mp', 1, true);
+  if(extension_loaded('lz4')){
+    printResult('Fill buffer (LZ4, msgpack)', $metrics, 'fill_buffer_lz4_mp', 2, true);
+    printResult('Iterate over buffer (LZ4, msgpack)', $metrics, 'iterate_buffer_lz4_mp', 1, true);
+  }
+}

composer.json

@@ -1,6 +1,6 @@
 {
     "name": "nerou/large-array-buffer",
-    "version": "1.0.1",
+    "version": "1.1.0",
     "type": "library",
     "license": "MIT",
     "authors": [
@@ -14,8 +14,9 @@
     "suggest": {
         "ext-json": "Requirement of toJSONFile() method",
         "ext-zlib": "To enable support for GZIP compression",
-        "ext-lz4": "To enable support for LZ4 compression",
-        "ext-igbinary": "To enable support for igbinary serializer"
+        "ext-lz4": "To enable support of LZ4 compression",
+        "ext-igbinary": "To enable support for igbinary serializer",
+        "ext-msgpack": "To enable support for msgpack serializer"
     },
     "require": {
         "php": ">=8.0"

psalm.baseline.xml

@@ -4,6 +4,8 @@
     <UndefinedFunction>
       <code>lz4_compress($serialized)</code>
       <code>lz4_uncompress($compressed)</code>
+      <code>msgpack_serialize($item)</code>
+      <code>msgpack_unserialize($this->current)</code>
     </UndefinedFunction>
   </file>
 </files>

psalm.xml

@@ -7,7 +7,8 @@
 		    errorBaseline="psalm.baseline.xml"
 		    errorLevel="2"
 		    findUnusedBaselineEntry="false"
-		    findUnusedCode="false">
+		    findUnusedCode="false"
+		    phpVersion="8.0">
 
     <projectFiles>
         <directory name="src" />

src/LargeArrayBuffer.php

@@ -12,6 +12,7 @@ class LargeArrayBuffer implements ArrayBufferInterface {
 
   public const SERIALIZER_PHP = 1;
   public const SERIALIZER_IGBINARY = 2;
+  public const SERIALIZER_MSGPACK = 3;
 
   public const COMPRESSION_NONE = 0;
   public const COMPRESSION_GZIP = 1;
@@ -57,13 +58,16 @@ class LargeArrayBuffer implements ArrayBufferInterface {
    */
   public function __construct(int $maxMemoryMiB = 1024, int $serializer = self::SERIALIZER_PHP, int $compression = self::COMPRESSION_NONE) {
     $this->serializer = $serializer;
-    if($this->serializer === self::SERIALIZER_IGBINARY && !function_exists('igbinary_serialize')){
-      throw new \InvalidArgumentException('igbinary serializer was requested, but ext-igbinary is not installed');
+    if($this->serializer === self::SERIALIZER_IGBINARY && !extension_loaded('igbinary')){
+      throw new \InvalidArgumentException('igbinary serializer was requested, but ext-igbinary is not loaded');
+    }
+    if($this->serializer === self::SERIALIZER_MSGPACK && !extension_loaded('msgpack')){
+      throw new \InvalidArgumentException('msgpack serializer was requested, but ext-msgpack is not loaded');
     }
 
     $this->compression = $compression;
-    if($this->compression === self::COMPRESSION_LZ4 && !function_exists('lz4_compress')){
-      throw new \InvalidArgumentException('LZ4 compression was requested, but ext-lz4 is not installed');
+    if($this->compression === self::COMPRESSION_LZ4 && !extension_loaded('lz4')){
+      throw new \InvalidArgumentException('LZ4 compression was requested, but ext-lz4 is not loaded');
     }
 
     $stream = fopen('php://temp/maxmemory:'.($maxMemoryMiB * 1024 * 1024), 'r+');
@@ -75,13 +79,17 @@ public function __construct(int $maxMemoryMiB = 1024, int $serializer = self::SE
 
   /**
    * @psalm-param E $item
-   * @throws \RuntimeException if unable to write to php://temp
+   * @throws \RuntimeException if unable to write to php://temp, the serialization failed or the compression failed
    */
   public function push(mixed $item): void {
     $serialized = match($this->serializer){
       self::SERIALIZER_IGBINARY => igbinary_serialize($item),
+      self::SERIALIZER_MSGPACK => msgpack_serialize($item),
       default => serialize($item)
     };
+    if($serialized === false){
+      throw new \RuntimeException('failed to serialize data');
+    }
     /** @var string|false $compressed */
     $compressed = match($this->compression){
       self::COMPRESSION_GZIP => gzdeflate($serialized),
@@ -148,6 +156,7 @@ public function current(): mixed {
     /** @psalm-var E $res */
     $res = match($this->serializer){
       self::SERIALIZER_IGBINARY => igbinary_unserialize($this->current),
+      self::SERIALIZER_MSGPACK => msgpack_unserialize($this->current),
       default => unserialize($this->current)
     };
     return $res;
@@ -213,7 +222,14 @@ public function toJSONFile($dest, int $flags = JSON_THROW_ON_ERROR, int $depth =
       if(($flags & JSON_PRETTY_PRINT) > 0){
         fwrite($stream, PHP_EOL.'    ');
       }
-      fwrite($stream, json_encode($item, $flags, $depth));
+      $json = json_encode($item, $flags, $depth);
+      if($json === false){
+        if(is_string($dest)){
+          fclose($stream);
+        }
+        throw new \RuntimeException('failed to serialize data');
+      }
+      fwrite($stream, $json);
       fflush($stream);
       $first = false;
     }