diff --git a/LICENSE b/LICENSE index 5b19074..c99f1e9 100644 --- a/LICENSE +++ b/LICENSE @@ -1,4 +1,4 @@ -Copyright (c) 2024, Mapbox +Copyright (c) 2026, Mapbox All rights reserved. Redistribution and use in source and binary forms, with or without diff --git a/README.md b/README.md index 75ffb33..a3e3200 100644 --- a/README.md +++ b/README.md @@ -7,17 +7,20 @@ A low-level, fast, ultra-lightweight (3KB gzipped) JavaScript library for decodi ## Performance -This library is extremely fast — much faster than native `JSON.parse`/`JSON.stringify` -and the [protocol-buffers](https://github.com/mafintosh/protocol-buffers) module. -Here's a result from running a real-world benchmark on Node v6.5 -(decoding and encoding a sample of 439 vector tiles, 22.6 MB total): - -- **pbf** decode: 387ms, or 57 MB/s -- **pbf** encode: 396ms, or 56 MB/s -- **protocol-buffers** decode: 837ms, or 26 MB/s -- **protocol-buffers** encode: 4197ms, or 5 MB/s -- **JSON.parse**: 1540ms, or 15 MB/s (parsing an equivalent 77.5 MB JSON file) -- **JSON.stringify**: 607ms, or 49 MB/s +This library is fast — competitive with or faster than other JS protobuf implementations, +and orders of magnitude smaller. Here's a result from a real-world benchmark on Node v26 +(decoding and encoding 439 Mapbox vector tiles, 37.5 MB total; the equivalent JSON is 136 MB): + +| | decode | encode | +|------------------|-------------------|-------------------| +| **pbf** | 200ms, 187 MB/s | 188ms, 200 MB/s | +| protocol-buffers | 297ms, 126 MB/s | 620ms, 60 MB/s | +| protobuf.js | 226ms, 169 MB/s | 510ms, 74 MB/s | +| JSON | 488ms, 278 MB/s | 267ms, 509 MB/s | + +`JSON` throughput is measured against the 136 MB JSON payload, not the 37.5 MB pbf payload — +on the same data, pbf is ~2× faster to decode and ~2.5× faster to encode, and produces output +roughly a quarter the size. See `bench/bench-tiles.js`. ## Examples @@ -33,14 +36,14 @@ $ pbf example.proto > example.js Then read and write objects using the module like this: ```js -import Pbf from 'pbf'; +import {PbfReader, PbfWriter} from 'pbf'; import {readExample, writeExample} from './example.js'; // read -var obj = readExample(new Pbf(buffer)); +const obj = readExample(new PbfReader(buffer)); // write -const pbf = new Pbf(); +const pbf = new PbfWriter(); writeExample(obj, pbf); const buffer = pbf.finish(); ``` @@ -58,7 +61,7 @@ const {readExample, writeExample} = compile(proto); #### Custom Reading ```js -var data = new Pbf(buffer).readFields(readData, {}); +const data = new PbfReader(buffer).readFields(readData, {}); function readData(tag, data, pbf) { if (tag === 1) data.name = pbf.readString(); @@ -74,9 +77,9 @@ function readLayer(tag, layer, pbf) { #### Custom Writing ```js -var pbf = new Pbf(); +const pbf = new PbfWriter(); writeData(data, pbf); -var buffer = pbf.finish(); +const buffer = pbf.finish(); function writeData(data, pbf) { pbf.writeStringField(1, data.name); @@ -94,18 +97,18 @@ function writeLayer(layer, pbf) { Install using NPM with `npm install pbf`, then import as a module: ```js -import Pbf from 'pbf'; +import {PbfReader, PbfWriter} from 'pbf'; ``` Or use as a module directly in the browser with [jsDelivr](https://www.jsdelivr.com/esm): ```html ``` -Alternatively, there's a browser bundle with a `Pbf` global variable: +Alternatively, there's a browser bundle exposing a `Pbf` global with `PbfReader` and `PbfWriter` properties: ```html @@ -113,17 +116,19 @@ Alternatively, there's a browser bundle with a `Pbf` global variable: ## API -Create a `Pbf` object, optionally given a `Buffer` or `Uint8Array` as input data: +The library exposes two classes: `PbfReader` for decoding and `PbfWriter` for encoding. Splitting them lets bundlers tree-shake the half you don't use. + +Create a `PbfReader` from a `Buffer` or `Uint8Array`: ```js // parse a pbf file from disk in Node -const pbf = new Pbf(fs.readFileSync('data.pbf')); +const pbf = new PbfReader(fs.readFileSync('data.pbf')); // parse a pbf file in a browser after an ajax request with responseType="arraybuffer" -const pbf = new Pbf(new Uint8Array(xhr.response)); +const pbf = new PbfReader(new Uint8Array(xhr.response)); ``` -`Pbf` object properties: +Both classes expose the following properties: ```js pbf.length; // length of the underlying buffer @@ -143,7 +148,7 @@ pbf.readFields((tag) => { ``` It optionally accepts an object that will be passed to the reading function for easier construction of decoded data, -and also passes the `Pbf` object as a third argument: +and also passes the `PbfReader` object as a third argument: ```js const result = pbf.readFields(readField, {}) @@ -205,6 +210,12 @@ Packed reading methods: #### Writing +Create a `PbfWriter` (optionally with a pre-allocated `Buffer` or `Uint8Array`): + +```js +const pbf = new PbfWriter(); +``` + Write values: ```js @@ -287,7 +298,7 @@ The `--legacy` switch makes it generate a CommonJS module instead of ESM. `Pbf` will generate `read` and `write` functions for every message in the schema. For nested messages, their names will be concatenated — e.g. `Message` inside `Test` will produce `readTestMessage` and `writeTestMessage` functions. -* `read(pbf)` - decodes an object from the given `Pbf` instance. -* `write(obj, pbf)` - encodes an object into the given `Pbf` instance (usually empty). +* `read(pbf)` - decodes an object from the given `PbfReader` instance. +* `write(obj, pbf)` - encodes an object into the given `PbfWriter` instance (usually empty). The resulting code is clean and simple, so it's meant to be customized. diff --git a/bench/bench-tiles.js b/bench/bench-tiles.js index 1a512ee..66d356d 100644 --- a/bench/bench-tiles.js +++ b/bench/bench-tiles.js @@ -3,21 +3,44 @@ import {createHash} from 'crypto'; import {mkdirSync, existsSync, readFileSync, writeFileSync} from 'fs'; import {join, dirname} from 'path'; import {fileURLToPath} from 'url'; +import protocolBuffers from 'protocol-buffers'; +import protobufjs from 'protobufjs'; import {readTile, writeTile} from '../test/fixtures/vector_tile.js'; -import Pbf from '../index.js'; +import {PbfReader, PbfWriter} from '../index.js'; -const token = process.env.ACCESS_TOKEN; -if (!token) throw new Error('Missing ACCESS_TOKEN environment variable (Mapbox access token).'); +const vtProtoUrl = new URL('../test/fixtures/vector_tile.proto', import.meta.url); +const ProtocolBuffersTile = protocolBuffers(readFileSync(vtProtoUrl)).Tile; +const ProtobufjsTile = protobufjs.loadSync(fileURLToPath(vtProtoUrl)).lookup('vector_tile.Tile'); const tilesetId = 'mapbox.mapbox-streets-v8'; -const url = `https://api.mapbox.com/v4/${tilesetId}/{z}/{x}/{y}.mvt?access_token=${token}`; +const urlTemplate = `https://api.mapbox.com/v4/${tilesetId}/{z}/{x}/{y}.mvt?access_token={token}`; + +const libs = { + pbf: { + decode: body => readTile(new PbfReader(body)), + encode: (tile) => { const pbf = new PbfWriter(); writeTile(tile, pbf); return pbf.finish(); } + }, + 'protocol-buffers': { + decode: body => ProtocolBuffersTile.decode(body), + encode: tile => ProtocolBuffersTile.encode(tile) + }, + 'protobuf.js': { + decode: body => ProtobufjsTile.decode(body), + encode: tile => ProtobufjsTile.encode(tile).finish() + }, + JSON: { + decode: body => JSON.parse(body), + encode: tile => JSON.stringify(tile) + } +}; -let readTime = 0; -let writeTime = 0; +const stats = {}; +for (const name of Object.keys(libs)) stats[name] = {read: 0, write: 0}; let size = 0; +let jsonSize = 0; let numTiles = 0; -await runStats(url, processTile, showStats, { +await runStats(urlTemplate, processTile, showStats, { width: 2880, height: 1800, minZoom: 0, @@ -30,23 +53,47 @@ function processTile(body) { size += body.length; numTiles++; - let now = clock(); - const tile = readTile(new Pbf(body)); - readTime += clock(now); - - now = clock(); - const pbf = new Pbf(); - writeTile(tile, pbf); - const buf = pbf.finish(); - writeTime += clock(now); - - console.assert(buf); + // pre-decode once with pbf to get a tile object usable as input for encoders + const tile = readTile(new PbfReader(body)); + const tileJSON = JSON.stringify(tile); + jsonSize += tileJSON.length; + const pbInput = ProtocolBuffersTile.decode(body); + const pbjsInput = ProtobufjsTile.decode(body); + + const inputs = { + pbf: body, + 'protocol-buffers': body, + 'protobuf.js': body, + JSON: tileJSON + }; + const encodeInputs = { + pbf: tile, + 'protocol-buffers': pbInput, + 'protobuf.js': pbjsInput, + JSON: tile + }; + + for (const name of Object.keys(libs)) { + let now = clock(); + libs[name].decode(inputs[name]); + stats[name].read += clock(now); + + now = clock(); + libs[name].encode(encodeInputs[name]); + stats[name].write += clock(now); + } } function showStats() { - console.log('%d tiles, %d KB total', numTiles, Math.round(size / 1024)); - console.log('read time: %dms, or %d MB/s', Math.round(readTime), speed(readTime, size)); - console.log('write time: %dms, or %d MB/s', Math.round(writeTime), speed(writeTime, size)); + console.log('%d tiles, %d KB pbf / %d KB JSON', numTiles, Math.round(size / 1024), Math.round(jsonSize / 1024)); + for (const name of Object.keys(libs)) { + const inSize = name === 'JSON' ? jsonSize : size; + const s = stats[name]; + console.log('%s decode: %dms, %d MB/s encode: %dms, %d MB/s', + name.padEnd(18), + Math.round(s.read), speed(s.read, inSize), + Math.round(s.write), speed(s.write, inSize)); + } } function speed(time, size) { @@ -61,7 +108,8 @@ function clock(start) { async function runStats(urlTemplate, onTile, onDone, options) { const cacheRoot = join(dirname(fileURLToPath(import.meta.url)), 'tile-cache'); - const cachePath = join(cacheRoot, createHash('sha1').update(urlTemplate).digest('hex').slice(0, 8)); + const cacheKey = urlTemplate.replace(/access_token=[^&]*/, ''); + const cachePath = join(cacheRoot, createHash('sha1').update(cacheKey).digest('hex').slice(0, 8)); mkdirSync(cachePath, {recursive: true}); const tilePromises = []; @@ -97,10 +145,14 @@ async function loadTile(z, x, y, urlTemplate, cachePath) { return readFileSync(tilePath); } + const token = process.env.MAPBOX_ACCESS_TOKEN; + if (!token) throw new Error('Missing MAPBOX_ACCESS_TOKEN environment variable.'); + const tileUrl = urlTemplate .replace('{x}', x) .replace('{y}', y) - .replace('{z}', z); + .replace('{z}', z) + .replace('{token}', token); const response = await fetch(tileUrl); diff --git a/bench/bench.html b/bench/bench.html index ca8e469..09244af 100644 --- a/bench/bench.html +++ b/bench/bench.html @@ -6,14 +6,14 @@