From 210d90da7434735befa93708c2d5ab624e035494 Mon Sep 17 00:00:00 2001 From: Felix Lee Date: Fri, 1 Sep 2017 12:01:48 -0700 Subject: [PATCH 1/3] add benchmarks --- bench/README.md | 17 +++++++ bench/bench.js | 118 +++++++++++++++++++++++++++++++++++++++++++++++ bench/cargo.json | 108 +++++++++++++++++++++++++++++++++++++++++++ bench/cargo.toml | 77 +++++++++++++++++++++++++++++++ bench/cargo.yaml | 90 ++++++++++++++++++++++++++++++++++++ bench/empty.ini | 1 + bench/empty.json | 1 + bench/empty.toml | 1 + bench/empty.yaml | 1 + bench/hello.ini | 2 + bench/hello.json | 5 ++ bench/hello.toml | 2 + bench/hello.yaml | 2 + bench/run.js | 12 +++++ package.json | 11 ++++- 15 files changed, 447 insertions(+), 1 deletion(-) create mode 100644 bench/README.md create mode 100644 bench/bench.js create mode 100644 bench/cargo.json create mode 100644 bench/cargo.toml create mode 100644 bench/cargo.yaml create mode 100644 bench/empty.ini create mode 100644 bench/empty.json create mode 100644 bench/empty.toml create mode 100644 bench/empty.yaml create mode 100644 bench/hello.ini create mode 100644 bench/hello.json create mode 100644 bench/hello.toml create mode 100644 bench/hello.yaml create mode 100644 bench/run.js mode change 100644 => 100755 package.json diff --git a/bench/README.md b/bench/README.md new file mode 100644 index 0000000..55809bd --- /dev/null +++ b/bench/README.md @@ -0,0 +1,17 @@ +Multi-parser benchmarks +======================= + +`node run.js` will run all benchmarks. + +`node run.js file.json ...` will run just the named benchmarks. + +Each benchmark is a collection of files that are equivalent +representations of the same data in different formats: + +- `{name}.json` is the expected result. +- `{name}.ini` is the data in ini format. +- `{name}.toml` is the data in toml format. +- `{name}.yaml` is the data in yaml format. + +Only the `.json` file is required. A parser will only be +benchmarked if there's an appropriate input file for it. diff --git a/bench/bench.js b/bench/bench.js new file mode 100644 index 0000000..99473d1 --- /dev/null +++ b/bench/bench.js @@ -0,0 +1,118 @@ +'use strict'; + +const Benchmark = require('benchmark'); +const deepEqual = require('deep-equal'); +const fs = require('fs'); +const glob = require('glob'); +const upath = require('upath'); + +const ini = require('ini'); +const jsYaml = require('js-yaml'); +const tomlJ = require('toml-j0.4'); +const tomlNode = require('../index.js'); +const yaml = require('yaml'); +const yamljs = require('yamljs'); + +const parsers = [ + { + name: 'ini', + format: 'ini', + parse: (str) => ini.parse(str), + }, + { + name: 'js-yaml', + format: 'yaml', + parse: (str) => jsYaml.safeLoad(str), + }, + // js-yaml can reliably parse json. the others, not so much. + { + name: 'js-yaml(json)', + format: 'json', + parse: (str) => jsYaml.safeLoad(str), + }, + { + name: 'json', + format: 'json', + parse: (str) => JSON.parse(str), + }, + { + name: 'toml-j0.4', + format: 'toml', + parse: (str) => tomlJ.parse(str), + }, + { + name: 'toml-node', + format: 'toml', + parse: (str) => tomlNode.parse(str), + }, + { + name: 'yaml', + format: 'yaml', + parse: (str) => yaml.eval(str), + }, + { + name: 'yamljs', + format: 'yaml', + parse: (str) => yamljs.parse(str), + }, +]; + +const formats = new Set(parsers.map((p) => p.format)); + +const longestName = parsers.reduce((a, p) => Math.max(a, p.name.length), 0); + +function benchAll() { + for (let fname of glob.sync(upath.join(__dirname, '*.json'))) { + benchOne(fname); + } +}; + +function benchOne(name) { + let suite = new Benchmark.Suite(); + let example = loadExample(name); + for (let parser of parsers) { + let benchName = `parse ${example.name} with ` + parser.name.padEnd(longestName, ' '); + + let input = example[parser.format]; + if (input == null) { + console.log(`${benchName} skipped`); + continue; + } + + // Verify the parser works correctly. + let expected = JSON.parse(example.json); + let actual; + try { + actual = parser.parse(input); + } catch (e) { + actual = String(e); + } + if (!deepEqual(actual, expected)) { + console.log(`${benchName} broken`); + console.error(' actual:\n', actual); + continue; + } + + suite.add(benchName, () => parser.parse(input)); + } + + suite.on('cycle', (event) => console.log(String(event.target))); + suite.run(); +} + +function loadExample(name) { + let path = upath.parse(name); + let example = { + name: path.name, + }; + for (let format of formats) { + let f = upath.join(path.dir, path.name + '.' + format); + if (fs.existsSync(f)) { + example[format] = fs.readFileSync(f, 'utf-8'); + } + } + return example; +} + +exports.benchOne = benchOne; +exports.benchAll = benchAll; diff --git a/bench/cargo.json b/bench/cargo.json new file mode 100644 index 0000000..260ef89 --- /dev/null +++ b/bench/cargo.json @@ -0,0 +1,108 @@ +{ + "package": { + "name": "cargo", + "version": "0.23.0", + "authors": [ + "Yehuda Katz ", + "Carl Lerche ", + "Alex Crichton " + ], + "license": "MIT/Apache-2.0", + "homepage": "https://crates.io", + "repository": "https://github.com/rust-lang/cargo", + "documentation": "https://docs.rs/cargo", + "description": "Cargo, a package manager for Rust.\n" + }, + "lib": { + "name": "cargo", + "path": "src/cargo/lib.rs" + }, + "dependencies": { + "atty": "0.2", + "crates-io": { + "path": "src/crates-io", + "version": "0.12" + }, + "crossbeam": "0.2", + "curl": "0.4.6", + "docopt": "0.8.1", + "env_logger": "0.4", + "error-chain": "0.11.0-rc.2", + "filetime": "0.1", + "flate2": "0.2", + "fs2": "0.4", + "git2": "0.6", + "git2-curl": "0.7", + "glob": "0.2", + "hex": "0.2", + "home": "0.3", + "ignore": "^0.2.2", + "jobserver": "0.1.6", + "libc": "0.2", + "libgit2-sys": "0.6", + "log": "0.3", + "num_cpus": "1.0", + "same-file": "0.1", + "scoped-tls": "0.1", + "semver": { + "version": "0.7.0", + "features": [ + "serde" + ] + }, + "serde": "1.0", + "serde_derive": "1.0", + "serde_ignored": "0.0.3", + "serde_json": "1.0", + "shell-escape": "0.1", + "tar": { + "version": "0.4", + "default-features": false + }, + "tempdir": "0.3", + "termcolor": "0.3", + "toml": "0.4", + "url": "1.1" + }, + "target": { + "cfg(unix)": { + "dependencies": { + "openssl": "0.9" + } + }, + "cfg(target_os = 'macos')": { + "dependencies": { + "core-foundation": { + "version": "0.4.4", + "features": [ + "mac_os_10_7_support" + ] + } + } + }, + "cfg(windows)": { + "dependencies": { + "advapi32-sys": "0.2", + "kernel32-sys": "0.2", + "miow": "0.2", + "psapi-sys": "0.1", + "winapi": "0.2" + } + } + }, + "dev-dependencies": { + "bufstream": "0.1", + "cargotest": { + "path": "tests/cargotest" + }, + "filetime": "0.1", + "hamcrest": "=0.1.1" + }, + "bin": [ + { + "name": "cargo", + "test": false, + "doc": false + } + ] +} diff --git a/bench/cargo.toml b/bench/cargo.toml new file mode 100644 index 0000000..c2b7737 --- /dev/null +++ b/bench/cargo.toml @@ -0,0 +1,77 @@ +[package] +name = "cargo" +version = "0.23.0" +authors = ["Yehuda Katz ", + "Carl Lerche ", + "Alex Crichton "] +license = "MIT/Apache-2.0" +homepage = "https://crates.io" +repository = "https://github.com/rust-lang/cargo" +documentation = "https://docs.rs/cargo" +description = """ +Cargo, a package manager for Rust. +""" + +[lib] +name = "cargo" +path = "src/cargo/lib.rs" + +[dependencies] +atty = "0.2" +crates-io = { path = "src/crates-io", version = "0.12" } +crossbeam = "0.2" +curl = "0.4.6" +docopt = "0.8.1" +env_logger = "0.4" +error-chain = "0.11.0-rc.2" +filetime = "0.1" +flate2 = "0.2" +fs2 = "0.4" +git2 = "0.6" +git2-curl = "0.7" +glob = "0.2" +hex = "0.2" +home = "0.3" +ignore = "^0.2.2" +jobserver = "0.1.6" +libc = "0.2" +libgit2-sys = "0.6" +log = "0.3" +num_cpus = "1.0" +same-file = "0.1" +scoped-tls = "0.1" +semver = { version = "0.7.0", features = ["serde"] } +serde = "1.0" +serde_derive = "1.0" +serde_ignored = "0.0.3" +serde_json = "1.0" +shell-escape = "0.1" +tar = { version = "0.4", default-features = false } +tempdir = "0.3" +termcolor = "0.3" +toml = "0.4" +url = "1.1" + +[target."cfg(unix)".dependencies] +openssl = "0.9" + +[target."cfg(target_os = 'macos')".dependencies] +core-foundation = { version = "0.4.4", features = ["mac_os_10_7_support"] } + +[target."cfg(windows)".dependencies] +advapi32-sys = "0.2" +kernel32-sys = "0.2" +miow = "0.2" +psapi-sys = "0.1" +winapi = "0.2" + +[dev-dependencies] +bufstream = "0.1" +cargotest = { path = "tests/cargotest" } +filetime = "0.1" +hamcrest = "=0.1.1" + +[[bin]] +name = "cargo" +test = false +doc = false diff --git a/bench/cargo.yaml b/bench/cargo.yaml new file mode 100644 index 0000000..a525e63 --- /dev/null +++ b/bench/cargo.yaml @@ -0,0 +1,90 @@ +package: + name: "cargo" + version: "0.23.0" + authors: + - "Yehuda Katz " + - "Carl Lerche " + - "Alex Crichton " + license: "MIT/Apache-2.0" + homepage: "https://crates.io" + repository: "https://github.com/rust-lang/cargo" + documentation: "https://docs.rs/cargo" + description: "Cargo, a package manager for Rust.\n" + +lib: + name: "cargo" + path: "src/cargo/lib.rs" + +dependencies: + atty: "0.2" + crates-io: + path: "src/crates-io" + version: "0.12" + crossbeam: "0.2" + curl: "0.4.6" + docopt: "0.8.1" + env_logger: "0.4" + error-chain: "0.11.0-rc.2" + filetime: "0.1" + flate2: "0.2" + fs2: "0.4" + git2: "0.6" + git2-curl: "0.7" + glob: "0.2" + hex: "0.2" + home: "0.3" + ignore: "^0.2.2" + jobserver: "0.1.6" + libc: "0.2" + libgit2-sys: "0.6" + log: "0.3" + num_cpus: "1.0" + same-file: "0.1" + scoped-tls: "0.1" + semver: + version: "0.7.0" + features: + - "serde" + serde: "1.0" + serde_derive: "1.0" + serde_ignored: "0.0.3" + serde_json: "1.0" + shell-escape: "0.1" + tar: + version: "0.4" + default-features: false + tempdir: "0.3" + termcolor: "0.3" + toml: "0.4" + url: "1.1" + +target: + "cfg(unix)": + dependencies: + openssl: "0.9" + "cfg(target_os = 'macos')": + dependencies: + core-foundation: + version: "0.4.4" + features: + - "mac_os_10_7_support" + "cfg(windows)": + dependencies: + advapi32-sys: "0.2" + kernel32-sys: "0.2" + miow: "0.2" + psapi-sys: "0.1" + winapi: "0.2" + +dev-dependencies: + bufstream: "0.1" + cargotest: + path: "tests/cargotest" + filetime: "0.1" + hamcrest: "=0.1.1" + +bin: + - + name: "cargo" + test: false + doc: false diff --git a/bench/empty.ini b/bench/empty.ini new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/bench/empty.ini @@ -0,0 +1 @@ + diff --git a/bench/empty.json b/bench/empty.json new file mode 100644 index 0000000..0967ef4 --- /dev/null +++ b/bench/empty.json @@ -0,0 +1 @@ +{} diff --git a/bench/empty.toml b/bench/empty.toml new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/bench/empty.toml @@ -0,0 +1 @@ + diff --git a/bench/empty.yaml b/bench/empty.yaml new file mode 100644 index 0000000..0967ef4 --- /dev/null +++ b/bench/empty.yaml @@ -0,0 +1 @@ +{} diff --git a/bench/hello.ini b/bench/hello.ini new file mode 100644 index 0000000..3f75031 --- /dev/null +++ b/bench/hello.ini @@ -0,0 +1,2 @@ +[hello] +world = "!" diff --git a/bench/hello.json b/bench/hello.json new file mode 100644 index 0000000..be0facf --- /dev/null +++ b/bench/hello.json @@ -0,0 +1,5 @@ +{ + "hello": { + "world": "!" + } +} diff --git a/bench/hello.toml b/bench/hello.toml new file mode 100644 index 0000000..3f75031 --- /dev/null +++ b/bench/hello.toml @@ -0,0 +1,2 @@ +[hello] +world = "!" diff --git a/bench/hello.yaml b/bench/hello.yaml new file mode 100644 index 0000000..3768cd6 --- /dev/null +++ b/bench/hello.yaml @@ -0,0 +1,2 @@ +hello: + world: "!" diff --git a/bench/run.js b/bench/run.js new file mode 100644 index 0000000..b7b5bdb --- /dev/null +++ b/bench/run.js @@ -0,0 +1,12 @@ +'usr strict'; + +const bench = require('./bench'); + +let args = process.argv.slice(2); +if (args.length) { + for (let arg of args) { + bench.benchOne(arg); + } +} else { + bench.benchAll(); +} diff --git a/package.json b/package.json old mode 100644 new mode 100755 index 53418e9..959c8e7 --- a/package.json +++ b/package.json @@ -17,8 +17,17 @@ "author": "Michelle Tilley ", "license": "MIT", "devDependencies": { + "benchmark": "^2.1.4", + "deep-equal": "^1.0.1", + "glob": "^7.1.2", + "ini": "^1.3.4", + "js-yaml": "^3.9.1", "jshint": "*", "nodeunit": "~0.9.0", - "pegjs": "~0.8.0" + "pegjs": "~0.8.0", + "toml-j0.4": "^1.1.1", + "upath": "^1.0.0", + "yaml": "^0.3.0", + "yamljs": "^0.3.0" } } From a243176bf16c0a810e72bc3a29bffd8bf07c3caf Mon Sep 17 00:00:00 2001 From: Felix Lee Date: Sat, 2 Sep 2017 10:15:45 -0700 Subject: [PATCH 2/3] allow selection of parsers --- bench/README.md | 3 +++ bench/bench.js | 18 +++++++++++++----- bench/run.js | 20 +++++++++++++++++--- 3 files changed, 33 insertions(+), 8 deletions(-) diff --git a/bench/README.md b/bench/README.md index 55809bd..2e58610 100644 --- a/bench/README.md +++ b/bench/README.md @@ -5,6 +5,9 @@ Multi-parser benchmarks `node run.js file.json ...` will run just the named benchmarks. +`node run.js parsers=a,b,...` will run just the named parsers. If +`parsers=` is specified more than once, the values are combined. + Each benchmark is a collection of files that are equivalent representations of the same data in different formats: diff --git a/bench/bench.js b/bench/bench.js index 99473d1..c6fa2f6 100644 --- a/bench/bench.js +++ b/bench/bench.js @@ -61,20 +61,28 @@ const formats = new Set(parsers.map((p) => p.format)); const longestName = parsers.reduce((a, p) => Math.max(a, p.name.length), 0); -function benchAll() { +function benchAll(onlyParsers) { for (let fname of glob.sync(upath.join(__dirname, '*.json'))) { - benchOne(fname); + benchOne(fname, onlyParsers); + } +} + +function benchOne(name, onlyParsers) { + if (onlyParsers && !onlyParsers.size) { + onlyParsers = null; } -}; -function benchOne(name) { let suite = new Benchmark.Suite(); let example = loadExample(name); for (let parser of parsers) { + if (onlyParsers && !onlyParsers.has(parser.name)) { + continue; + } + let benchName = `parse ${example.name} with ` + parser.name.padEnd(longestName, ' '); let input = example[parser.format]; - if (input == null) { + if (input === undefined) { console.log(`${benchName} skipped`); continue; } diff --git a/bench/run.js b/bench/run.js index b7b5bdb..0f1a35a 100644 --- a/bench/run.js +++ b/bench/run.js @@ -2,11 +2,25 @@ const bench = require('./bench'); -let args = process.argv.slice(2); +let args = []; +let onlyParsers = new Set(); + +// process any parsers= arguments +for (let arg of process.argv.slice(2)) { + let m = /parsers?=(.*)/.exec(arg); + if (m) { + for (let p of m[1].split(/[,\s]+/)) { + onlyParsers.add(p); + } + } else { + args.push(m); + } +} + if (args.length) { for (let arg of args) { - bench.benchOne(arg); + bench.benchOne(arg, onlyParsers); } } else { - bench.benchAll(); + bench.benchAll(onlyParsers); } From 289c332dbc0db340c7296af6088be13981ac44cf Mon Sep 17 00:00:00 2001 From: Felix Lee Date: Sat, 2 Sep 2017 11:55:54 -0700 Subject: [PATCH 3/3] fix typo --- bench/run.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bench/run.js b/bench/run.js index 0f1a35a..e6f2b25 100644 --- a/bench/run.js +++ b/bench/run.js @@ -13,7 +13,7 @@ for (let arg of process.argv.slice(2)) { onlyParsers.add(p); } } else { - args.push(m); + args.push(arg); } }