diff --git a/.gitignore b/.gitignore index 1fdad4c..ab7b2bd 100644 --- a/.gitignore +++ b/.gitignore @@ -3,3 +3,4 @@ dist/ build/ .env data/ +*.wav \ No newline at end of file diff --git a/package-lock.json b/package-lock.json index 244a372..09d5917 100644 --- a/package-lock.json +++ b/package-lock.json @@ -14,7 +14,8 @@ "@picovoice/pvrecorder-node": "^1.2.8", "@subspace/reed-solomon-erasure.wasm": "^0.2.5", "chalk": "^5.6.2", - "fft.js": "^4.0.4" + "fft.js": "^4.0.4", + "wavefile": "^11.0.0" }, "devDependencies": { "@types/node": "^25.3.3", @@ -492,6 +493,18 @@ "integrity": "sha512-AsuCzffGHJybSaRrmr5eHr81mwJU3kjw6M+uprWvCXiNeN9SOGwQ3Jn8jb8m3Z6izVgknn1R0FTCEAP2QrLY/w==", "dev": true, "license": "MIT" + }, + "node_modules/wavefile": { + "version": "11.0.0", + "resolved": "https://registry.npmjs.org/wavefile/-/wavefile-11.0.0.tgz", + "integrity": "sha512-/OBiAALgWU24IG7sC84cDO/KfFuvajWc5Uec0oV2zrpOOZZDgGdOwHwgEzOrwh8jkubBk7PtZfQBIcI1OaE5Ng==", + "license": "MIT", + "bin": { + "wavefile": "bin/wavefile.js" + }, + "engines": { + "node": ">=8" + } } } } diff --git a/package.json b/package.json index fcdd9ae..1e28874 100644 --- a/package.json +++ b/package.json @@ -29,6 +29,7 @@ "@picovoice/pvrecorder-node": "^1.2.8", "@subspace/reed-solomon-erasure.wasm": "^0.2.5", "chalk": "^5.6.2", - "fft.js": "^4.0.4" + "fft.js": "^4.0.4", + "wavefile": "^11.0.0" } } diff --git a/src/core/modulator/embedChips.ts b/src/core/modulator/embedChips.ts index ef869ff..908b41d 100644 --- a/src/core/modulator/embedChips.ts +++ b/src/core/modulator/embedChips.ts @@ -4,7 +4,7 @@ */ export function embedFrameChips( - fftComplex: Float32Array, + fftComplex: Float32Array | number[], chipMap: Map, N: number, delta: number = 0.02, @@ -15,6 +15,10 @@ export function embedFrameChips( const rIndex = binIndex * 2; const iIndex = rIndex + 1; + if (iIndex >= fftComplex.length) { + continue; + } + const real = fftComplex[rIndex]!; const imag = fftComplex[iIndex]!; const magnitude = Math.sqrt(real * real + imag * imag); diff --git a/src/core/profiler/fft.ts b/src/core/profiler/fft.ts index db935fb..a4ef802 100644 --- a/src/core/profiler/fft.ts +++ b/src/core/profiler/fft.ts @@ -1,18 +1,27 @@ import FFT from "fft.js"; const FRAME_SIZE = 1024; const f = new FFT(FRAME_SIZE); -const out = f.createComplexArray(); +/** + * Processes a windowed frame of audio samples using the Fast Fourier Transform (FFT) + * @param {Float32Array} windowedFrame - a windowed frame of audio samples + * @returns {Object} - an object containing the FFT complex array and power spectrum array + * @property {number[]} fftComplex - the FFT complex array + * @property {Float32Array} powerSpectrum - the power spectrum array + */ export function processFFT(windowedFrame: Float32Array) { - f.realTransform(out, windowedFrame); + const fftComplex = f.createComplexArray(); + f.realTransform(fftComplex, windowedFrame); + f.completeSpectrum(fftComplex); + const powerSpectrum = new Float32Array(FRAME_SIZE / 2); for (let i = 0; i < FRAME_SIZE / 2; i++) { - const real = out[2 * i]!; - const imag = out[2 * i + 1]!; + const real = fftComplex[2 * i]!; + const imag = fftComplex[2 * i + 1]!; powerSpectrum[i] = real * real + imag * imag; } - return powerSpectrum; + return { fftComplex, powerSpectrum }; } diff --git a/src/core/profiler/processFrame.ts b/src/core/profiler/processFrame.ts index ce1d39b..3963594 100644 --- a/src/core/profiler/processFrame.ts +++ b/src/core/profiler/processFrame.ts @@ -10,33 +10,36 @@ import { HAMMING_WINDOW } from "./window.js"; const HOP_SIZE = 512; const FRAME_SIZE = 1024; - -let frameCount = 0; +interface FrameState { + count: number; +} export function processSTFT( audioBuffer: AudioRingBuffer, bitstream: Uint8Array, bitPtr: { index: number }, pnGen: PNGenerator, + frameState: FrameState = { count: 0 }, ) { const outSpectra = []; while (audioBuffer.size >= FRAME_SIZE) { - const frames = audioBuffer.getFrames(FRAME_SIZE); + const rawFrames = audioBuffer.getFrames(FRAME_SIZE); + const frames = new Float32Array(rawFrames); for (let i = 0; i < FRAME_SIZE; i++) frames[i] = frames[i]! * HAMMING_WINDOW[i]!; console.log("Frame min/max:", Math.min(...frames), Math.max(...frames)); - const fftComplex = processFFT(frames); - const bandEnergy = computeBarkEnergy(fftComplex); + const { fftComplex, powerSpectrum } = processFFT(frames); + const bandEnergy = computeBarkEnergy(powerSpectrum); const maskingThresholds = estimateMasking(bandEnergy); - const safeBins = identifySafeBins(fftComplex, maskingThresholds); + const safeBins = identifySafeBins(powerSpectrum, maskingThresholds); console.log(`Found ${safeBins.length} safe bins for data injection`); - let finalSpectrum = fftComplex; + let finalSpectrum: Float32Array = new Float32Array(fftComplex); if (safeBins.length >= 64 && bitPtr.index < bitstream.length) { const bit = bitstream[bitPtr.index]!; @@ -44,23 +47,24 @@ export function processSTFT( const pnSequence = pnGen.generateSequence(64); const spreadChips = DSSS_Spreader.spread(symbol, pnSequence); - const chipMap = BinMapper.mapToBins(spreadChips, safeBins, frameCount); + const chipMap = BinMapper.mapToBins( + spreadChips, + safeBins, + frameState.count, + ); finalSpectrum = embedFrameChips(fftComplex, chipMap, FRAME_SIZE, 0.02); bitPtr.index++; } - const map = { - frameIndex: frameCount++, - safeBins, - bandEnergy: new Float32Array(bandEnergy), - maskingThresholds: new Float32Array(maskingThresholds), - }; + const currentFrameIndex = frameState.count++; outSpectra.push({ spectrum: finalSpectrum, - frameIndex: frameCount++, + frameIndex: currentFrameIndex, + safeBins, + bandEnergy: new Float32Array(bandEnergy), + maskingThresholds: new Float32Array(maskingThresholds), }); - audioBuffer.advance(HOP_SIZE); } diff --git a/src/core/profiler/recorder.ts b/src/core/profiler/recorder.ts index b110fb1..56d3b64 100644 --- a/src/core/profiler/recorder.ts +++ b/src/core/profiler/recorder.ts @@ -2,6 +2,10 @@ import { PvRecorder } from "@picovoice/pvrecorder-node"; import { buffer } from "../../types/AudioRingBuffer.js"; import { processSTFT } from "./processFrame.js"; import { PNGenerator } from "../modulator/pnGen.js"; +import { processIFFT } from "../recombination/ifft.js"; +import { overlapAdd } from "../recombination/reconstructor.js"; +import { floatToInt16 } from "../recombination/pcmConverter.js"; +import { collectOutput, saveWAV } from "../recombination/writer.js"; export async function recorder(bitstream: Uint8Array) { const frameSize = 512; @@ -19,37 +23,23 @@ export async function recorder(bitstream: Uint8Array) { if (buffer.size >= 1024) { const modifiedFrames = processSTFT(buffer, bitstream, bitPtr, pnGen); - for (const frame of modifiedFrames) + for (const frame of modifiedFrames) { console.log( `Frame: ${frame.frameIndex} | Progress: ${bitPtr.index}/${bitstream.length}`, ); + const timeframe = processIFFT(frame.spectrum); + const synthFrame = overlapAdd(timeframe); + const pcmFrame = floatToInt16(synthFrame); + + collectOutput(pcmFrame); + } + if (bitPtr.index >= bitstream.length) { console.log("SUCCESS! Payload fully modulated into spectra."); + saveWAV("output.wav"); + pvRecorder.stop(); } - - // if (maskingMap.length > 0) { - // const latestFrame = maskingMap[maskingMap.length - 1]!; - // const safebins = latestFrame.safeBins; - - // if (safebins.length > 0 && bitPtr < bitstream.length) { - // for (const bitIndex of safebins) { - // if (bitPtr >= bitstream.length) break; - - // const currentBit = bitstream[bitPtr]; - - // bitPtr++; - // } - // } - - // console.log( - // `Frame: ${latestFrame.frameIndex} | Safe Bins: ${safebins.length} | Progress: ${bitPtr}/${bitstream.length} bits`, - // ); - // if (bitPtr >= bitstream.length) { - // console.log("SUCCESS! entire bitstream injected"); - // // break; - // } - // } } } } catch (error: unknown) { diff --git a/src/core/recombination/ifft.ts b/src/core/recombination/ifft.ts new file mode 100644 index 0000000..7ebf9d5 --- /dev/null +++ b/src/core/recombination/ifft.ts @@ -0,0 +1,27 @@ +import FFT from "fft.js"; + +const FRAME_SIZE = 1024; +const f = new FFT(FRAME_SIZE); + +const outputGain = 0.5; + +/** + * Converts the frequency domain into the time domain PCM samples + * @param complexSpectrum interleaved Float32Array + * @returns real-valued Float32Array (PCM) + */ + +export function processIFFT(complexSpectrum: Float32Array): Float32Array { + const outTimeDomain = f.createComplexArray(); + // const realOutput = new Float32Array(outTimeDomain); + const realOutput = new Float32Array(FRAME_SIZE); + + f.inverseTransform(outTimeDomain, complexSpectrum); + + for (let i = 0; i < FRAME_SIZE; i++) { + const amplitude = outTimeDomain[i * 2] * outputGain; + realOutput[i] = amplitude; + } + + return realOutput; +} diff --git a/src/core/recombination/pcmConverter.ts b/src/core/recombination/pcmConverter.ts new file mode 100644 index 0000000..be36ee1 --- /dev/null +++ b/src/core/recombination/pcmConverter.ts @@ -0,0 +1,19 @@ +/** + * Converts Float32 audio samples to Int16 PCM + * Clamps values to prevent wrap-around distortion. + */ + +export function floatToInt16(floatFrame: Float32Array): Int16Array { + const intFrame = new Int16Array(floatFrame.length); + + for (let i = 0; i < floatFrame.length; i++) { + let s = floatFrame[i]! * 32767.0; + + if (s > 32767.0) s = 32767.0; + if (s < -32768.0) s = -32768.0; + + intFrame[i] = Math.round(s); + } + + return intFrame; +} diff --git a/src/core/recombination/reconstructor.ts b/src/core/recombination/reconstructor.ts new file mode 100644 index 0000000..4d4cc61 --- /dev/null +++ b/src/core/recombination/reconstructor.ts @@ -0,0 +1,21 @@ +const FRAME_SIZE = 1024; +const HOP_SIZE = 512; + +let overlapBuffer = new Float32Array(HOP_SIZE).fill(0); + +/** + * Stitches IFFT frames using Overlap-Add logic. + * @param timeFrame 1024 samples + * @returns 512 samples of finished PCM + */ + +export function overlapAdd(timeFrame: Float32Array): Float32Array { + const output = new Float32Array(HOP_SIZE); + + for (let i = 0; i < HOP_SIZE; i++) { + output[i] = (overlapBuffer[i]! + timeFrame[i]!) / 1.08; + overlapBuffer[i] = timeFrame[i + HOP_SIZE]!; + } + + return output; +} diff --git a/src/core/recombination/writer.ts b/src/core/recombination/writer.ts new file mode 100644 index 0000000..48cf6e7 --- /dev/null +++ b/src/core/recombination/writer.ts @@ -0,0 +1,21 @@ +import wavefile from "wavefile"; +import fs from "node:fs"; + +let pcmAccumulator: Int16Array[] = []; + +export function collectOutput(pcmFrame: Int16Array): void { + // const newBuffer = new Int16Array(pcmAccumulator.length + pcmFrame.length); + // newBuffer.set(pcmAccumulator); + // newBuffer.set(pcmFrame, pcmAccumulator.length); + // pcmAccumulator = newBuffer; + + pcmAccumulator.push(pcmFrame.slice()); +} + +export function saveWAV(filename: string = "output.wav"): void { + const wav = new wavefile.WaveFile(); + wav.fromScratch(1, 16000, "16", pcmAccumulator); + fs.writeFileSync(filename, wav.toBuffer()); + console.log(`\n stealth audio saved: ${filename}`); + // pcmAccumulator = new Int16Array(0); +}