Audio Reconstruction Pipeline

.gitignore

@@ -3,3 +3,4 @@ dist/
 build/
 .env
 data/
+*.wav

package.json

@@ -29,6 +29,7 @@
     "@picovoice/pvrecorder-node": "^1.2.8",
     "@subspace/reed-solomon-erasure.wasm": "^0.2.5",
     "chalk": "^5.6.2",
-    "fft.js": "^4.0.4"
+    "fft.js": "^4.0.4",
+    "wavefile": "^11.0.0"
   }
 }

src/core/modulator/embedChips.ts

@@ -4,7 +4,7 @@
  */
 
 export function embedFrameChips(
-  fftComplex: Float32Array,
+  fftComplex: Float32Array | number[],
   chipMap: Map<number, number>,
   N: number,
   delta: number = 0.02,
@@ -15,6 +15,10 @@ export function embedFrameChips(
     const rIndex = binIndex * 2;
     const iIndex = rIndex + 1;
 
+    if (iIndex >= fftComplex.length) {
+      continue;
+    }
+
     const real = fftComplex[rIndex]!;
     const imag = fftComplex[iIndex]!;
     const magnitude = Math.sqrt(real * real + imag * imag);

src/core/profiler/fft.ts

@@ -1,18 +1,27 @@
 import FFT from "fft.js";
 const FRAME_SIZE = 1024;
 const f = new FFT(FRAME_SIZE);
-const out = f.createComplexArray();
 
+/**
+ * Processes a windowed frame of audio samples using the Fast Fourier Transform (FFT)
+ * @param {Float32Array} windowedFrame - a windowed frame of audio samples
+ * @returns {Object} - an object containing the FFT complex array and power spectrum array
+ * @property {number[]} fftComplex - the FFT complex array
+ * @property {Float32Array} powerSpectrum - the power spectrum array
+ */
 export function processFFT(windowedFrame: Float32Array) {
-  f.realTransform(out, windowedFrame);
+  const fftComplex = f.createComplexArray();
+  f.realTransform(fftComplex, windowedFrame);
+  f.completeSpectrum(fftComplex);
+
   const powerSpectrum = new Float32Array(FRAME_SIZE / 2);
 
   for (let i = 0; i < FRAME_SIZE / 2; i++) {
-    const real = out[2 * i]!;
-    const imag = out[2 * i + 1]!;
+    const real = fftComplex[2 * i]!;
+    const imag = fftComplex[2 * i + 1]!;
 
     powerSpectrum[i] = real * real + imag * imag;
   }
 
-  return powerSpectrum;
+  return { fftComplex, powerSpectrum };
 }

src/core/profiler/processFrame.ts

@@ -10,57 +10,61 @@ import { HAMMING_WINDOW } from "./window.js";
 
 const HOP_SIZE = 512;
 const FRAME_SIZE = 1024;
-
-let frameCount = 0;
+interface FrameState {
+  count: number;
+}
 
 export function processSTFT(
   audioBuffer: AudioRingBuffer,
   bitstream: Uint8Array,
   bitPtr: { index: number },
   pnGen: PNGenerator,
+  frameState: FrameState = { count: 0 },
 ) {
   const outSpectra = [];
 
   while (audioBuffer.size >= FRAME_SIZE) {
-    const frames = audioBuffer.getFrames(FRAME_SIZE);
+    const rawFrames = audioBuffer.getFrames(FRAME_SIZE);
+    const frames = new Float32Array(rawFrames);
 
     for (let i = 0; i < FRAME_SIZE; i++)
       frames[i] = frames[i]! * HAMMING_WINDOW[i]!;
 
     console.log("Frame min/max:", Math.min(...frames), Math.max(...frames));
 
-    const fftComplex = processFFT(frames);
-    const bandEnergy = computeBarkEnergy(fftComplex);
+    const { fftComplex, powerSpectrum } = processFFT(frames);
+    const bandEnergy = computeBarkEnergy(powerSpectrum);
     const maskingThresholds = estimateMasking(bandEnergy);
-    const safeBins = identifySafeBins(fftComplex, maskingThresholds);
+    const safeBins = identifySafeBins(powerSpectrum, maskingThresholds);
 
     console.log(`Found ${safeBins.length} safe bins for data injection`);
 
-    let finalSpectrum = fftComplex;
+    let finalSpectrum: Float32Array = new Float32Array(fftComplex);
 
     if (safeBins.length >= 64 && bitPtr.index < bitstream.length) {
       const bit = bitstream[bitPtr.index]!;
       const symbol = (bit << 1) - 1;
       const pnSequence = pnGen.generateSequence(64);
       const spreadChips = DSSS_Spreader.spread(symbol, pnSequence);
 
-      const chipMap = BinMapper.mapToBins(spreadChips, safeBins, frameCount);
+      const chipMap = BinMapper.mapToBins(
+        spreadChips,
+        safeBins,
+        frameState.count,
+      );
       finalSpectrum = embedFrameChips(fftComplex, chipMap, FRAME_SIZE, 0.02);
       bitPtr.index++;
     }
 
-    const map = {
-      frameIndex: frameCount++,
-      safeBins,
-      bandEnergy: new Float32Array(bandEnergy),
-      maskingThresholds: new Float32Array(maskingThresholds),
-    };
+    const currentFrameIndex = frameState.count++;
 
     outSpectra.push({
       spectrum: finalSpectrum,
-      frameIndex: frameCount++,
+      frameIndex: currentFrameIndex,
+      safeBins,
+      bandEnergy: new Float32Array(bandEnergy),
+      maskingThresholds: new Float32Array(maskingThresholds),
     });
-
     audioBuffer.advance(HOP_SIZE);
   }
 

src/core/profiler/recorder.ts

@@ -2,6 +2,10 @@ import { PvRecorder } from "@picovoice/pvrecorder-node";
 import { buffer } from "../../types/AudioRingBuffer.js";
 import { processSTFT } from "./processFrame.js";
 import { PNGenerator } from "../modulator/pnGen.js";
+import { processIFFT } from "../recombination/ifft.js";
+import { overlapAdd } from "../recombination/reconstructor.js";
+import { floatToInt16 } from "../recombination/pcmConverter.js";
+import { collectOutput, saveWAV } from "../recombination/writer.js";
 
 export async function recorder(bitstream: Uint8Array) {
   const frameSize = 512;
@@ -19,37 +23,23 @@ export async function recorder(bitstream: Uint8Array) {
       if (buffer.size >= 1024) {
         const modifiedFrames = processSTFT(buffer, bitstream, bitPtr, pnGen);
 
-        for (const frame of modifiedFrames)
+        for (const frame of modifiedFrames) {
           console.log(
             `Frame: ${frame.frameIndex} | Progress: ${bitPtr.index}/${bitstream.length}`,
           );
 
+          const timeframe = processIFFT(frame.spectrum);
+          const synthFrame = overlapAdd(timeframe);
+          const pcmFrame = floatToInt16(synthFrame);
+
+          collectOutput(pcmFrame);
+        }
+
         if (bitPtr.index >= bitstream.length) {
           console.log("SUCCESS! Payload fully modulated into spectra.");
+          saveWAV("output.wav");
+          pvRecorder.stop();
         }
-
-        // if (maskingMap.length > 0) {
-        //   const latestFrame = maskingMap[maskingMap.length - 1]!;
-        //   const safebins = latestFrame.safeBins;
-
-        //   if (safebins.length > 0 && bitPtr < bitstream.length) {
-        //     for (const bitIndex of safebins) {
-        //       if (bitPtr >= bitstream.length) break;
-
-        //       const currentBit = bitstream[bitPtr];
-
-        //       bitPtr++;
-        //     }
-        //   }
-
-        //   console.log(
-        //     `Frame: ${latestFrame.frameIndex} | Safe Bins: ${safebins.length} | Progress: ${bitPtr}/${bitstream.length} bits`,
-        //   );
-        //   if (bitPtr >= bitstream.length) {
-        //     console.log("SUCCESS! entire bitstream injected");
-        //     // break;
-        //   }
-        // }
       }
     }
   } catch (error: unknown) {

src/core/recombination/ifft.ts

@@ -0,0 +1,27 @@
+import FFT from "fft.js";
+
+const FRAME_SIZE = 1024;
+const f = new FFT(FRAME_SIZE);
+
+const outputGain = 0.5;
+
+/**
+ * Converts the frequency domain into the time domain PCM samples
+ * @param complexSpectrum interleaved Float32Array
+ * @returns real-valued Float32Array (PCM)
+ */
+
+export function processIFFT(complexSpectrum: Float32Array): Float32Array {
+  const outTimeDomain = f.createComplexArray();
+  // const realOutput = new Float32Array(outTimeDomain);
+  const realOutput = new Float32Array(FRAME_SIZE);
+
+  f.inverseTransform(outTimeDomain, complexSpectrum);
+
+  for (let i = 0; i < FRAME_SIZE; i++) {
+    const amplitude = outTimeDomain[i * 2] * outputGain;
+    realOutput[i] = amplitude;
+  }
+
+  return realOutput;
+}

src/core/recombination/pcmConverter.ts

@@ -0,0 +1,19 @@
+/**
+ * Converts Float32 audio samples to Int16 PCM
+ * Clamps values to prevent wrap-around distortion.
+ */
+
+export function floatToInt16(floatFrame: Float32Array): Int16Array {
+  const intFrame = new Int16Array(floatFrame.length);
+
+  for (let i = 0; i < floatFrame.length; i++) {
+    let s = floatFrame[i]! * 32767.0;
+
+    if (s > 32767.0) s = 32767.0;
+    if (s < -32768.0) s = -32768.0;
+
+    intFrame[i] = Math.round(s);
+  }
+
+  return intFrame;
+}

src/core/recombination/reconstructor.ts

@@ -0,0 +1,21 @@
+const FRAME_SIZE = 1024;
+const HOP_SIZE = 512;
+
+let overlapBuffer = new Float32Array(HOP_SIZE).fill(0);
+
+/**
+ * Stitches IFFT frames using Overlap-Add logic.
+ * @param timeFrame 1024 samples
+ * @returns 512 samples of finished PCM
+ */
+
+export function overlapAdd(timeFrame: Float32Array): Float32Array {
+  const output = new Float32Array(HOP_SIZE);
+
+  for (let i = 0; i < HOP_SIZE; i++) {
+    output[i] = (overlapBuffer[i]! + timeFrame[i]!) / 1.08;
+    overlapBuffer[i] = timeFrame[i + HOP_SIZE]!;
+  }
+
+  return output;
+}

src/core/recombination/writer.ts

@@ -0,0 +1,21 @@
+import wavefile from "wavefile";
+import fs from "node:fs";
+
+let pcmAccumulator: Int16Array[] = [];
+
+export function collectOutput(pcmFrame: Int16Array): void {
+  // const newBuffer = new Int16Array(pcmAccumulator.length + pcmFrame.length);
+  // newBuffer.set(pcmAccumulator);
+  // newBuffer.set(pcmFrame, pcmAccumulator.length);
+  // pcmAccumulator = newBuffer;
+
+  pcmAccumulator.push(pcmFrame.slice());
+}
+
+export function saveWAV(filename: string = "output.wav"): void {
+  const wav = new wavefile.WaveFile();
+  wav.fromScratch(1, 16000, "16", pcmAccumulator);
+  fs.writeFileSync(filename, wav.toBuffer());
+  console.log(`\n stealth audio saved: ${filename}`);
+  // pcmAccumulator = new Int16Array(0);
+}