Log undecodable CAN frames to per-ID CSVs under unknown_ids and document E2E procedure

README.md

@@ -175,6 +175,13 @@ timestamp_ns,value,raw_hex
 The directory is created on startup if it does not already exist. Files are
 opened in append mode, so repeated runs accumulate rows in the same CSV.
 
+Frames whose CAN ID does not successfully decode to any configured signal are
+written to per-ID CSV files under `<outdir>/unknown_ids/` with columns:
+
+```
+timestamp_ns,dlc,data_hex
+```
+
 Send `SIGINT` (Ctrl-C) or `SIGTERM` to stop the reader and flush/close all
 open CSV files cleanly.
 
@@ -192,8 +199,26 @@ sudo ip link set up vcan0
 cansend vcan0 101#0000FA00640000000
 
 ls logs/   # expect pack_current.csv, pack_voltage.csv, soc.csv, soh.csv
+
+# Send a frame on an ID not present in format.json.
+cansend vcan0 777#1122334455667788
+
+ls logs/unknown_ids/          # expect 00000777.csv
+tail -n +1 logs/unknown_ids/00000777.csv
 ```
 
+## End-to-end test procedure
+
+1. Set up vcan and start `can_telem` as shown in the smoke test above.
+2. Send one known frame (e.g., CAN ID `0x101`) and verify known signal CSVs are
+   updated in `<outdir>/`.
+3. Send one unknown frame (e.g., CAN ID `0x777`) and verify
+   `<outdir>/unknown_ids/00000777.csv` is created with header
+   `timestamp_ns,dlc,data_hex` and a new row.
+4. Send another frame on `0x777` and verify it appends a second row to the same
+   per-ID file.
+5. Stop `can_telem` with Ctrl-C and confirm files remain readable and complete.
+
 ## Notes and caveats
 
 - Decoding assumes little-endian (Intel) byte order, which matches the

src/can_reader.c

@@ -93,6 +93,7 @@ int can_reader_loop(int fd,
         if (!(frame.can_id & CAN_EFF_FLAG)) id &= CAN_SFF_MASK;
 
         const sig_node_t *node = signal_table_lookup(table, id);
+        bool decoded_any = false;
         for (; node; node = node->next) {
             if (node->sig.can_id != id) continue;
             if (node->sig.placeholder) continue; /* "FFF" -> unassigned */
@@ -102,9 +103,13 @@ int can_reader_loop(int fd,
                                 frame.can_dlc, &dv) != 0) {
                 continue; /* overflow or misaligned */
             }
+            decoded_any = true;
             writer_append(w, &node->sig, &dv);
             if (influx) influx_accumulate(influx, &node->sig, &dv);
         }
+        if (!decoded_any) {
+            writer_append_unknown(w, id, frame.data, frame.can_dlc);
+        }
         if (influx) influx_tick(influx);
     }
 

src/can_reader.h

@@ -18,7 +18,8 @@ int can_reader_open(const char *ifname);
 /*
  * Blocking receive loop: reads frames from `fd`, matches each ID against
  * `table`, decodes every matching signal (skipping placeholders with
- * can_id==0xFFF) and appends the value via the writer.
+ * can_id==0xFFF) and appends the value via the writer. Frames that do not
+ * decode any configured signal are logged to per-ID CSVs under unknown_ids/.
  * If `influx` is non-NULL and enabled, updates Influx aggregators and may
  * flush to the cloud on a timer (see config).
  * Runs until `*running` becomes 0.

src/writer.c

@@ -12,6 +12,13 @@
 #include <unistd.h>
 
 static const char CSV_HEADER[] = "timestamp_ns,value,raw_hex\n";
+static const char UNKNOWN_CSV_HEADER[] = "timestamp_ns,dlc,data_hex\n";
+static const char UNKNOWN_DIR_NAME[] = "unknown_ids";
+static const uint8_t MAX_CAN_DLC = 8;
+enum {
+    UNKNOWN_PATH_SUFFIX_MAX = 16, /* '/' + 8 hex chars + ".csv" + '\0' */
+    UNKNOWN_HEX_BUFFER_SIZE = (8 * 2) + 1
+};
 
 /* djb2 string hash */
 static size_t hash_name(const char *s) {
@@ -47,6 +54,17 @@ int writer_init(writer_t *w, const char *out_dir) {
         fprintf(stderr, "writer: mkdir %s: %s\n", w->out_dir, strerror(errno));
         return -1;
     }
+    int dlen = snprintf(w->unknown_dir, sizeof w->unknown_dir,
+                        "%s/%s", w->out_dir, UNKNOWN_DIR_NAME);
+    if (dlen < 0 || (size_t)dlen >= sizeof w->unknown_dir) {
+        fprintf(stderr, "writer: unknown output path too long\n");
+        return -1;
+    }
+    if (mkdir_p(w->unknown_dir) != 0) {
+        fprintf(stderr, "writer: mkdir %s: %s\n",
+                w->unknown_dir, strerror(errno));
+        return -1;
+    }
     return 0;
 }
 
@@ -108,6 +126,62 @@ int writer_append(writer_t *w,
     return 0;
 }
 
+int writer_append_unknown(writer_t *w,
+                          uint32_t can_id,
+                          const uint8_t *payload,
+                          uint8_t dlc) {
+    if (!w || !payload) return -1;
+
+    if (dlc > MAX_CAN_DLC) dlc = MAX_CAN_DLC;
+
+    char path[sizeof(w->unknown_dir) + UNKNOWN_PATH_SUFFIX_MAX];
+    int len = snprintf(path, sizeof path, "%s/%08X.csv", w->unknown_dir, can_id);
+    if (len < 0 || (size_t)len >= sizeof path) {
+        fprintf(stderr, "writer: unknown path too long for %08X\n", can_id);
+        return -1;
+    }
+
+    struct stat st;
+    errno = 0;
+    int st_rc = stat(path, &st);
+    int newly_created = (st_rc != 0 && errno == ENOENT);
+    if (st_rc != 0 && errno != ENOENT) {
+        fprintf(stderr, "writer: stat %s: %s\n", path, strerror(errno));
+        return -1;
+    }
+    FILE *f = fopen(path, "a");
+    if (!f) {
+        fprintf(stderr, "writer: fopen %s: %s\n", path, strerror(errno));
+        return -1;
+    }
+    if (newly_created) {
+        fwrite(UNKNOWN_CSV_HEADER, 1, sizeof UNKNOWN_CSV_HEADER - 1, f);
+    }
+
+    static const char HEX[] = "0123456789ABCDEF";
+    char hex[UNKNOWN_HEX_BUFFER_SIZE];
+    for (uint8_t i = 0; i < dlc; ++i) {
+        hex[(size_t)(i * 2)] = HEX[(payload[i] >> 4) & 0x0F];
+        hex[(size_t)(i * 2) + 1] = HEX[payload[i] & 0x0F];
+    }
+    hex[(size_t)dlc * 2] = '\0';
+
+    struct timespec ts;
+    clock_gettime(CLOCK_REALTIME, &ts);
+    long long ns = (long long)ts.tv_sec * 1000000000LL + (long long)ts.tv_nsec;
+
+    int rc = 0;
+    if (fprintf(f, "%lld,%u,%s\n", ns, (unsigned)dlc, hex) < 0) {
+        fprintf(stderr, "writer: fprintf unknown %08X: %s\n",
+                can_id, strerror(errno));
+        rc = -1;
+    } else {
+        fflush(f);
+    }
+    fclose(f);
+    return rc;
+}
+
 void writer_close(writer_t *w) {
     if (!w) return;
     for (size_t i = 0; i < WRITER_CACHE_SIZE; ++i) {

src/writer.h

@@ -17,6 +17,7 @@ typedef struct writer_entry {
 
 typedef struct {
     char            out_dir[512];
+    char            unknown_dir[512];
     writer_entry_t  entries[WRITER_CACHE_SIZE];
     size_t          open_count;
 } writer_t;
@@ -35,6 +36,15 @@ int writer_append(writer_t *w,
                   const signal_def_t *sig,
                   const decoded_value_t *dv);
 
+/*
+ * Append one raw CAN frame for an unknown/undecodable CAN ID into
+ * <out_dir>/unknown_ids/<can_id>.csv.
+ */
+int writer_append_unknown(writer_t *w,
+                          uint32_t can_id,
+                          const uint8_t *payload,
+                          uint8_t dlc);
+
 /*
  * Flush and close all open CSV files.
  */