added copycut and paste redcuers

saminur · saminur · commit 745ba3bf1fe8 · 2026-03-02T06:28:35.000-05:00
diff --git a/modules/writing_observer/writing_observer/copy_paste_utils.py b/modules/writing_observer/writing_observer/copy_paste_utils.py
@@ -0,0 +1,283 @@
+"""
+Helpers for extracting copy, cut, and paste signals from writing analytics
+events.
+"""
+
+from __future__ import annotations
+
+import datetime as dt
+import re
+
+
+PASTE_WAIT_MS = 2500
+MENU_FLAG_MS = 1500
+DEDUP_MS = 750
+BIG_PASTE_THRESHOLD = 200
+
+DOC_URL_RE = re.compile(r"^https://docs.google.com/document/d/(?P<DOCID>[^/\s]+)/(?P<ACT>[a-zA-Z]+)")
+
+
+def unwrap_event(event):
+    if isinstance(event, dict) and isinstance(event.get("client"), dict):
+        return event["client"]
+    return event if isinstance(event, dict) else {}
+
+
+def get_doc_id(event):
+    client = unwrap_event(event)
+    doc_id = client.get("doc_id")
+    if doc_id:
+        return doc_id
+
+    url = client.get("object", {}).get("url")
+    if not url or not DOC_URL_RE.match(url):
+        return None
+
+    return client.get("object", {}).get("id")
+
+
+def event_action(client):
+    action = client.get("action") or client.get("event") or client.get("type") or client.get("event_type") or ""
+    if action:
+        return str(action).lower()
+    keystroke = client.get("keystroke", {}) if isinstance(client.get("keystroke"), dict) else {}
+    return str(keystroke.get("action") or keystroke.get("type") or "").lower()
+
+
+def keys_info(client):
+    keystroke = client.get("keystroke", {}) if isinstance(client.get("keystroke"), dict) else {}
+    key = keystroke.get("key") or client.get("key") or ""
+    code = keystroke.get("code") or client.get("code") or ""
+    event_type = keystroke.get("type") or client.get("type") or ""
+    ctrl = bool(keystroke.get("ctrl") or client.get("ctrl") or keystroke.get("ctrlKey") or client.get("ctrlKey"))
+    meta = bool(keystroke.get("metaKey") or client.get("metaKey"))
+    key_code = keystroke.get("keyCode") or client.get("keyCode") or keystroke.get("which") or client.get("which")
+    return {
+        "key": str(key).lower() if key else "",
+        "code": str(code),
+        "event_type": str(event_type).lower(),
+        "ctrl_or_meta": bool(ctrl or meta),
+        "key_code": int(key_code) if isinstance(key_code, (int, float)) else None,
+    }
+
+
+def is_copy(client):
+    action = event_action(client)
+    if action in {"copy", "clipboard_copy", "gdocs_copy", "menu_copy", "edit_copy"}:
+        return True
+    info = keys_info(client)
+    return info["event_type"] == "keydown" and info["ctrl_or_meta"] and (
+        info["key"] == "c" or info["code"] == "KeyC" or info["key_code"] == 67
+    )
+
+
+def is_cut(client):
+    action = event_action(client)
+    if action in {"cut", "clipboard_cut", "gdocs_cut", "menu_cut", "edit_cut"}:
+        return True
+    info = keys_info(client)
+    return info["event_type"] == "keydown" and info["ctrl_or_meta"] and (
+        info["key"] == "x" or info["code"] == "KeyX" or info["key_code"] == 88
+    )
+
+
+def is_paste_keyboard(client):
+    action = event_action(client)
+    if action in {"paste", "clipboard_paste", "gdocs_paste", "insert_from_clipboard"}:
+        return True
+    info = keys_info(client)
+    return info["event_type"] == "keydown" and info["ctrl_or_meta"] and (
+        info["key"] == "v" or info["code"] == "KeyV" or info["key_code"] == 86
+    )
+
+
+def looks_like_menu_paste(client):
+    action = event_action(client)
+    if action in {"menu_paste", "edit_paste", "contextmenu_paste"}:
+        return True
+    return action == "contextmenu"
+
+
+def timestamp_ms(event, client=None):
+    client = client or unwrap_event(event)
+    for source in (client, event):
+        for key in ("timestamp", "ts", "time", "t"):
+            value = source.get(key)
+            if isinstance(value, (int, float)):
+                return int(value * 1000) if value < 10**11 else int(value)
+
+    server = event.get("server", {}) if isinstance(event, dict) else {}
+    value = server.get("time") if isinstance(server, dict) else None
+    if isinstance(value, (int, float)):
+        return int(value * 1000)
+
+    return int(dt.datetime.utcnow().timestamp() * 1000)
+
+
+def collect_inserted_text(command, output):
+    if not isinstance(command, dict):
+        return
+
+    if command.get("ty") == "is":
+        string_value = command.get("s")
+        if isinstance(string_value, str) and string_value:
+            output.append(string_value)
+
+    if isinstance(command.get("nmc"), dict):
+        collect_inserted_text(command["nmc"], output)
+
+    for child in command.get("mts") or []:
+        collect_inserted_text(child, output)
+
+
+def extract_insert_from_gdocs_save(client):
+    parts = []
+    for bundle in client.get("bundles") or []:
+        for command in (bundle or {}).get("commands") or []:
+            collect_inserted_text(command, parts)
+    return "".join(parts)
+
+
+def paste_length_bin(length):
+    if length <= 0:
+        return "none"
+    if length <= 20:
+        return "short_1_20"
+    if length <= 200:
+        return "medium_21_200"
+    return "long_201_plus"
+
+
+def append_recent(items, entry, limit=10):
+    items = list(items or [])
+    items.append(entry)
+    if len(items) > limit:
+        items = items[-limit:]
+    return items
+
+
+def default_paste_state():
+    return {
+        "paste_count": 0,
+        "pastes_with_length": 0,
+        "total_paste_chars": 0,
+        "max_paste_len": 0,
+        "last_paste_len": 0,
+        "big_pastes": 0,
+        "length_bins": {
+            "short_1_20": 0,
+            "medium_21_200": 0,
+            "long_201_plus": 0,
+        },
+        "recent_pastes": [],
+        "awaiting_paste_until": 0,
+        "maybe_menu_paste_until": 0,
+        "last_paste_signal_ms": 0,
+    }
+
+
+def default_copy_cut_state():
+    return {
+        "copy_count": 0,
+        "cut_count": 0,
+        "last_copy_ts": 0,
+        "last_cut_ts": 0,
+        "recent_events": [],
+    }
+
+
+def update_paste_state(event, state):
+    state = dict(default_paste_state() if state is None else state)
+    client = event.get("client", {}) or {}
+    ts_ms = timestamp_ms(event, client)
+
+    if is_paste_keyboard(client):
+        if ts_ms - state.get("last_paste_signal_ms", 0) <= DEDUP_MS:
+            return False
+        state["paste_count"] = state.get("paste_count", 0) + 1
+        state["last_paste_signal_ms"] = ts_ms
+        state["awaiting_paste_until"] = ts_ms + PASTE_WAIT_MS
+        state["recent_pastes"] = append_recent(
+            state.get("recent_pastes"),
+            {"timestamp_ms": ts_ms, "length": None, "source": "keyboard_signal"},
+        )
+        return state
+
+    if looks_like_menu_paste(client):
+        state["maybe_menu_paste_until"] = ts_ms + MENU_FLAG_MS
+        return state
+
+    if event_action(client) != "google_docs_save":
+        return False
+
+    inserted_text = extract_insert_from_gdocs_save(client)
+    if not inserted_text:
+        return False
+
+    paste_length = len(inserted_text)
+    awaiting_paste_until = state.get("awaiting_paste_until", 0)
+    maybe_menu_paste_until = state.get("maybe_menu_paste_until", 0)
+    counted_from_save = False
+
+    if ts_ms <= maybe_menu_paste_until and ts_ms > awaiting_paste_until:
+        if ts_ms - state.get("last_paste_signal_ms", 0) <= DEDUP_MS:
+            return False
+        state["paste_count"] = state.get("paste_count", 0) + 1
+        state["last_paste_signal_ms"] = ts_ms
+        counted_from_save = True
+
+    if ts_ms > awaiting_paste_until and not counted_from_save:
+        return False
+
+    state["pastes_with_length"] = state.get("pastes_with_length", 0) + 1
+    state["total_paste_chars"] = state.get("total_paste_chars", 0) + paste_length
+    state["max_paste_len"] = max(state.get("max_paste_len", 0), paste_length)
+    state["last_paste_len"] = paste_length
+    if paste_length >= BIG_PASTE_THRESHOLD:
+        state["big_pastes"] = state.get("big_pastes", 0) + 1
+
+    bin_name = paste_length_bin(paste_length)
+    if bin_name != "none":
+        state.setdefault("length_bins", {})
+        state["length_bins"][bin_name] = state["length_bins"].get(bin_name, 0) + 1
+
+    state["recent_pastes"] = append_recent(
+        state.get("recent_pastes"),
+        {
+            "timestamp_ms": ts_ms,
+            "length": paste_length,
+            "source": "menu_inferred" if counted_from_save else "google_docs_save",
+        },
+    )
+    state["awaiting_paste_until"] = 0
+    state["maybe_menu_paste_until"] = 0
+    return state
+
+
+def update_copy_cut_state(event, state):
+    state = dict(default_copy_cut_state() if state is None else state)
+    client = event.get("client", {}) or {}
+    ts_ms = timestamp_ms(event, client)
+    event_type = None
+
+    if is_copy(client):
+        if ts_ms - state.get("last_copy_ts", 0) <= DEDUP_MS:
+            return False
+        state["copy_count"] = state.get("copy_count", 0) + 1
+        state["last_copy_ts"] = ts_ms
+        event_type = "copy"
+    elif is_cut(client):
+        if ts_ms - state.get("last_cut_ts", 0) <= DEDUP_MS:
+            return False
+        state["cut_count"] = state.get("cut_count", 0) + 1
+        state["last_cut_ts"] = ts_ms
+        event_type = "cut"
+
+    if not event_type:
+        return False
+
+    state["recent_events"] = append_recent(
+        state.get("recent_events"),
+        {"timestamp_ms": ts_ms, "event_type": event_type},
+    )
+    return state
diff --git a/modules/writing_observer/writing_observer/module.py b/modules/writing_observer/writing_observer/module.py
@@ -97,6 +97,8 @@
         'update_docs': update_via_google(runtime=q.parameter("runtime"), doc_ids=q.variable('doc_sources')),
         "docs": q.select(q.keys('writing_observer.reconstruct', STUDENTS=q.variable("roster"), STUDENTS_path='user_id', RESOURCES=q.variable("update_docs"), RESOURCES_path='doc_id'), fields={'text': 'text'}),
         "docs_combined": q.join(LEFT=q.variable("docs"), RIGHT=q.variable("roster"), LEFT_ON='provenance.provenance.STUDENT.value.user_id', RIGHT_ON='user_id'),
+        "paste_metrics": q.select(q.keys('writing_observer.lo_paste_reducer', STUDENTS=q.variable("roster"), STUDENTS_path='user_id', RESOURCES=q.variable("doc_sources"), RESOURCES_path='doc_id'), fields='All'),
+        "copy_cut_metrics": q.select(q.keys('writing_observer.lo_copy_cut_reducer', STUDENTS=q.variable("roster"), STUDENTS_path='user_id', RESOURCES=q.variable("doc_sources"), RESOURCES_path='doc_id'), fields='All'),
         'nlp': process_texts(writing_data=q.variable('docs'), options=q.parameter('nlp_options', required=False, default=[])),
         'nlp_sep_proc': q.select(q.keys('writing_observer.nlp_components', STUDENTS=q.variable('roster'), STUDENTS_path='user_id', RESOURCES=q.variable("doc_ids"), RESOURCES_path='doc_id'), fields='All'),
         'nlp_combined': q.join(LEFT=q.variable(nlp_source), LEFT_ON='provenance.provenance.STUDENT.value.user_id', RIGHT=q.variable('roster'), RIGHT_ON='user_id'),
@@ -162,6 +164,16 @@
             "parameters": ["course_id"],
             "output": ""
         },
+        "paste_metrics": {
+            "returns": "paste_metrics",
+            "parameters": ["course_id"],
+            "output": ""
+        },
+        "copy_cut_metrics": {
+            "returns": "copy_cut_metrics",
+            "parameters": ["course_id"],
+            "output": ""
+        },
         "roster": {
             "returns": "roster",
             "parameters": ["course_id"],
@@ -258,6 +270,36 @@
 
 # Incoming event APIs
 REDUCERS = [
+    {
+        'context': "org.mitros.writing_analytics",
+        'scope': writing_observer.writing_analysis.gdoc_scope,
+        'function': writing_observer.writing_analysis.lo_paste_reducer,
+        'default': {
+            'paste_count': 0,
+            'pastes_with_length': 0,
+            'total_paste_chars': 0,
+            'max_paste_len': 0,
+            'last_paste_len': 0,
+            'big_pastes': 0,
+            'length_bins': {'short_1_20': 0, 'medium_21_200': 0, 'long_201_plus': 0},
+            'recent_pastes': [],
+            'awaiting_paste_until': 0,
+            'maybe_menu_paste_until': 0,
+            'last_paste_signal_ms': 0
+        }
+    },
+    {
+        'context': "org.mitros.writing_analytics",
+        'scope': writing_observer.writing_analysis.gdoc_scope,
+        'function': writing_observer.writing_analysis.lo_copy_cut_reducer,
+        'default': {
+            'copy_count': 0,
+            'cut_count': 0,
+            'last_copy_ts': 0,
+            'last_cut_ts': 0,
+            'recent_events': []
+        }
+    },
     {
         'context': "org.mitros.writing_analytics",
         'scope': writing_observer.writing_analysis.gdoc_scope,
@@ -366,4 +408,4 @@
     'name': 'NLP Options',
     'suburl': 'nlp-options',
     'static_json': INDICATOR_JSONS
-}]
+}]
diff --git a/modules/writing_observer/writing_observer/writing_analysis.py b/modules/writing_observer/writing_observer/writing_analysis.py
