Fixing merge conflicts.

Author: DrLynch

autodocs/.gitignore

@@ -1,3 +1,4 @@
 _build/
 generated/
 apidocs/
+module_readmes/

autodocs/api.rst

@@ -2,5 +2,6 @@ API
 ===
 
 .. toctree::
+   :maxdepth: 4
 
    apidocs/index

autodocs/concepts.rst

@@ -0,0 +1,49 @@
+Concepts
+=============
+
+Explanations of key ideas, principles, and background knowledge.
+Follow this recommended sequence to build context before diving into
+implementation details:
+
+- :doc:`History <docs/concepts/history>` - establishes the background and
+  problem space the project is addressing.
+- :doc:`System Design <docs/concepts/system_design>` - explains how the product
+  strategy and user needs translate into an overall system approach.
+- :doc:`Architecture <docs/concepts/architecture>` - outlines the concrete
+  architecture that implements the system design.
+- :doc:`Technologies <docs/concepts/technologies>` - surveys the primary tools
+  and platforms we rely on to realize the architecture.
+- :doc:`System Settings <docs/concepts/system_settings>` - describes how the
+  system loads global and cascading settings.
+- :doc:`Events <docs/concepts/events>` - introduces the event model that drives
+  data flowing through the system.
+- :doc:`Reducers <docs/concepts/reducers>` - details how incoming events are
+  aggregated into the state our experiences depend on.
+- :doc:`Communication Protocol <docs/concepts/communication_protocol>` - discusses how
+  the system queries data from reducers for dashboards.
+- :doc:`Student Identity Mapping <docs/concepts/student_identity_mapping>` - explain
+  how learners information is mapped across integrations.
+- :doc:`Scaling <docs/concepts/scaling>` - covers strategies for growing the
+  system once the fundamentals are in place.
+- :doc:`Auth <docs/concepts/auth>` - describes authentication considerations
+  that secure access to the system.
+- :doc:`Privacy <docs/concepts/privacy>` - documents how we protect learner data
+  and comply with privacy expectations.
+
+.. toctree::
+   :hidden:
+   :maxdepth: 1
+   :titlesonly:
+
+   docs/concepts/history
+   docs/concepts/system_design
+   docs/concepts/architecture
+   docs/concepts/technologies
+   docs/concepts/system_settings
+   docs/concepts/events
+   docs/concepts/reducers
+   docs/concepts/communication_protocol
+   docs/concepts/student_identity_mapping
+   docs/concepts/scaling
+   docs/concepts/auth
+   docs/concepts/privacy

autodocs/conf.py

@@ -1,5 +1,10 @@
-import sys
 import os
+import pathlib
+import re
+import shutil
+import sphinx.util
+import sys
+import unicodedata
 # Configuration file for the Sphinx documentation builder.
 #
 # For the full list of built-in configuration values, see the documentation:
@@ -9,7 +14,7 @@
 # https://www.sphinx-doc.org/en/master/usage/configuration.html#project-information
 
 project = 'Learning Observer'
-copyright = '2023, Bradley Erickson'
+copyright = '2020-2025, Bradley Erickson'
 author = 'Bradley Erickson'
 
 # -- General configuration ---------------------------------------------------
@@ -26,6 +31,9 @@
     '../modules/writing_observer/writing_observer'
 ]
 
+autodoc2_output_dir = 'apidocs'
+autodoc2_member_order = 'bysource'
+
 source_suffix = {
     '.rst': 'restructuredtext',
     '.md': 'markdown',
@@ -40,3 +48,143 @@
 
 html_theme = 'alabaster'
 html_static_path = ['_static']
+
+LOGGER = sphinx.util.logging.getLogger(__name__)
+
+
+_MARKDOWN_IMAGE_PATTERN = re.compile(r'!\[[^\]]*\]\(([^)]+)\)')
+_RST_IMAGE_PATTERNS = [
+    re.compile(r'\.\.\s+image::\s+([^\s]+)'),
+    re.compile(r'\.\.\s+figure::\s+([^\s]+)'),
+]
+
+
+def _extract_local_assets(text):
+    """Return relative asset paths referenced in the provided README text."""
+
+    asset_paths = set()
+    for match in _MARKDOWN_IMAGE_PATTERN.findall(text):
+        asset_paths.add(match)
+    for pattern in _RST_IMAGE_PATTERNS:
+        asset_paths.update(pattern.findall(text))
+
+    filtered_assets = set()
+    for raw_path in asset_paths:
+        candidate = raw_path.strip()
+        if not candidate:
+            continue
+        # Remove optional titles ("path "optional title"") and URL fragments
+        candidate = candidate.split()[0]
+        candidate = candidate.split('#', maxsplit=1)[0]
+        candidate = candidate.split('?', maxsplit=1)[0]
+
+        if candidate.startswith(('http://', 'https://', 'data:')):
+            continue
+        if candidate.startswith('#'):
+            continue
+
+        filtered_assets.add(candidate)
+
+    return sorted(filtered_assets)
+
+
+def _copy_module_assets(readme_path, destination_dir):
+    """Copy image assets referenced by ``readme_path`` into ``destination_dir``."""
+
+    module_dir = readme_path.parent.resolve()
+    readme_text = readme_path.read_text(encoding='utf-8')
+    asset_paths = _extract_local_assets(readme_text)
+    for asset in asset_paths:
+        relative_posix_path = pathlib.PurePosixPath(asset)
+        if relative_posix_path.is_absolute():
+            LOGGER.warning(
+                "Skipping absolute image path %s referenced in %s", asset, readme_path
+            )
+            continue
+
+        normalized_relative_path = pathlib.Path(*relative_posix_path.parts)
+        source_path = (module_dir / normalized_relative_path).resolve(strict=False)
+
+        try:
+            source_path.relative_to(module_dir)
+        except ValueError:
+            LOGGER.warning(
+                "Skipping image outside module directory: %s referenced in %s",
+                asset,
+                readme_path,
+            )
+            continue
+
+        if not source_path.exists():
+            LOGGER.warning(
+                "Referenced image %s in %s was not found", asset, readme_path
+            )
+            continue
+
+        destination_path = destination_dir / normalized_relative_path
+        destination_path.parent.mkdir(parents=True, exist_ok=True)
+        shutil.copy2(source_path, destination_path)
+
+
+def _extract_readme_title(readme_path: pathlib.Path) -> str:
+    """Return the first Markdown heading in ``readme_path``.
+
+    Defaults to the parent directory name if no heading can be found.
+    """
+
+    try:
+        for line in readme_path.read_text(encoding='utf-8').splitlines():
+            stripped = line.strip()
+            if stripped.startswith('#'):
+                title = stripped.lstrip('#').strip()
+                if title:
+                    return title
+    except OSError as exc:  # pragma: no cover - filesystem error propagation
+        LOGGER.warning("unable to read %s: %s", readme_path, exc)
+
+    return readme_path.parent.name
+
+
+def _slugify(text: str) -> str:
+    """Convert ``text`` to a lowercase filename-safe slug."""
+
+    normalized = unicodedata.normalize('NFKD', text)
+    without_diacritics = ''.join(ch for ch in normalized if not unicodedata.combining(ch))
+    slug = re.sub(r'[^a-z0-9]+', '-', without_diacritics.casefold()).strip('-')
+    return slug or 'module'
+
+
+def _copy_module_readmes(app):
+    """Populate ``module_readmes`` with module README files and assets."""
+
+    docs_root = pathlib.Path(__file__).parent.resolve()
+    modules_root = docs_root.parent / 'modules'
+    destination_root = docs_root / 'module_readmes'
+
+    if not modules_root.exists():
+        LOGGER.warning("modules directory %s was not found", modules_root)
+        return
+
+    if destination_root.exists():
+        shutil.rmtree(destination_root)
+    destination_root.mkdir(parents=True, exist_ok=True)
+
+    readme_info = []
+    for readme_path in modules_root.glob('*/README.md'):
+        title = _extract_readme_title(readme_path)
+        readme_info.append((title, readme_path))
+
+    readme_info.sort(key=lambda item: item[0].casefold())
+
+    for title, readme_path in readme_info:
+        module_name = readme_path.parent.name
+        slug = _slugify(title)
+        module_destination = destination_root / f'{slug}--{module_name}'
+        module_destination.mkdir(parents=True, exist_ok=True)
+        destination_path = module_destination / "README.md"
+        shutil.copy2(readme_path, destination_path)
+        _copy_module_assets(readme_path, module_destination)
+
+
+def setup(app):
+    app.connect('builder-inited', _copy_module_readmes)

autodocs/development.rst

@@ -0,0 +1 @@
+../docs/

autodocs/docs

@@ -0,0 +1,25 @@
+How-to
+=============
+
+Practical instructions for achieving specific goals within Learning Observer. Use these guides when you know what outcome you need and want a proven recipe to follow:
+
+- :doc:`Communication Protocol <docs/how-to/communication_protocol>` - How to query data from reducers or system endpoints for dashboards.
+- :doc:`Configure Learning Observer <docs/how-to/config>` - Set up credentials, environment variables, and other configuration details required for a smooth deployment.
+- :doc:`Build Dashboards <docs/how-to/dashboards>` - Walk through creating dashboards from reducer outputs, including layout choices and data wiring.
+- :doc:`LTI <docs/how-to/lti>` - Cover how to install Learning Observer as an LTI application.
+- :doc:`Run with Docker <docs/how-to/docker>` - Learn how to containerize the stack, manage images, and operate the project using Docker Compose.
+- :doc:`Writing Observer Extension <docs/how-to/extension>` - Install, configure, and validate the Writing Observer browser extension for capturing events.
+- :doc:`Interactive Environments <docs/how-to/interactive_environments>` - Connect Learning Observer to Jupyter and other live coding setups for iterative development.
+
+.. toctree::
+   :hidden:
+   :maxdepth: 1
+   :titlesonly:
+
+   docs/how-to/communication_protocol.md
+   docs/how-to/config.md
+   docs/how-to/dashboards.md
+   docs/how-to/lti.md
+   docs/how-to/docker.md
+   docs/how-to/extension.md
+   docs/how-to/interactive_environments.md

autodocs/extension.rst

@@ -12,14 +12,21 @@ per-student writing data, and aggegators to make dashboards. We've
 tested this in math and writing, but our focus is on writing process
 data.
 
+Our documentation is organized into four main categories, each serving a different purpose. You can explore them below:
+
+- :doc:`Tutorials <tutorials>` - Step-by-step guides to help you learn by doing.
+- :doc:`Concepts <concepts>` - Explanations of key ideas and background knowledge.
+- :doc:`How-To <how-to>` - Practical instructions to solve specific goals.
+- :doc:`Reference <reference>` - Detailed API/configuration information.
+
 .. toctree::
-   :maxdepth: 2
+   :hidden:
+   :maxdepth: 3
 
-   development
-   system_design
-   modules
-   extension
-   api
+   tutorials
+   concepts
+   how-to
+   reference
 
 Additional Information
 ----------------------