Metrics and logs - MongoDB Submission (#43)

Adds MongoDB-backed submission for survey statistics/metrics and introduces MongoDB ObjectID-based participant/task identifiers to support upserts and linking metrics to a participant.

It introduces breaking changes to the exsisting json stat and metric files. Delete the old ones

Changes:

Add internal/storage/MongoStorage and wire pm survey submit / pm start to submit local .pm/*.json files to MongoDB.
Extend stats/metrics models to include MongoDB _id / participant_id fields and add participant ID propagation into task metrics.
Update app initialization to generate ObjectIDs for new participants and expose GetParticipantID() via StatsService.

notebooks/task-data-analysis.ipynb

@@ -1,418 +0,0 @@
-{
- "cells": [
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "# Task Data Analysis Notebook\n",
-    "\n",
-    "This notebook loads task run stats from `./.pm` and gives a quick view of completion rates, validation behavior, timings, and common failure patterns."
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "8626d9d2",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "from pathlib import Path\n",
-    "import json\n",
-    "\n",
-    "import pandas as pd\n",
-    "import matplotlib.pyplot as plt\n",
-    "\n",
-    "pd.set_option(\"display.max_columns\", 200)\n",
-    "plt.style.use(\"ggplot\")"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "7bc3e98d",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "candidate_dirs = [\n",
-    "    Path.cwd(),\n",
-    "    Path.cwd() / \"./.pm\",\n",
-    "    Path.cwd().parent / \"./.pm\",\n",
-    "]\n",
-    "\n",
-    "stats_dir = None\n",
-    "stat_files = []\n",
-    "for d in candidate_dirs:\n",
-    "    if d.is_dir():\n",
-    "        files = sorted(d.glob(\"*-stats.json\"))\n",
-    "        if files:\n",
-    "            stats_dir = d\n",
-    "            stat_files = files\n",
-    "            break\n",
-    "\n",
-    "print(f\"Working directory: {Path.cwd()}\")\n",
-    "print(f\"Using stats directory: {stats_dir}\")\n",
-    "print(f\"Found {len(stat_files)} stats file(s).\")\n",
-    "stat_files"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "c453e1ad",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "datasets = []\n",
-    "for file in stat_files:\n",
-    "    with file.open() as f:\n",
-    "        data = json.load(f)\n",
-    "    data[\"_file\"] = file.name\n",
-    "    datasets.append(data)\n",
-    "\n",
-    "print(f\"Loaded {len(datasets)} task dataset(s).\")"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "cbb0dd6b",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "summary_rows = []\n",
-    "run_rows = []\n",
-    "log_rows = []\n",
-    "\n",
-    "for ds in datasets:\n",
-    "    task_name = ds.get(\"task_name\")\n",
-    "    file_name = ds.get(\"_file\")\n",
-    "\n",
-    "    summary = ds.get(\"summary\", {}).copy()\n",
-    "    summary.update({\"task_name\": task_name, \"file\": file_name})\n",
-    "    summary_rows.append(summary)\n",
-    "\n",
-    "    for run in ds.get(\"runs\", []):\n",
-    "        row = run.copy()\n",
-    "        row.update({\"task_name\": task_name, \"file\": file_name})\n",
-    "        run_rows.append(row)\n",
-    "\n",
-    "        for log in run.get(\"logs\", []):\n",
-    "            lrow = log.copy()\n",
-    "            lrow.update({\n",
-    "                \"task_name\": task_name,\n",
-    "                \"file\": file_name,\n",
-    "                \"run_id\": run.get(\"run_id\"),\n",
-    "                \"run_completed\": run.get(\"completed\"),\n",
-    "            })\n",
-    "            log_rows.append(lrow)\n",
-    "\n",
-    "summary_df = pd.DataFrame(summary_rows)\n",
-    "runs_df = pd.DataFrame(run_rows)\n",
-    "logs_df = pd.DataFrame(log_rows)\n",
-    "\n",
-    "for col in [\"updated_at\", \"first_run_started_at\", \"last_run_started_at\", \"last_run_ended_at\"]:\n",
-    "    if col in summary_df.columns:\n",
-    "        summary_df[col] = pd.to_datetime(summary_df[col], errors=\"coerce\")\n",
-    "\n",
-    "for col in [\"started_at\", \"ended_at\"]:\n",
-    "    if col in runs_df.columns:\n",
-    "        runs_df[col] = pd.to_datetime(runs_df[col], errors=\"coerce\")\n",
-    "\n",
-    "if \"timestamp\" in logs_df.columns:\n",
-    "    logs_df[\"timestamp\"] = pd.to_datetime(logs_df[\"timestamp\"], errors=\"coerce\")\n",
-    "\n",
-    "summary_df.shape, runs_df.shape, logs_df.shape"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "id": "c974d4f8",
-   "metadata": {},
-   "source": [
-    "## Task-level summary"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "92aea6d2",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "summary_cols = [\n",
-    "    \"task_name\",\n",
-    "    \"total_runs\",\n",
-    "    \"completed_runs\",\n",
-    "    \"incomplete_runs\",\n",
-    "    \"average_duration_ms\",\n",
-    "    \"validation_attempts\",\n",
-    "    \"validation_successes\",\n",
-    "    \"validation_failures\",\n",
-    "    \"total_user_actions\",\n",
-    "]\n",
-    "\n",
-    "summary_view = summary_df.reindex(columns=summary_cols).copy()\n",
-    "\n",
-    "if not summary_view.empty:\n",
-    "    total_runs_nonzero = summary_view[\"total_runs\"].replace({0: pd.NA})\n",
-    "    summary_view[\"completion_rate_pct\"] = (summary_view[\"completed_runs\"] / total_runs_nonzero * 100).round(1)\n",
-    "    summary_view[\"avg_duration_s\"] = (summary_view[\"average_duration_ms\"] / 1000).round(2)\n",
-    "\n",
-    "summary_view.sort_values(\"completion_rate_pct\", ascending=False)"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "6bd0a081",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "if not summary_view.empty and summary_view[\"completion_rate_pct\"].notna().any():\n",
-    "    plot_df = summary_view[[\"task_name\", \"completion_rate_pct\"]].sort_values(\"completion_rate_pct\")\n",
-    "    ax = plot_df.plot(kind=\"barh\", x=\"task_name\", y=\"completion_rate_pct\", legend=False, figsize=(8, 4))\n",
-    "    ax.set_xlabel(\"Completion rate (%)\")\n",
-    "    ax.set_ylabel(\"\")\n",
-    "    ax.set_title(\"Completion rate by task\")\n",
-    "    plt.tight_layout()\n",
-    "else:\n",
-    "    print(\"No summary data available for plotting.\")"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "id": "8d7da541",
-   "metadata": {},
-   "source": [
-    "## Run-level diagnostics"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "694e0476",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "run_cols = [\n",
-    "    \"task_name\",\n",
-    "    \"run_id\",\n",
-    "    \"interface_type\",\n",
-    "    \"completed\",\n",
-    "    \"duration_ms\",\n",
-    "    \"validation_attempts\",\n",
-    "    \"validation_successes\",\n",
-    "    \"validation_failures\",\n",
-    "    \"questionnaire_completed\",\n",
-    "]\n",
-    "\n",
-    "runs_view = runs_df.reindex(columns=run_cols).copy()\n",
-    "if \"duration_ms\" in runs_view.columns:\n",
-    "    runs_view[\"duration_s\"] = (runs_view[\"duration_ms\"] / 1000).round(2)\n",
-    "\n",
-    "runs_view.sort_values([\"task_name\", \"run_id\"]).head(20)"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "7e0aae07",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "if not runs_df.empty and {\"task_name\", \"run_id\", \"completed\", \"duration_ms\", \"validation_attempts\"}.issubset(runs_df.columns):\n",
-    "    agg = runs_df.groupby(\"task_name\", dropna=False).agg(\n",
-    "        runs=(\"run_id\", \"count\"),\n",
-    "        completed_runs=(\"completed\", \"sum\"),\n",
-    "        avg_duration_s=(\"duration_ms\", lambda s: round(s.mean() / 1000, 2)),\n",
-    "        median_duration_s=(\"duration_ms\", lambda s: round(s.median() / 1000, 2)),\n",
-    "        avg_validation_attempts=(\"validation_attempts\", \"mean\"),\n",
-    "    )\n",
-    "\n",
-    "    agg[\"completion_rate_pct\"] = (agg[\"completed_runs\"] / agg[\"runs\"] * 100).round(1)\n",
-    "    agg[\"avg_validation_attempts\"] = agg[\"avg_validation_attempts\"].round(2)\n",
-    "    display(agg.sort_values(\"completion_rate_pct\", ascending=False))\n",
-    "else:\n",
-    "    print(\"Not enough run data to build aggregate diagnostics.\")"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "25d3f1a6",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "if not runs_df.empty and {\"duration_ms\", \"task_name\"}.issubset(runs_df.columns):\n",
-    "    runs_df.boxplot(column=\"duration_ms\", by=\"task_name\", figsize=(10, 5), rot=20)\n",
-    "    plt.title(\"Run duration distribution by task\")\n",
-    "    plt.suptitle(\"\")\n",
-    "    plt.ylabel(\"Duration (ms)\")\n",
-    "    plt.tight_layout()\n",
-    "else:\n",
-    "    print(\"No run duration data available for boxplot.\")"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "id": "6b89c284",
-   "metadata": {},
-   "source": [
-    "## Validation check failure hotspots"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "7e75fb47",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "if not logs_df.empty and {\"action\", \"result\", \"task_name\", \"target\"}.issubset(logs_df.columns):\n",
-    "    validation_checks = logs_df[(logs_df[\"action\"] == \"validate_check\") & (logs_df[\"result\"] == \"failed\")].copy()\n",
-    "\n",
-    "    if not validation_checks.empty:\n",
-    "        failed_by_target = (\n",
-    "            validation_checks\n",
-    "            .groupby([\"task_name\", \"target\"], dropna=False)\n",
-    "            .size()\n",
-    "            .reset_index(name=\"failed_count\")\n",
-    "            .sort_values([\"task_name\", \"failed_count\"], ascending=[True, False])\n",
-    "        )\n",
-    "        display(failed_by_target.groupby(\"task_name\", dropna=False).head(10))\n",
-    "    else:\n",
-    "        print(\"No failed validation checks found.\")\n",
-    "else:\n",
-    "    print(\"No validation logs available for failure hotspot analysis.\")"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "e44f3e85",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "if not logs_df.empty and {\"action\", \"task_name\", \"run_id\"}.issubset(logs_df.columns):\n",
-    "    attempt_logs = logs_df[logs_df[\"action\"] == \"validate_attempt\"].copy()\n",
-    "\n",
-    "    if not attempt_logs.empty:\n",
-    "        attempts_per_run = (\n",
-    "            attempt_logs\n",
-    "            .groupby([\"task_name\", \"run_id\"], dropna=False)\n",
-    "            .size()\n",
-    "            .reset_index(name=\"attempt_count\")\n",
-    "            .sort_values([\"task_name\", \"run_id\"])\n",
-    "        )\n",
-    "\n",
-    "        display(attempts_per_run.head(30))\n",
-    "\n",
-    "        fig, ax = plt.subplots(figsize=(10, 4))\n",
-    "        for task_name, g in attempts_per_run.groupby(\"task_name\"):\n",
-    "            ax.plot(g[\"run_id\"], g[\"attempt_count\"], marker=\"o\", label=task_name)\n",
-    "\n",
-    "        ax.set_title(\"Validation attempts per run\")\n",
-    "        ax.set_xlabel(\"Run ID\")\n",
-    "        ax.set_ylabel(\"Validation attempts\")\n",
-    "        ax.legend()\n",
-    "        plt.tight_layout()\n",
-    "    else:\n",
-    "        print(\"No validation attempt logs found.\")\n",
-    "else:\n",
-    "    print(\"No logs available for attempt trend analysis.\")"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "id": "a28fd392",
-   "metadata": {},
-   "source": [
-    "## User action endpoint activity"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "6f82ba83",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "if not logs_df.empty and {\"level\", \"task_name\", \"target\"}.issubset(logs_df.columns):\n",
-    "    user_actions = logs_df[logs_df[\"level\"] == \"user_action\"].copy()\n",
-    "\n",
-    "    if not user_actions.empty:\n",
-    "        endpoint_counts = (\n",
-    "            user_actions\n",
-    "            .groupby([\"task_name\", \"target\"], dropna=False)\n",
-    "            .size()\n",
-    "            .reset_index(name=\"count\")\n",
-    "            .sort_values([\"task_name\", \"count\"], ascending=[True, False])\n",
-    "        )\n",
-    "\n",
-    "        display(endpoint_counts.groupby(\"task_name\", dropna=False).head(15))\n",
-    "    else:\n",
-    "        print(\"No user_action logs found.\")\n",
-    "else:\n",
-    "    print(\"No logs available for endpoint activity analysis.\")"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "id": "32a64572",
-   "metadata": {},
-   "source": [
-    "## Quick filters for ad-hoc debugging"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "f8502e4a",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "# Example: inspect one task and one run.\n",
-    "if not runs_df.empty and {\"task_name\", \"run_id\"}.issubset(runs_df.columns):\n",
-    "    task = runs_df[\"task_name\"].iloc[0]\n",
-    "    run_id = 1\n",
-    "\n",
-    "    print(f\"Task: {task}, run_id: {run_id}\")\n",
-    "    display(runs_df[(runs_df[\"task_name\"] == task) & (runs_df[\"run_id\"] == run_id)])\n",
-    "\n",
-    "    if not logs_df.empty and {\"task_name\", \"run_id\"}.issubset(logs_df.columns):\n",
-    "        display(logs_df[(logs_df[\"task_name\"] == task) & (logs_df[\"run_id\"] == run_id)].head(50))\n",
-    "else:\n",
-    "    print(\"No run data available.\")"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "e6a94bb5",
-   "metadata": {},
-   "outputs": [],
-   "source": []
-  }
- ],
- "metadata": {
-  "kernelspec": {
-   "display_name": ".venv",
-   "language": "python",
-   "name": "python3"
-  },
-  "language_info": {
-   "codemirror_mode": {
-    "name": "ipython",
-    "version": 3
-   },
-   "file_extension": ".py",
-   "mimetype": "text/x-python",
-   "name": "python",
-   "nbconvert_exporter": "python",
-   "pygments_lexer": "ipython3",
-   "version": "3.14.3"
-  }
- },
- "nbformat": 4,
- "nbformat_minor": 5
-}