diff --git a/notebooks/task-data-analysis.ipynb b/notebooks/task-data-analysis.ipynb deleted file mode 100644 index de212e6..0000000 --- a/notebooks/task-data-analysis.ipynb +++ /dev/null @@ -1,418 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# Task Data Analysis Notebook\n", - "\n", - "This notebook loads task run stats from `./.pm` and gives a quick view of completion rates, validation behavior, timings, and common failure patterns." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "8626d9d2", - "metadata": {}, - "outputs": [], - "source": [ - "from pathlib import Path\n", - "import json\n", - "\n", - "import pandas as pd\n", - "import matplotlib.pyplot as plt\n", - "\n", - "pd.set_option(\"display.max_columns\", 200)\n", - "plt.style.use(\"ggplot\")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "7bc3e98d", - "metadata": {}, - "outputs": [], - "source": [ - "candidate_dirs = [\n", - " Path.cwd(),\n", - " Path.cwd() / \"./.pm\",\n", - " Path.cwd().parent / \"./.pm\",\n", - "]\n", - "\n", - "stats_dir = None\n", - "stat_files = []\n", - "for d in candidate_dirs:\n", - " if d.is_dir():\n", - " files = sorted(d.glob(\"*-stats.json\"))\n", - " if files:\n", - " stats_dir = d\n", - " stat_files = files\n", - " break\n", - "\n", - "print(f\"Working directory: {Path.cwd()}\")\n", - "print(f\"Using stats directory: {stats_dir}\")\n", - "print(f\"Found {len(stat_files)} stats file(s).\")\n", - "stat_files" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "c453e1ad", - "metadata": {}, - "outputs": [], - "source": [ - "datasets = []\n", - "for file in stat_files:\n", - " with file.open() as f:\n", - " data = json.load(f)\n", - " data[\"_file\"] = file.name\n", - " datasets.append(data)\n", - "\n", - "print(f\"Loaded {len(datasets)} task dataset(s).\")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "cbb0dd6b", - "metadata": {}, - "outputs": [], - "source": [ - "summary_rows = []\n", - "run_rows = []\n", - "log_rows = []\n", - "\n", - "for ds in datasets:\n", - " task_name = ds.get(\"task_name\")\n", - " file_name = ds.get(\"_file\")\n", - "\n", - " summary = ds.get(\"summary\", {}).copy()\n", - " summary.update({\"task_name\": task_name, \"file\": file_name})\n", - " summary_rows.append(summary)\n", - "\n", - " for run in ds.get(\"runs\", []):\n", - " row = run.copy()\n", - " row.update({\"task_name\": task_name, \"file\": file_name})\n", - " run_rows.append(row)\n", - "\n", - " for log in run.get(\"logs\", []):\n", - " lrow = log.copy()\n", - " lrow.update({\n", - " \"task_name\": task_name,\n", - " \"file\": file_name,\n", - " \"run_id\": run.get(\"run_id\"),\n", - " \"run_completed\": run.get(\"completed\"),\n", - " })\n", - " log_rows.append(lrow)\n", - "\n", - "summary_df = pd.DataFrame(summary_rows)\n", - "runs_df = pd.DataFrame(run_rows)\n", - "logs_df = pd.DataFrame(log_rows)\n", - "\n", - "for col in [\"updated_at\", \"first_run_started_at\", \"last_run_started_at\", \"last_run_ended_at\"]:\n", - " if col in summary_df.columns:\n", - " summary_df[col] = pd.to_datetime(summary_df[col], errors=\"coerce\")\n", - "\n", - "for col in [\"started_at\", \"ended_at\"]:\n", - " if col in runs_df.columns:\n", - " runs_df[col] = pd.to_datetime(runs_df[col], errors=\"coerce\")\n", - "\n", - "if \"timestamp\" in logs_df.columns:\n", - " logs_df[\"timestamp\"] = pd.to_datetime(logs_df[\"timestamp\"], errors=\"coerce\")\n", - "\n", - "summary_df.shape, runs_df.shape, logs_df.shape" - ] - }, - { - "cell_type": "markdown", - "id": "c974d4f8", - "metadata": {}, - "source": [ - "## Task-level summary" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "92aea6d2", - "metadata": {}, - "outputs": [], - "source": [ - "summary_cols = [\n", - " \"task_name\",\n", - " \"total_runs\",\n", - " \"completed_runs\",\n", - " \"incomplete_runs\",\n", - " \"average_duration_ms\",\n", - " \"validation_attempts\",\n", - " \"validation_successes\",\n", - " \"validation_failures\",\n", - " \"total_user_actions\",\n", - "]\n", - "\n", - "summary_view = summary_df.reindex(columns=summary_cols).copy()\n", - "\n", - "if not summary_view.empty:\n", - " total_runs_nonzero = summary_view[\"total_runs\"].replace({0: pd.NA})\n", - " summary_view[\"completion_rate_pct\"] = (summary_view[\"completed_runs\"] / total_runs_nonzero * 100).round(1)\n", - " summary_view[\"avg_duration_s\"] = (summary_view[\"average_duration_ms\"] / 1000).round(2)\n", - "\n", - "summary_view.sort_values(\"completion_rate_pct\", ascending=False)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "6bd0a081", - "metadata": {}, - "outputs": [], - "source": [ - "if not summary_view.empty and summary_view[\"completion_rate_pct\"].notna().any():\n", - " plot_df = summary_view[[\"task_name\", \"completion_rate_pct\"]].sort_values(\"completion_rate_pct\")\n", - " ax = plot_df.plot(kind=\"barh\", x=\"task_name\", y=\"completion_rate_pct\", legend=False, figsize=(8, 4))\n", - " ax.set_xlabel(\"Completion rate (%)\")\n", - " ax.set_ylabel(\"\")\n", - " ax.set_title(\"Completion rate by task\")\n", - " plt.tight_layout()\n", - "else:\n", - " print(\"No summary data available for plotting.\")" - ] - }, - { - "cell_type": "markdown", - "id": "8d7da541", - "metadata": {}, - "source": [ - "## Run-level diagnostics" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "694e0476", - "metadata": {}, - "outputs": [], - "source": [ - "run_cols = [\n", - " \"task_name\",\n", - " \"run_id\",\n", - " \"interface_type\",\n", - " \"completed\",\n", - " \"duration_ms\",\n", - " \"validation_attempts\",\n", - " \"validation_successes\",\n", - " \"validation_failures\",\n", - " \"questionnaire_completed\",\n", - "]\n", - "\n", - "runs_view = runs_df.reindex(columns=run_cols).copy()\n", - "if \"duration_ms\" in runs_view.columns:\n", - " runs_view[\"duration_s\"] = (runs_view[\"duration_ms\"] / 1000).round(2)\n", - "\n", - "runs_view.sort_values([\"task_name\", \"run_id\"]).head(20)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "7e0aae07", - "metadata": {}, - "outputs": [], - "source": [ - "if not runs_df.empty and {\"task_name\", \"run_id\", \"completed\", \"duration_ms\", \"validation_attempts\"}.issubset(runs_df.columns):\n", - " agg = runs_df.groupby(\"task_name\", dropna=False).agg(\n", - " runs=(\"run_id\", \"count\"),\n", - " completed_runs=(\"completed\", \"sum\"),\n", - " avg_duration_s=(\"duration_ms\", lambda s: round(s.mean() / 1000, 2)),\n", - " median_duration_s=(\"duration_ms\", lambda s: round(s.median() / 1000, 2)),\n", - " avg_validation_attempts=(\"validation_attempts\", \"mean\"),\n", - " )\n", - "\n", - " agg[\"completion_rate_pct\"] = (agg[\"completed_runs\"] / agg[\"runs\"] * 100).round(1)\n", - " agg[\"avg_validation_attempts\"] = agg[\"avg_validation_attempts\"].round(2)\n", - " display(agg.sort_values(\"completion_rate_pct\", ascending=False))\n", - "else:\n", - " print(\"Not enough run data to build aggregate diagnostics.\")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "25d3f1a6", - "metadata": {}, - "outputs": [], - "source": [ - "if not runs_df.empty and {\"duration_ms\", \"task_name\"}.issubset(runs_df.columns):\n", - " runs_df.boxplot(column=\"duration_ms\", by=\"task_name\", figsize=(10, 5), rot=20)\n", - " plt.title(\"Run duration distribution by task\")\n", - " plt.suptitle(\"\")\n", - " plt.ylabel(\"Duration (ms)\")\n", - " plt.tight_layout()\n", - "else:\n", - " print(\"No run duration data available for boxplot.\")" - ] - }, - { - "cell_type": "markdown", - "id": "6b89c284", - "metadata": {}, - "source": [ - "## Validation check failure hotspots" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "7e75fb47", - "metadata": {}, - "outputs": [], - "source": [ - "if not logs_df.empty and {\"action\", \"result\", \"task_name\", \"target\"}.issubset(logs_df.columns):\n", - " validation_checks = logs_df[(logs_df[\"action\"] == \"validate_check\") & (logs_df[\"result\"] == \"failed\")].copy()\n", - "\n", - " if not validation_checks.empty:\n", - " failed_by_target = (\n", - " validation_checks\n", - " .groupby([\"task_name\", \"target\"], dropna=False)\n", - " .size()\n", - " .reset_index(name=\"failed_count\")\n", - " .sort_values([\"task_name\", \"failed_count\"], ascending=[True, False])\n", - " )\n", - " display(failed_by_target.groupby(\"task_name\", dropna=False).head(10))\n", - " else:\n", - " print(\"No failed validation checks found.\")\n", - "else:\n", - " print(\"No validation logs available for failure hotspot analysis.\")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "e44f3e85", - "metadata": {}, - "outputs": [], - "source": [ - "if not logs_df.empty and {\"action\", \"task_name\", \"run_id\"}.issubset(logs_df.columns):\n", - " attempt_logs = logs_df[logs_df[\"action\"] == \"validate_attempt\"].copy()\n", - "\n", - " if not attempt_logs.empty:\n", - " attempts_per_run = (\n", - " attempt_logs\n", - " .groupby([\"task_name\", \"run_id\"], dropna=False)\n", - " .size()\n", - " .reset_index(name=\"attempt_count\")\n", - " .sort_values([\"task_name\", \"run_id\"])\n", - " )\n", - "\n", - " display(attempts_per_run.head(30))\n", - "\n", - " fig, ax = plt.subplots(figsize=(10, 4))\n", - " for task_name, g in attempts_per_run.groupby(\"task_name\"):\n", - " ax.plot(g[\"run_id\"], g[\"attempt_count\"], marker=\"o\", label=task_name)\n", - "\n", - " ax.set_title(\"Validation attempts per run\")\n", - " ax.set_xlabel(\"Run ID\")\n", - " ax.set_ylabel(\"Validation attempts\")\n", - " ax.legend()\n", - " plt.tight_layout()\n", - " else:\n", - " print(\"No validation attempt logs found.\")\n", - "else:\n", - " print(\"No logs available for attempt trend analysis.\")" - ] - }, - { - "cell_type": "markdown", - "id": "a28fd392", - "metadata": {}, - "source": [ - "## User action endpoint activity" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "6f82ba83", - "metadata": {}, - "outputs": [], - "source": [ - "if not logs_df.empty and {\"level\", \"task_name\", \"target\"}.issubset(logs_df.columns):\n", - " user_actions = logs_df[logs_df[\"level\"] == \"user_action\"].copy()\n", - "\n", - " if not user_actions.empty:\n", - " endpoint_counts = (\n", - " user_actions\n", - " .groupby([\"task_name\", \"target\"], dropna=False)\n", - " .size()\n", - " .reset_index(name=\"count\")\n", - " .sort_values([\"task_name\", \"count\"], ascending=[True, False])\n", - " )\n", - "\n", - " display(endpoint_counts.groupby(\"task_name\", dropna=False).head(15))\n", - " else:\n", - " print(\"No user_action logs found.\")\n", - "else:\n", - " print(\"No logs available for endpoint activity analysis.\")" - ] - }, - { - "cell_type": "markdown", - "id": "32a64572", - "metadata": {}, - "source": [ - "## Quick filters for ad-hoc debugging" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "f8502e4a", - "metadata": {}, - "outputs": [], - "source": [ - "# Example: inspect one task and one run.\n", - "if not runs_df.empty and {\"task_name\", \"run_id\"}.issubset(runs_df.columns):\n", - " task = runs_df[\"task_name\"].iloc[0]\n", - " run_id = 1\n", - "\n", - " print(f\"Task: {task}, run_id: {run_id}\")\n", - " display(runs_df[(runs_df[\"task_name\"] == task) & (runs_df[\"run_id\"] == run_id)])\n", - "\n", - " if not logs_df.empty and {\"task_name\", \"run_id\"}.issubset(logs_df.columns):\n", - " display(logs_df[(logs_df[\"task_name\"] == task) & (logs_df[\"run_id\"] == run_id)].head(50))\n", - "else:\n", - " print(\"No run data available.\")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "e6a94bb5", - "metadata": {}, - "outputs": [], - "source": [] - } - ], - "metadata": { - "kernelspec": { - "display_name": ".venv", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.14.3" - } - }, - "nbformat": 4, - "nbformat_minor": 5 -}