419 lines
13 KiB
Plaintext
419 lines
13 KiB
Plaintext
{
|
|
"cells": [
|
|
{
|
|
"cell_type": "markdown",
|
|
"metadata": {},
|
|
"source": [
|
|
"# Task Data Analysis Notebook\n",
|
|
"\n",
|
|
"This notebook loads task run stats from `./.pm` and gives a quick view of completion rates, validation behavior, timings, and common failure patterns."
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "8626d9d2",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"from pathlib import Path\n",
|
|
"import json\n",
|
|
"\n",
|
|
"import pandas as pd\n",
|
|
"import matplotlib.pyplot as plt\n",
|
|
"\n",
|
|
"pd.set_option(\"display.max_columns\", 200)\n",
|
|
"plt.style.use(\"ggplot\")"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "7bc3e98d",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"candidate_dirs = [\n",
|
|
" Path.cwd(),\n",
|
|
" Path.cwd() / \"./.pm\",\n",
|
|
" Path.cwd().parent / \"./.pm\",\n",
|
|
"]\n",
|
|
"\n",
|
|
"stats_dir = None\n",
|
|
"stat_files = []\n",
|
|
"for d in candidate_dirs:\n",
|
|
" if d.is_dir():\n",
|
|
" files = sorted(d.glob(\"*-stats.json\"))\n",
|
|
" if files:\n",
|
|
" stats_dir = d\n",
|
|
" stat_files = files\n",
|
|
" break\n",
|
|
"\n",
|
|
"print(f\"Working directory: {Path.cwd()}\")\n",
|
|
"print(f\"Using stats directory: {stats_dir}\")\n",
|
|
"print(f\"Found {len(stat_files)} stats file(s).\")\n",
|
|
"stat_files"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "c453e1ad",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"datasets = []\n",
|
|
"for file in stat_files:\n",
|
|
" with file.open() as f:\n",
|
|
" data = json.load(f)\n",
|
|
" data[\"_file\"] = file.name\n",
|
|
" datasets.append(data)\n",
|
|
"\n",
|
|
"print(f\"Loaded {len(datasets)} task dataset(s).\")"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "cbb0dd6b",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"summary_rows = []\n",
|
|
"run_rows = []\n",
|
|
"log_rows = []\n",
|
|
"\n",
|
|
"for ds in datasets:\n",
|
|
" task_name = ds.get(\"task_name\")\n",
|
|
" file_name = ds.get(\"_file\")\n",
|
|
"\n",
|
|
" summary = ds.get(\"summary\", {}).copy()\n",
|
|
" summary.update({\"task_name\": task_name, \"file\": file_name})\n",
|
|
" summary_rows.append(summary)\n",
|
|
"\n",
|
|
" for run in ds.get(\"runs\", []):\n",
|
|
" row = run.copy()\n",
|
|
" row.update({\"task_name\": task_name, \"file\": file_name})\n",
|
|
" run_rows.append(row)\n",
|
|
"\n",
|
|
" for log in run.get(\"logs\", []):\n",
|
|
" lrow = log.copy()\n",
|
|
" lrow.update({\n",
|
|
" \"task_name\": task_name,\n",
|
|
" \"file\": file_name,\n",
|
|
" \"run_id\": run.get(\"run_id\"),\n",
|
|
" \"run_completed\": run.get(\"completed\"),\n",
|
|
" })\n",
|
|
" log_rows.append(lrow)\n",
|
|
"\n",
|
|
"summary_df = pd.DataFrame(summary_rows)\n",
|
|
"runs_df = pd.DataFrame(run_rows)\n",
|
|
"logs_df = pd.DataFrame(log_rows)\n",
|
|
"\n",
|
|
"for col in [\"updated_at\", \"first_run_started_at\", \"last_run_started_at\", \"last_run_ended_at\"]:\n",
|
|
" if col in summary_df.columns:\n",
|
|
" summary_df[col] = pd.to_datetime(summary_df[col], errors=\"coerce\")\n",
|
|
"\n",
|
|
"for col in [\"started_at\", \"ended_at\"]:\n",
|
|
" if col in runs_df.columns:\n",
|
|
" runs_df[col] = pd.to_datetime(runs_df[col], errors=\"coerce\")\n",
|
|
"\n",
|
|
"if \"timestamp\" in logs_df.columns:\n",
|
|
" logs_df[\"timestamp\"] = pd.to_datetime(logs_df[\"timestamp\"], errors=\"coerce\")\n",
|
|
"\n",
|
|
"summary_df.shape, runs_df.shape, logs_df.shape"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"id": "c974d4f8",
|
|
"metadata": {},
|
|
"source": [
|
|
"## Task-level summary"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "92aea6d2",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"summary_cols = [\n",
|
|
" \"task_name\",\n",
|
|
" \"total_runs\",\n",
|
|
" \"completed_runs\",\n",
|
|
" \"incomplete_runs\",\n",
|
|
" \"average_duration_ms\",\n",
|
|
" \"validation_attempts\",\n",
|
|
" \"validation_successes\",\n",
|
|
" \"validation_failures\",\n",
|
|
" \"total_user_actions\",\n",
|
|
"]\n",
|
|
"\n",
|
|
"summary_view = summary_df.reindex(columns=summary_cols).copy()\n",
|
|
"\n",
|
|
"if not summary_view.empty:\n",
|
|
" total_runs_nonzero = summary_view[\"total_runs\"].replace({0: pd.NA})\n",
|
|
" summary_view[\"completion_rate_pct\"] = (summary_view[\"completed_runs\"] / total_runs_nonzero * 100).round(1)\n",
|
|
" summary_view[\"avg_duration_s\"] = (summary_view[\"average_duration_ms\"] / 1000).round(2)\n",
|
|
"\n",
|
|
"summary_view.sort_values(\"completion_rate_pct\", ascending=False)"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "6bd0a081",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"if not summary_view.empty and summary_view[\"completion_rate_pct\"].notna().any():\n",
|
|
" plot_df = summary_view[[\"task_name\", \"completion_rate_pct\"]].sort_values(\"completion_rate_pct\")\n",
|
|
" ax = plot_df.plot(kind=\"barh\", x=\"task_name\", y=\"completion_rate_pct\", legend=False, figsize=(8, 4))\n",
|
|
" ax.set_xlabel(\"Completion rate (%)\")\n",
|
|
" ax.set_ylabel(\"\")\n",
|
|
" ax.set_title(\"Completion rate by task\")\n",
|
|
" plt.tight_layout()\n",
|
|
"else:\n",
|
|
" print(\"No summary data available for plotting.\")"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"id": "8d7da541",
|
|
"metadata": {},
|
|
"source": [
|
|
"## Run-level diagnostics"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "694e0476",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"run_cols = [\n",
|
|
" \"task_name\",\n",
|
|
" \"run_id\",\n",
|
|
" \"interface_type\",\n",
|
|
" \"completed\",\n",
|
|
" \"duration_ms\",\n",
|
|
" \"validation_attempts\",\n",
|
|
" \"validation_successes\",\n",
|
|
" \"validation_failures\",\n",
|
|
" \"questionnaire_completed\",\n",
|
|
"]\n",
|
|
"\n",
|
|
"runs_view = runs_df.reindex(columns=run_cols).copy()\n",
|
|
"if \"duration_ms\" in runs_view.columns:\n",
|
|
" runs_view[\"duration_s\"] = (runs_view[\"duration_ms\"] / 1000).round(2)\n",
|
|
"\n",
|
|
"runs_view.sort_values([\"task_name\", \"run_id\"]).head(20)"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "7e0aae07",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"if not runs_df.empty and {\"task_name\", \"run_id\", \"completed\", \"duration_ms\", \"validation_attempts\"}.issubset(runs_df.columns):\n",
|
|
" agg = runs_df.groupby(\"task_name\", dropna=False).agg(\n",
|
|
" runs=(\"run_id\", \"count\"),\n",
|
|
" completed_runs=(\"completed\", \"sum\"),\n",
|
|
" avg_duration_s=(\"duration_ms\", lambda s: round(s.mean() / 1000, 2)),\n",
|
|
" median_duration_s=(\"duration_ms\", lambda s: round(s.median() / 1000, 2)),\n",
|
|
" avg_validation_attempts=(\"validation_attempts\", \"mean\"),\n",
|
|
" )\n",
|
|
"\n",
|
|
" agg[\"completion_rate_pct\"] = (agg[\"completed_runs\"] / agg[\"runs\"] * 100).round(1)\n",
|
|
" agg[\"avg_validation_attempts\"] = agg[\"avg_validation_attempts\"].round(2)\n",
|
|
" display(agg.sort_values(\"completion_rate_pct\", ascending=False))\n",
|
|
"else:\n",
|
|
" print(\"Not enough run data to build aggregate diagnostics.\")"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "25d3f1a6",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"if not runs_df.empty and {\"duration_ms\", \"task_name\"}.issubset(runs_df.columns):\n",
|
|
" runs_df.boxplot(column=\"duration_ms\", by=\"task_name\", figsize=(10, 5), rot=20)\n",
|
|
" plt.title(\"Run duration distribution by task\")\n",
|
|
" plt.suptitle(\"\")\n",
|
|
" plt.ylabel(\"Duration (ms)\")\n",
|
|
" plt.tight_layout()\n",
|
|
"else:\n",
|
|
" print(\"No run duration data available for boxplot.\")"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"id": "6b89c284",
|
|
"metadata": {},
|
|
"source": [
|
|
"## Validation check failure hotspots"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "7e75fb47",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"if not logs_df.empty and {\"action\", \"result\", \"task_name\", \"target\"}.issubset(logs_df.columns):\n",
|
|
" validation_checks = logs_df[(logs_df[\"action\"] == \"validate_check\") & (logs_df[\"result\"] == \"failed\")].copy()\n",
|
|
"\n",
|
|
" if not validation_checks.empty:\n",
|
|
" failed_by_target = (\n",
|
|
" validation_checks\n",
|
|
" .groupby([\"task_name\", \"target\"], dropna=False)\n",
|
|
" .size()\n",
|
|
" .reset_index(name=\"failed_count\")\n",
|
|
" .sort_values([\"task_name\", \"failed_count\"], ascending=[True, False])\n",
|
|
" )\n",
|
|
" display(failed_by_target.groupby(\"task_name\", dropna=False).head(10))\n",
|
|
" else:\n",
|
|
" print(\"No failed validation checks found.\")\n",
|
|
"else:\n",
|
|
" print(\"No validation logs available for failure hotspot analysis.\")"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "e44f3e85",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"if not logs_df.empty and {\"action\", \"task_name\", \"run_id\"}.issubset(logs_df.columns):\n",
|
|
" attempt_logs = logs_df[logs_df[\"action\"] == \"validate_attempt\"].copy()\n",
|
|
"\n",
|
|
" if not attempt_logs.empty:\n",
|
|
" attempts_per_run = (\n",
|
|
" attempt_logs\n",
|
|
" .groupby([\"task_name\", \"run_id\"], dropna=False)\n",
|
|
" .size()\n",
|
|
" .reset_index(name=\"attempt_count\")\n",
|
|
" .sort_values([\"task_name\", \"run_id\"])\n",
|
|
" )\n",
|
|
"\n",
|
|
" display(attempts_per_run.head(30))\n",
|
|
"\n",
|
|
" fig, ax = plt.subplots(figsize=(10, 4))\n",
|
|
" for task_name, g in attempts_per_run.groupby(\"task_name\"):\n",
|
|
" ax.plot(g[\"run_id\"], g[\"attempt_count\"], marker=\"o\", label=task_name)\n",
|
|
"\n",
|
|
" ax.set_title(\"Validation attempts per run\")\n",
|
|
" ax.set_xlabel(\"Run ID\")\n",
|
|
" ax.set_ylabel(\"Validation attempts\")\n",
|
|
" ax.legend()\n",
|
|
" plt.tight_layout()\n",
|
|
" else:\n",
|
|
" print(\"No validation attempt logs found.\")\n",
|
|
"else:\n",
|
|
" print(\"No logs available for attempt trend analysis.\")"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"id": "a28fd392",
|
|
"metadata": {},
|
|
"source": [
|
|
"## User action endpoint activity"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "6f82ba83",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"if not logs_df.empty and {\"level\", \"task_name\", \"target\"}.issubset(logs_df.columns):\n",
|
|
" user_actions = logs_df[logs_df[\"level\"] == \"user_action\"].copy()\n",
|
|
"\n",
|
|
" if not user_actions.empty:\n",
|
|
" endpoint_counts = (\n",
|
|
" user_actions\n",
|
|
" .groupby([\"task_name\", \"target\"], dropna=False)\n",
|
|
" .size()\n",
|
|
" .reset_index(name=\"count\")\n",
|
|
" .sort_values([\"task_name\", \"count\"], ascending=[True, False])\n",
|
|
" )\n",
|
|
"\n",
|
|
" display(endpoint_counts.groupby(\"task_name\", dropna=False).head(15))\n",
|
|
" else:\n",
|
|
" print(\"No user_action logs found.\")\n",
|
|
"else:\n",
|
|
" print(\"No logs available for endpoint activity analysis.\")"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"id": "32a64572",
|
|
"metadata": {},
|
|
"source": [
|
|
"## Quick filters for ad-hoc debugging"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "f8502e4a",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"# Example: inspect one task and one run.\n",
|
|
"if not runs_df.empty and {\"task_name\", \"run_id\"}.issubset(runs_df.columns):\n",
|
|
" task = runs_df[\"task_name\"].iloc[0]\n",
|
|
" run_id = 1\n",
|
|
"\n",
|
|
" print(f\"Task: {task}, run_id: {run_id}\")\n",
|
|
" display(runs_df[(runs_df[\"task_name\"] == task) & (runs_df[\"run_id\"] == run_id)])\n",
|
|
"\n",
|
|
" if not logs_df.empty and {\"task_name\", \"run_id\"}.issubset(logs_df.columns):\n",
|
|
" display(logs_df[(logs_df[\"task_name\"] == task) & (logs_df[\"run_id\"] == run_id)].head(50))\n",
|
|
"else:\n",
|
|
" print(\"No run data available.\")"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "e6a94bb5",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": []
|
|
}
|
|
],
|
|
"metadata": {
|
|
"kernelspec": {
|
|
"display_name": ".venv",
|
|
"language": "python",
|
|
"name": "python3"
|
|
},
|
|
"language_info": {
|
|
"codemirror_mode": {
|
|
"name": "ipython",
|
|
"version": 3
|
|
},
|
|
"file_extension": ".py",
|
|
"mimetype": "text/x-python",
|
|
"name": "python",
|
|
"nbconvert_exporter": "python",
|
|
"pygments_lexer": "ipython3",
|
|
"version": "3.14.3"
|
|
}
|
|
},
|
|
"nbformat": 4,
|
|
"nbformat_minor": 5
|
|
}
|