{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Task Data Analysis Notebook\n",
    "\n",
    "This notebook loads task run stats from `./.pm` and gives a quick view of completion rates, validation behavior, timings, and common failure patterns."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8626d9d2",
   "metadata": {},
   "outputs": [],
   "source": [
    "from pathlib import Path\n",
    "import json\n",
    "\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "pd.set_option(\"display.max_columns\", 200)\n",
    "plt.style.use(\"ggplot\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7bc3e98d",
   "metadata": {},
   "outputs": [],
   "source": [
    "candidate_dirs = [\n",
    "    Path.cwd(),\n",
    "    Path.cwd() / \"./.pm\",\n",
    "    Path.cwd().parent / \"./.pm\",\n",
    "]\n",
    "\n",
    "stats_dir = None\n",
    "stat_files = []\n",
    "for d in candidate_dirs:\n",
    "    if d.is_dir():\n",
    "        files = sorted(d.glob(\"*-stats.json\"))\n",
    "        if files:\n",
    "            stats_dir = d\n",
    "            stat_files = files\n",
    "            break\n",
    "\n",
    "print(f\"Working directory: {Path.cwd()}\")\n",
    "print(f\"Using stats directory: {stats_dir}\")\n",
    "print(f\"Found {len(stat_files)} stats file(s).\")\n",
    "stat_files"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c453e1ad",
   "metadata": {},
   "outputs": [],
   "source": [
    "datasets = []\n",
    "for file in stat_files:\n",
    "    with file.open() as f:\n",
    "        data = json.load(f)\n",
    "    data[\"_file\"] = file.name\n",
    "    datasets.append(data)\n",
    "\n",
    "print(f\"Loaded {len(datasets)} task dataset(s).\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "cbb0dd6b",
   "metadata": {},
   "outputs": [],
   "source": [
    "summary_rows = []\n",
    "run_rows = []\n",
    "log_rows = []\n",
    "\n",
    "for ds in datasets:\n",
    "    task_name = ds.get(\"task_name\")\n",
    "    file_name = ds.get(\"_file\")\n",
    "\n",
    "    summary = ds.get(\"summary\", {}).copy()\n",
    "    summary.update({\"task_name\": task_name, \"file\": file_name})\n",
    "    summary_rows.append(summary)\n",
    "\n",
    "    for run in ds.get(\"runs\", []):\n",
    "        row = run.copy()\n",
    "        row.update({\"task_name\": task_name, \"file\": file_name})\n",
    "        run_rows.append(row)\n",
    "\n",
    "        for log in run.get(\"logs\", []):\n",
    "            lrow = log.copy()\n",
    "            lrow.update({\n",
    "                \"task_name\": task_name,\n",
    "                \"file\": file_name,\n",
    "                \"run_id\": run.get(\"run_id\"),\n",
    "                \"run_completed\": run.get(\"completed\"),\n",
    "            })\n",
    "            log_rows.append(lrow)\n",
    "\n",
    "summary_df = pd.DataFrame(summary_rows)\n",
    "runs_df = pd.DataFrame(run_rows)\n",
    "logs_df = pd.DataFrame(log_rows)\n",
    "\n",
    "for col in [\"updated_at\", \"first_run_started_at\", \"last_run_started_at\", \"last_run_ended_at\"]:\n",
    "    if col in summary_df.columns:\n",
    "        summary_df[col] = pd.to_datetime(summary_df[col], errors=\"coerce\")\n",
    "\n",
    "for col in [\"started_at\", \"ended_at\"]:\n",
    "    if col in runs_df.columns:\n",
    "        runs_df[col] = pd.to_datetime(runs_df[col], errors=\"coerce\")\n",
    "\n",
    "if \"timestamp\" in logs_df.columns:\n",
    "    logs_df[\"timestamp\"] = pd.to_datetime(logs_df[\"timestamp\"], errors=\"coerce\")\n",
    "\n",
    "summary_df.shape, runs_df.shape, logs_df.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c974d4f8",
   "metadata": {},
   "source": [
    "## Task-level summary"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "92aea6d2",
   "metadata": {},
   "outputs": [],
   "source": [
    "summary_cols = [\n",
    "    \"task_name\",\n",
    "    \"total_runs\",\n",
    "    \"completed_runs\",\n",
    "    \"incomplete_runs\",\n",
    "    \"average_duration_ms\",\n",
    "    \"validation_attempts\",\n",
    "    \"validation_successes\",\n",
    "    \"validation_failures\",\n",
    "    \"total_user_actions\",\n",
    "]\n",
    "\n",
    "summary_view = summary_df.reindex(columns=summary_cols).copy()\n",
    "\n",
    "if not summary_view.empty:\n",
    "    total_runs_nonzero = summary_view[\"total_runs\"].replace({0: pd.NA})\n",
    "    summary_view[\"completion_rate_pct\"] = (summary_view[\"completed_runs\"] / total_runs_nonzero * 100).round(1)\n",
    "    summary_view[\"avg_duration_s\"] = (summary_view[\"average_duration_ms\"] / 1000).round(2)\n",
    "\n",
    "summary_view.sort_values(\"completion_rate_pct\", ascending=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6bd0a081",
   "metadata": {},
   "outputs": [],
   "source": [
    "if not summary_view.empty and summary_view[\"completion_rate_pct\"].notna().any():\n",
    "    plot_df = summary_view[[\"task_name\", \"completion_rate_pct\"]].sort_values(\"completion_rate_pct\")\n",
    "    ax = plot_df.plot(kind=\"barh\", x=\"task_name\", y=\"completion_rate_pct\", legend=False, figsize=(8, 4))\n",
    "    ax.set_xlabel(\"Completion rate (%)\")\n",
    "    ax.set_ylabel(\"\")\n",
    "    ax.set_title(\"Completion rate by task\")\n",
    "    plt.tight_layout()\n",
    "else:\n",
    "    print(\"No summary data available for plotting.\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "8d7da541",
   "metadata": {},
   "source": [
    "## Run-level diagnostics"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "694e0476",
   "metadata": {},
   "outputs": [],
   "source": [
    "run_cols = [\n",
    "    \"task_name\",\n",
    "    \"run_id\",\n",
    "    \"interface_type\",\n",
    "    \"completed\",\n",
    "    \"duration_ms\",\n",
    "    \"validation_attempts\",\n",
    "    \"validation_successes\",\n",
    "    \"validation_failures\",\n",
    "    \"questionnaire_completed\",\n",
    "]\n",
    "\n",
    "runs_view = runs_df.reindex(columns=run_cols).copy()\n",
    "if \"duration_ms\" in runs_view.columns:\n",
    "    runs_view[\"duration_s\"] = (runs_view[\"duration_ms\"] / 1000).round(2)\n",
    "\n",
    "runs_view.sort_values([\"task_name\", \"run_id\"]).head(20)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7e0aae07",
   "metadata": {},
   "outputs": [],
   "source": [
    "if not runs_df.empty and {\"task_name\", \"run_id\", \"completed\", \"duration_ms\", \"validation_attempts\"}.issubset(runs_df.columns):\n",
    "    agg = runs_df.groupby(\"task_name\", dropna=False).agg(\n",
    "        runs=(\"run_id\", \"count\"),\n",
    "        completed_runs=(\"completed\", \"sum\"),\n",
    "        avg_duration_s=(\"duration_ms\", lambda s: round(s.mean() / 1000, 2)),\n",
    "        median_duration_s=(\"duration_ms\", lambda s: round(s.median() / 1000, 2)),\n",
    "        avg_validation_attempts=(\"validation_attempts\", \"mean\"),\n",
    "    )\n",
    "\n",
    "    agg[\"completion_rate_pct\"] = (agg[\"completed_runs\"] / agg[\"runs\"] * 100).round(1)\n",
    "    agg[\"avg_validation_attempts\"] = agg[\"avg_validation_attempts\"].round(2)\n",
    "    display(agg.sort_values(\"completion_rate_pct\", ascending=False))\n",
    "else:\n",
    "    print(\"Not enough run data to build aggregate diagnostics.\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "25d3f1a6",
   "metadata": {},
   "outputs": [],
   "source": [
    "if not runs_df.empty and {\"duration_ms\", \"task_name\"}.issubset(runs_df.columns):\n",
    "    runs_df.boxplot(column=\"duration_ms\", by=\"task_name\", figsize=(10, 5), rot=20)\n",
    "    plt.title(\"Run duration distribution by task\")\n",
    "    plt.suptitle(\"\")\n",
    "    plt.ylabel(\"Duration (ms)\")\n",
    "    plt.tight_layout()\n",
    "else:\n",
    "    print(\"No run duration data available for boxplot.\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "6b89c284",
   "metadata": {},
   "source": [
    "## Validation check failure hotspots"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7e75fb47",
   "metadata": {},
   "outputs": [],
   "source": [
    "if not logs_df.empty and {\"action\", \"result\", \"task_name\", \"target\"}.issubset(logs_df.columns):\n",
    "    validation_checks = logs_df[(logs_df[\"action\"] == \"validate_check\") & (logs_df[\"result\"] == \"failed\")].copy()\n",
    "\n",
    "    if not validation_checks.empty:\n",
    "        failed_by_target = (\n",
    "            validation_checks\n",
    "            .groupby([\"task_name\", \"target\"], dropna=False)\n",
    "            .size()\n",
    "            .reset_index(name=\"failed_count\")\n",
    "            .sort_values([\"task_name\", \"failed_count\"], ascending=[True, False])\n",
    "        )\n",
    "        display(failed_by_target.groupby(\"task_name\", dropna=False).head(10))\n",
    "    else:\n",
    "        print(\"No failed validation checks found.\")\n",
    "else:\n",
    "    print(\"No validation logs available for failure hotspot analysis.\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e44f3e85",
   "metadata": {},
   "outputs": [],
   "source": [
    "if not logs_df.empty and {\"action\", \"task_name\", \"run_id\"}.issubset(logs_df.columns):\n",
    "    attempt_logs = logs_df[logs_df[\"action\"] == \"validate_attempt\"].copy()\n",
    "\n",
    "    if not attempt_logs.empty:\n",
    "        attempts_per_run = (\n",
    "            attempt_logs\n",
    "            .groupby([\"task_name\", \"run_id\"], dropna=False)\n",
    "            .size()\n",
    "            .reset_index(name=\"attempt_count\")\n",
    "            .sort_values([\"task_name\", \"run_id\"])\n",
    "        )\n",
    "\n",
    "        display(attempts_per_run.head(30))\n",
    "\n",
    "        fig, ax = plt.subplots(figsize=(10, 4))\n",
    "        for task_name, g in attempts_per_run.groupby(\"task_name\"):\n",
    "            ax.plot(g[\"run_id\"], g[\"attempt_count\"], marker=\"o\", label=task_name)\n",
    "\n",
    "        ax.set_title(\"Validation attempts per run\")\n",
    "        ax.set_xlabel(\"Run ID\")\n",
    "        ax.set_ylabel(\"Validation attempts\")\n",
    "        ax.legend()\n",
    "        plt.tight_layout()\n",
    "    else:\n",
    "        print(\"No validation attempt logs found.\")\n",
    "else:\n",
    "    print(\"No logs available for attempt trend analysis.\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a28fd392",
   "metadata": {},
   "source": [
    "## User action endpoint activity"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6f82ba83",
   "metadata": {},
   "outputs": [],
   "source": [
    "if not logs_df.empty and {\"level\", \"task_name\", \"target\"}.issubset(logs_df.columns):\n",
    "    user_actions = logs_df[logs_df[\"level\"] == \"user_action\"].copy()\n",
    "\n",
    "    if not user_actions.empty:\n",
    "        endpoint_counts = (\n",
    "            user_actions\n",
    "            .groupby([\"task_name\", \"target\"], dropna=False)\n",
    "            .size()\n",
    "            .reset_index(name=\"count\")\n",
    "            .sort_values([\"task_name\", \"count\"], ascending=[True, False])\n",
    "        )\n",
    "\n",
    "        display(endpoint_counts.groupby(\"task_name\", dropna=False).head(15))\n",
    "    else:\n",
    "        print(\"No user_action logs found.\")\n",
    "else:\n",
    "    print(\"No logs available for endpoint activity analysis.\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "32a64572",
   "metadata": {},
   "source": [
    "## Quick filters for ad-hoc debugging"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f8502e4a",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Example: inspect one task and one run.\n",
    "if not runs_df.empty and {\"task_name\", \"run_id\"}.issubset(runs_df.columns):\n",
    "    task = runs_df[\"task_name\"].iloc[0]\n",
    "    run_id = 1\n",
    "\n",
    "    print(f\"Task: {task}, run_id: {run_id}\")\n",
    "    display(runs_df[(runs_df[\"task_name\"] == task) & (runs_df[\"run_id\"] == run_id)])\n",
    "\n",
    "    if not logs_df.empty and {\"task_name\", \"run_id\"}.issubset(logs_df.columns):\n",
    "        display(logs_df[(logs_df[\"task_name\"] == task) & (logs_df[\"run_id\"] == run_id)].head(50))\n",
    "else:\n",
    "    print(\"No run data available.\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e6a94bb5",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": ".venv",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.14.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}