refine statistics collection

internal/app/statistics.go

@@ -64,7 +64,9 @@ func (s *StatisticsService) RecordTaskRun(ctx context.Context, run models.TaskRu
 	}
 
 	stats := s.storage.Data
-	now := time.Now()
+	if stats.MetricsVersion == 0 {
+		stats.MetricsVersion = models.CurrentMetricsVersion
+	}
 
 	if stats.StartTime.IsZero() {
 		stats.StartTime = run.StartedAt
@@ -73,13 +75,25 @@ func (s *StatisticsService) RecordTaskRun(ctx context.Context, run models.TaskRu
 		stats.StartTime = run.StartedAt
 	}
 
-	stats.EndTime = now
+	stats.EndTime = run.EndedAt
+	if stats.EndTime.IsZero() {
+		stats.EndTime = time.Now()
+	}
+	if run.StartedAt.IsZero() || run.EndedAt.IsZero() || run.EndedAt.Before(run.StartedAt) {
+		stats.SessionClockAnomaly = true
+	}
+	if stats.EndTime.Before(stats.StartTime) {
+		stats.SessionClockAnomaly = true
+	}
 	stats.DurationMs = stats.EndTime.Sub(stats.StartTime).Milliseconds()
 	stats.LastInterfaceType = run.InterfaceType
 
 	stats.TaskRuns++
 	stats.LastTaskName = run.TaskName
 	stats.LastRunID = run.RunID
+	if stats.LastRunID <= 0 {
+		stats.LastRunID = stats.TaskRuns
+	}
 
 	if run.Completed {
 		stats.TasksCompleted++
@@ -87,12 +101,22 @@ func (s *StatisticsService) RecordTaskRun(ctx context.Context, run models.TaskRu
 		stats.TasksFailed++
 	}
 
+	switch run.RunOutcome {
+	case models.RunOutcomeInfraError:
+		stats.InfraFailures++
+	case models.RunOutcomeUserQuit:
+		stats.UserQuits++
+	}
+
 	stats.TotalDurationMs += run.DurationMs
 	if stats.TaskRuns > 0 {
 		stats.AverageDurationMs = stats.TotalDurationMs / int64(stats.TaskRuns)
 	}
 
 	stats.ValidationAttempts += run.ValidationAttempts
+	stats.ValidationManualAttempts += run.ValidationManualAttempts
+	stats.ValidationAutoRefreshes += run.ValidationAutoRefreshes
+	stats.ValidationRefreshes += run.ValidationRefreshes
 	stats.ValidationSuccesses += run.ValidationSuccesses
 	stats.ValidationFailures += run.ValidationFailures
 	stats.ValidationChecksPassed += run.ValidationChecksPassed

internal/models/statistics.go

@@ -6,16 +6,32 @@ import (
 	"go.mongodb.org/mongo-driver/v2/bson"
 )
 
+const CurrentMetricsVersion = 3
+
+type RunOutcome string
+
+const (
+	RunOutcomeUnknown        RunOutcome = "unknown"
+	RunOutcomeCompleted      RunOutcome = "completed"
+	RunOutcomeUserIncomplete RunOutcome = "user_incomplete"
+	RunOutcomeInfraError     RunOutcome = "infra_error"
+	RunOutcomeUserQuit       RunOutcome = "user_quit"
+)
+
 type Statistics struct {
-	ID         bson.ObjectID `bson:"_id" json:"id"`
+	MetricsVersion      int           `bson:"metrics_version" json:"metrics_version"`
-	StartTime  time.Time     `bson:"start_time" json:"start_time"`
+	ID                  bson.ObjectID `bson:"_id" json:"id"`
-	EndTime    time.Time     `bson:"end_time" json:"end_time"`
+	StartTime           time.Time     `bson:"start_time" json:"start_time"`
-	DurationMs int64         `bson:"duration_ms" json:"duration_ms"`
+	EndTime             time.Time     `bson:"end_time" json:"end_time"`
+	DurationMs          int64         `bson:"duration_ms" json:"duration_ms"`
+	SessionClockAnomaly bool          `bson:"session_clock_anomaly" json:"session_clock_anomaly"`
 
 	LastInterfaceType InterfaceType `bson:"last_interface_type" json:"last_interface_type"`
 	TaskRuns          int           `bson:"task_runs" json:"task_runs"`
 	TasksCompleted    int           `bson:"tasks_completed" json:"tasks_completed"`
 	TasksFailed       int           `bson:"tasks_failed" json:"tasks_failed"`
+	InfraFailures     int           `bson:"infra_failures" json:"infra_failures"`
+	UserQuits         int           `bson:"user_quits" json:"user_quits"`
 
 	TotalDurationMs   int64 `bson:"total_duration_ms" json:"total_duration_ms"`
 	AverageDurationMs int64 `bson:"average_duration_ms" json:"average_duration_ms"`
@@ -27,64 +43,85 @@ type Statistics struct {
 	QuestionnairesCompleted int `bson:"questionnaires_completed" json:"questionnaires_completed"`
 	QuestionnairesAbandoned int `bson:"questionnaires_abandoned" json:"questionnaires_abandoned"`
 
-	ValidationAttempts     int `bson:"validation_attempts" json:"validation_attempts"`
+	ValidationAttempts       int `bson:"validation_attempts" json:"validation_attempts"`
-	ValidationSuccesses    int `bson:"validation_successes" json:"validation_successes"`
+	ValidationManualAttempts int `bson:"validation_manual_attempts" json:"validation_manual_attempts"`
-	ValidationFailures     int `bson:"validation_failures" json:"validation_failures"`
+	ValidationAutoRefreshes  int `bson:"validation_auto_refreshes" json:"validation_auto_refreshes"`
-	ValidationChecksPassed int `bson:"validation_checks_passed" json:"validation_checks_passed"`
+	ValidationRefreshes      int `bson:"validation_refreshes" json:"validation_refreshes"`
-	ValidationChecksFailed int `bson:"validation_checks_failed" json:"validation_checks_failed"`
+	ValidationSuccesses      int `bson:"validation_successes" json:"validation_successes"`
+	ValidationFailures       int `bson:"validation_failures" json:"validation_failures"`
+	ValidationChecksPassed   int `bson:"validation_checks_passed" json:"validation_checks_passed"`
+	ValidationChecksFailed   int `bson:"validation_checks_failed" json:"validation_checks_failed"`
 
 	LastTaskName string `bson:"last_task_name" json:"last_task_name"`
 	LastRunID    int    `bson:"last_run_id" json:"last_run_id"`
 }
 
 type TaskMetricsFile struct {
-	ID            bson.ObjectID    `bson:"_id" json:"id"`
+	MetricsVersion int              `bson:"metrics_version" json:"metrics_version"`
-	ParticipantID bson.ObjectID    `bson:"participant_id" json:"participant_id"`
+	ID             bson.ObjectID    `bson:"_id" json:"id"`
-	TaskName      string           `bson:"task_name" json:"task_name"`
+	ParticipantID  bson.ObjectID    `bson:"participant_id" json:"participant_id"`
-	UpdatedAt     time.Time        `bson:"updated_at" json:"updated_at"`
+	TaskName       string           `bson:"task_name" json:"task_name"`
-	Summary       TaskStatsSummary `bson:"summary" json:"summary"`
+	UpdatedAt      time.Time        `bson:"updated_at" json:"updated_at"`
-	Runs          []TaskRunMetrics `bson:"runs" json:"runs"`
+	Summary        TaskStatsSummary `bson:"summary" json:"summary"`
+	Runs           []TaskRunMetrics `bson:"runs" json:"runs"`
 }
 
 type TaskStatsSummary struct {
-	TotalRuns               int           `bson:"total_runs" json:"total_runs"`
+	TotalRuns                   int           `bson:"total_runs" json:"total_runs"`
-	CompletedRuns           int           `bson:"completed_runs" json:"completed_runs"`
+	CompletedRuns               int           `bson:"completed_runs" json:"completed_runs"`
-	IncompleteRuns          int           `bson:"incomplete_runs" json:"incomplete_runs"`
+	IncompleteRuns              int           `bson:"incomplete_runs" json:"incomplete_runs"`
-	TotalDurationMs         int64         `bson:"total_duration_ms" json:"total_duration_ms"`
+	TotalDurationMs             int64         `bson:"total_duration_ms" json:"total_duration_ms"`
-	AverageDurationMs       int64         `bson:"average_duration_ms" json:"average_duration_ms"`
+	AverageDurationMs           int64         `bson:"average_duration_ms" json:"average_duration_ms"`
-	TotalUserActions        int           `bson:"total_user_actions" json:"total_user_actions"`
+	TotalUserActions            int           `bson:"total_user_actions" json:"total_user_actions"`
-	QuestionnairesCompleted int           `bson:"questionnaires_completed" json:"questionnaires_completed"`
+	InfraFailures               int           `bson:"infra_failures" json:"infra_failures"`
-	QuestionnairesAbandoned int           `bson:"questionnaires_abandoned" json:"questionnaires_abandoned"`
+	UserQuits                   int           `bson:"user_quits" json:"user_quits"`
-	ValidationAttempts      int           `bson:"validation_attempts" json:"validation_attempts"`
+	QuestionnairesCompleted     int           `bson:"questionnaires_completed" json:"questionnaires_completed"`
-	ValidationSuccesses     int           `bson:"validation_successes" json:"validation_successes"`
+	QuestionnairesAbandoned     int           `bson:"questionnaires_abandoned" json:"questionnaires_abandoned"`
-	ValidationFailures      int           `bson:"validation_failures" json:"validation_failures"`
+	ValidationAttempts          int           `bson:"validation_attempts" json:"validation_attempts"`
-	ValidationChecksPassed  int           `bson:"validation_checks_passed" json:"validation_checks_passed"`
+	ValidationManualAttempts    int           `bson:"validation_manual_attempts" json:"validation_manual_attempts"`
-	ValidationChecksFailed  int           `bson:"validation_checks_failed" json:"validation_checks_failed"`
+	ValidationAutoRefreshes     int           `bson:"validation_auto_refreshes" json:"validation_auto_refreshes"`
-	LastInterfaceType       InterfaceType `bson:"last_interface_type" json:"last_interface_type"`
+	ValidationRefreshes         int           `bson:"validation_refreshes" json:"validation_refreshes"`
-	FirstRunStartedAt       time.Time     `bson:"first_run_started_at" json:"first_run_started_at"`
+	ValidationSuccesses         int           `bson:"validation_successes" json:"validation_successes"`
-	LastRunStartedAt        time.Time     `bson:"last_run_started_at" json:"last_run_started_at"`
+	ValidationFailures          int           `bson:"validation_failures" json:"validation_failures"`
-	LastRunEndedAt          time.Time     `bson:"last_run_ended_at" json:"last_run_ended_at"`
+	ValidationChecksPassed      int           `bson:"validation_checks_passed" json:"validation_checks_passed"`
+	ValidationChecksFailed      int           `bson:"validation_checks_failed" json:"validation_checks_failed"`
+	RunsWithFirstSuccess        int           `bson:"runs_with_first_success" json:"runs_with_first_success"`
+	TotalAttemptsToFirstSuccess int           `bson:"total_attempts_to_first_success" json:"total_attempts_to_first_success"`
+	TotalTimeToFirstSuccessMs   int64         `bson:"total_time_to_first_success_ms" json:"total_time_to_first_success_ms"`
+	LastInterfaceType           InterfaceType `bson:"last_interface_type" json:"last_interface_type"`
+	FirstRunStartedAt           time.Time     `bson:"first_run_started_at" json:"first_run_started_at"`
+	LastRunStartedAt            time.Time     `bson:"last_run_started_at" json:"last_run_started_at"`
+	LastRunEndedAt              time.Time     `bson:"last_run_ended_at" json:"last_run_ended_at"`
 }
 
 type TaskRunMetrics struct {
-	RunID                  int            `bson:"run_id" json:"run_id"`
+	MetricsVersion           int                     `bson:"metrics_version" json:"metrics_version"`
-	TaskName               string         `bson:"task_name" json:"task_name"`
+	RunID                    int                     `bson:"run_id" json:"run_id"`
-	InterfaceType          InterfaceType  `bson:"interface_type" json:"interface_type"`
+	TaskName                 string                  `bson:"task_name" json:"task_name"`
-	StartedAt              time.Time      `bson:"started_at" json:"started_at"`
+	InterfaceType            InterfaceType           `bson:"interface_type" json:"interface_type"`
-	EndedAt                time.Time      `bson:"ended_at" json:"ended_at"`
+	StartedAt                time.Time               `bson:"started_at" json:"started_at"`
-	DurationMs             int64          `bson:"duration_ms" json:"duration_ms"`
+	EndedAt                  time.Time               `bson:"ended_at" json:"ended_at"`
-	Completed              bool           `bson:"completed" json:"completed"`
+	DurationMs               int64                   `bson:"duration_ms" json:"duration_ms"`
-	ValidationAttempts     int            `bson:"validation_attempts" json:"validation_attempts"`
+	Completed                bool                    `bson:"completed" json:"completed"`
-	ValidationSuccesses    int            `bson:"validation_successes" json:"validation_successes"`
+	ValidationAttempts       int                     `bson:"validation_attempts" json:"validation_attempts"`
-	ValidationFailures     int            `bson:"validation_failures" json:"validation_failures"`
+	ValidationManualAttempts int                     `bson:"validation_manual_attempts" json:"validation_manual_attempts"`
-	ValidationChecksPassed int            `bson:"validation_checks_passed" json:"validation_checks_passed"`
+	ValidationAutoRefreshes  int                     `bson:"validation_auto_refreshes" json:"validation_auto_refreshes"`
-	ValidationChecksFailed int            `bson:"validation_checks_failed" json:"validation_checks_failed"`
+	ValidationRefreshes      int                     `bson:"validation_refreshes" json:"validation_refreshes"`
-	LastValidationMessage  string         `bson:"last_validation_message,omitempty" json:"last_validation_message,omitempty"`
+	ValidationSuccesses      int                     `bson:"validation_successes" json:"validation_successes"`
-	QuestionnaireCompleted bool           `bson:"questionnaire_completed" json:"questionnaire_completed"`
+	ValidationFailures       int                     `bson:"validation_failures" json:"validation_failures"`
-	QuestionnaireUserQuit  bool           `bson:"questionnaire_user_quit" json:"questionnaire_user_quit"`
+	ValidationChecksPassed   int                     `bson:"validation_checks_passed" json:"validation_checks_passed"`
-	QuestionnaireAnswers   map[string]any `bson:"questionnaire_answers,omitempty" json:"questionnaire_answers,omitempty"`
+	ValidationChecksFailed   int                     `bson:"validation_checks_failed" json:"validation_checks_failed"`
-	Logs                   []TaskLogEntry `bson:"logs" json:"logs"`
+	ValidationSource         ValidationTriggerSource `bson:"validation_source" json:"validation_source"`
-	Error                  string         `bson:"error,omitempty" json:"error,omitempty"`
+	AttemptsToFirstSuccess   int                     `bson:"attempts_to_first_success" json:"attempts_to_first_success"`
+	TimeToFirstSuccessMs     int64                   `bson:"time_to_first_success_ms" json:"time_to_first_success_ms"`
+	FailureReasonCode        string                  `bson:"failure_reason_code,omitempty" json:"failure_reason_code,omitempty"`
+	LastValidationMessage    string                  `bson:"last_validation_message,omitempty" json:"last_validation_message,omitempty"`
+	QuestionnaireCompleted   bool                    `bson:"questionnaire_completed" json:"questionnaire_completed"`
+	QuestionnaireUserQuit    bool                    `bson:"questionnaire_user_quit" json:"questionnaire_user_quit"`
+	RunOutcome               RunOutcome              `bson:"run_outcome" json:"run_outcome"`
+	QuestionnaireAnswers     map[string]any          `bson:"questionnaire_answers,omitempty" json:"questionnaire_answers,omitempty"`
+	Logs                     []TaskLogEntry          `bson:"logs" json:"logs"`
+	Error                    string                  `bson:"error,omitempty" json:"error,omitempty"`
 }
 
 type TaskLogEntry struct {

pkg/task/metrics_store.go

@@ -44,10 +44,11 @@ func (s *FileMetricsStore) Append(ctx context.Context, taskName string, run mode
 	}
 
 	metrics := models.TaskMetricsFile{
-		ID:            bson.NewObjectID(),
+		MetricsVersion: models.CurrentMetricsVersion,
-		ParticipantID: s.participantID,
+		ID:             bson.NewObjectID(),
-		TaskName:      taskName,
+		ParticipantID:  s.participantID,
-		Runs:          []models.TaskRunMetrics{},
+		TaskName:       taskName,
+		Runs:           []models.TaskRunMetrics{},
 	}
 
 	if err := readMetrics(&metrics, s.path); err != nil {
@@ -57,6 +58,12 @@ func (s *FileMetricsStore) Append(ctx context.Context, taskName string, run mode
 	if metrics.TaskName == "" {
 		metrics.TaskName = taskName
 	}
+	if metrics.MetricsVersion == 0 {
+		metrics.MetricsVersion = models.CurrentMetricsVersion
+	}
+	if run.MetricsVersion == 0 {
+		run.MetricsVersion = models.CurrentMetricsVersion
+	}
 
 	run.RunID = len(metrics.Runs) + 1
 	metrics.Runs = append(metrics.Runs, run)

pkg/task/metrics_summary.go

@@ -39,12 +39,27 @@ func buildTaskStatsSummary(runs []models.TaskRunMetrics) models.TaskStatsSummary
 			summary.IncompleteRuns++
 		}
 
+		switch run.RunOutcome {
+		case models.RunOutcomeInfraError:
+			summary.InfraFailures++
+		case models.RunOutcomeUserQuit:
+			summary.UserQuits++
+		}
+
 		// Validation
 		summary.ValidationAttempts += run.ValidationAttempts
+		summary.ValidationManualAttempts += run.ValidationManualAttempts
+		summary.ValidationAutoRefreshes += run.ValidationAutoRefreshes
+		summary.ValidationRefreshes += run.ValidationRefreshes
 		summary.ValidationSuccesses += run.ValidationSuccesses
 		summary.ValidationFailures += run.ValidationFailures
 		summary.ValidationChecksPassed += run.ValidationChecksPassed
 		summary.ValidationChecksFailed += run.ValidationChecksFailed
+		if run.AttemptsToFirstSuccess > 0 {
+			summary.RunsWithFirstSuccess++
+			summary.TotalAttemptsToFirstSuccess += run.AttemptsToFirstSuccess
+			summary.TotalTimeToFirstSuccessMs += run.TimeToFirstSuccessMs
+		}
 
 		// Questionnaire
 		if run.QuestionnaireCompleted {