Evaluation

ID

b8115ab8-5d40-4cae-a291-f56066a462e6

Submission ID

3692399b-a4eb-4009-8eb6-a1551f19ad75

Method ID

3682a853-b9d2-4aae-a49d-851f8377c821

Status

Succeeded

User

 lved 

Challenge

FRAME

Phase

FRAME Track - Pre-evaluation phase

Algorithm

VLM baseline (Image Version 3d7eaca9-6504-44dd-ba9b-ec53c2b1c607 )

Submission created

Aug. 8, 2026, 12:45 p.m.

Result created

Aug. 8, 2026, 12:45 p.m.

Position on leaderboard

51

Metrics

{
  "_docs": {
    "budget.batches": "Number of algorithm jobs (batches) whose answers were scored -- the population behind the latency figures above. A job that produced no answer.json is not counted here; its questions appear in questions_unanswered instead.",
    "budget.questions": "Number of questions in those batches, taken from the shipped batch composition rather than from the answers actually returned, so a container that drops answers cannot shrink its own budget.",
    "questions_forfeited": "Questions scored incorrect for latency. The budget is pooled per batch, so these were not individually slow -- they were selected deterministically, stratified across the 10 buckets, in proportion to how far the batch ran over its budget. 0 means the batch fit.",
    "pre_evaluation_score": "Headline ranking metric. Unweighted mean accuracy over the 10 buckets (5 capability groups x in-/out-of-distribution), so every bucket counts equally regardless of how many questions it holds. A forfeited or unanswered question counts as incorrect.",
    "questions_unanswered": "Ground-truth questions with no response at all (the container dropped them, or its job produced no answer.json). Scored incorrect. Disjoint from questions_forfeited by construction.",
    "mean_batch_duration_s": "Mean platform-measured duration of one JOB -- a whole batch of B questions, including model loading. This is NOT a per-question latency and must not be compared to the per-question budget; see mean_latency_per_question_s for that. Averaged over batches that produced answers and carried a measurement.",
    "accuracy_aggregation_id": "Accuracy on in-distribution aggregation questions. One of the 10 buckets averaged into pre_evaluation_score; null if the ground truth holds no such question.",
    "accuracy_aggregation_ood": "Accuracy on out-of-distribution aggregation questions. One of the 10 buckets averaged into pre_evaluation_score; null if the ground truth holds no such question.",
    "budget.setup_allowance_s": "One-time per-batch allowance for interpreter start, imports and model loading, granted before any per-question budget is charged. Amortizing this cost is why questions are batched at all.",
    "budget.saturation_fraction": "Relative overrun at which the entire batch is forfeited. Exceeding the allowed time by this fraction costs every question in the batch; the penalty grades linearly up to that point.",
    "throughput_questions_per_s": "Wall-clock questions per second: questions divided by the FULL job duration, setup time included. Deliberately not 1 / mean_latency_per_question_s, which is net of the setup allowance.",
    "mean_latency_per_question_s": "Mean processing time per QUESTION: total measured job time minus the one-time setup allowance, divided by the number of questions. Weighted by batch size, and batches that finished inside the setup allowance count as 0 rather than being dropped. This is the figure comparable to budget.latency_per_question_s.",
    "accuracy_complex_reasoning_id": "Accuracy on in-distribution complex reasoning questions. One of the 10 buckets averaged into pre_evaluation_score; null if the ground truth holds no such question.",
    "budget.latency_per_question_s": "Per-question processing budget X for this track, on top of the setup allowance. A batch of B questions is allowed setup_allowance_s + B * X seconds in total.",
    "accuracy_complex_reasoning_ood": "Accuracy on out-of-distribution complex reasoning questions. One of the 10 buckets averaged into pre_evaluation_score; null if the ground truth holds no such question.",
    "accuracy_object_recognition_id": "Accuracy on in-distribution object recognition questions. One of the 10 buckets averaged into pre_evaluation_score; null if the ground truth holds no such question.",
    "accuracy_temporal_grounding_id": "Accuracy on in-distribution temporal grounding questions. One of the 10 buckets averaged into pre_evaluation_score; null if the ground truth holds no such question.",
    "accuracy_event_understanding_id": "Accuracy on in-distribution event understanding questions. One of the 10 buckets averaged into pre_evaluation_score; null if the ground truth holds no such question.",
    "accuracy_object_recognition_ood": "Accuracy on out-of-distribution object recognition questions. One of the 10 buckets averaged into pre_evaluation_score; null if the ground truth holds no such question.",
    "accuracy_temporal_grounding_ood": "Accuracy on out-of-distribution temporal grounding questions. One of the 10 buckets averaged into pre_evaluation_score; null if the ground truth holds no such question.",
    "accuracy_event_understanding_ood": "Accuracy on out-of-distribution event understanding questions. One of the 10 buckets averaged into pre_evaluation_score; null if the ground truth holds no such question."
  },
  "budget": {
    "batches": 100,
    "questions": 2000,
    "setup_allowance_s": 120.0,
    "saturation_fraction": 0.2,
    "latency_per_question_s": 5.0
  },
  "questions_forfeited": 0,
  "pre_evaluation_score": 0.5168383851849065,
  "questions_unanswered": 0,
  "mean_batch_duration_s": 30.071844470000006,
  "accuracy_aggregation_id": 0.46835443037974683,
  "accuracy_aggregation_ood": 0.5638297872340425,
  "throughput_questions_per_s": 0.6650739371824105,
  "mean_latency_per_question_s": 0.0,
  "accuracy_complex_reasoning_id": null,
  "accuracy_complex_reasoning_ood": null,
  "accuracy_object_recognition_id": 0.5957161981258366,
  "accuracy_temporal_grounding_id": null,
  "accuracy_event_understanding_id": null,
  "accuracy_object_recognition_ood": 0.439453125,
  "accuracy_temporal_grounding_ood": null,
  "accuracy_event_understanding_ood": null
}