{
  "schema": "cad-agent-catalog-area/1",
  "origin": "https://cad-agent.valpatel.com",
  "revision": "f766bf2694e3e66e1ac84c901be1091bdb9863da",
  "fields": {
    "level": "domain, area or bucket",
    "domain": "areas: the domain it belongs to",
    "priority": "demo (a good demo)",
    "projects": "areas: keys of the projects in docs/areas.yaml (areas_source) that used it",
    "issues": "GitHub issues for work in this area",
    "topics": "areas: its topics, with scope, maturity and issues",
    "areas": "domains: its areas",
    "count": "entries here",
    "kinds": "count per kind",
    "items": "every entry placed here (a domain holds its areas' entries): the fields of catalog.json without summary and related"
  },
  "item_fields": "https://cad-agent.valpatel.com/catalog.json (fields)",
  "slug": "evaluation",
  "title": "Evaluation",
  "scope": "Standard jobs and graders that measure models, harnesses and our tooling.",
  "maturity": "idea",
  "level": "area",
  "domain": "practice",
  "priority": [],
  "projects": [],
  "issues": [
    "https://github.com/Valpatel/cad-agent/issues/39",
    "https://github.com/Valpatel/cad-agent/issues/142",
    "https://github.com/Valpatel/cad-agent/issues/143"
  ],
  "topics": [],
  "count": 13,
  "kinds": {
    "benchmark": 13
  },
  "page": "https://cad-agent.valpatel.com/catalog?area=evaluation",
  "index": "https://cad-agent.valpatel.com/catalog/index.json",
  "items": [
    {
      "type": "benchmark",
      "status": "reference",
      "name": "PCBWorld",
      "url": "https://github.com/LGAI-Research/PCBWorld",
      "license": "BSD-3-Clause",
      "id": "benchmark-pcbworld",
      "kind": "benchmark",
      "areas": [
        "evaluation",
        "circuit-boards"
      ],
      "maturity": "tooling",
      "what": "KiCad push-and-shove router as a Gymnasium environment, with 679 real boards and DRC-checked metrics",
      "notes": null,
      "cover": "https://cad-agent.valpatel.com/catalog/covers/benchmark-pcbworld.svg",
      "demo": null,
      "licence_family": "permissive",
      "updated": null
    },
    {
      "type": "benchmark",
      "status": "reference",
      "name": "FEABench",
      "url": "https://github.com/google/feabench",
      "license": "Apache-2.0",
      "id": "benchmark-feabench",
      "kind": "benchmark",
      "areas": [
        "evaluation",
        "structural-fea"
      ],
      "maturity": "tooling",
      "what": "Benchmark of FEA problems that LLM agents solve by driving COMSOL Multiphysics through its API",
      "notes": "LLM agents on FEA problems; needs a COMSOL licence; prior art only; the repo is archived",
      "cover": "https://cad-agent.valpatel.com/catalog/covers/benchmark-feabench.svg",
      "demo": null,
      "licence_family": "permissive",
      "updated": null
    },
    {
      "type": "benchmark",
      "status": "reference",
      "name": "CADGenBench",
      "url": "https://github.com/huggingface/cadgenbench",
      "license": "Apache-2.0",
      "stars": 146,
      "id": "benchmark-cadgenbench",
      "kind": "benchmark",
      "areas": [
        "evaluation"
      ],
      "maturity": "idea",
      "what": "Benchmark that scores AI-made STEP parts from engineering drawings and STEP edits on a hosted leaderboard",
      "notes": "Ground truth: private.",
      "cover": "https://cad-agent.valpatel.com/catalog/covers/benchmark-cadgenbench.svg",
      "demo": null,
      "licence_family": "permissive",
      "updated": null
    },
    {
      "type": "benchmark",
      "status": "reference",
      "name": "Text2CAD",
      "url": "https://github.com/SadilKhan/Text2CAD",
      "license": "CC-BY-NC-SA-4.0",
      "id": "benchmark-text2cad",
      "kind": "benchmark",
      "areas": [
        "evaluation"
      ],
      "maturity": "idea",
      "what": "Beginner-to-expert text prompts for DeepCAD models, and a model that generates CAD construction sequences",
      "notes": null,
      "cover": "https://cad-agent.valpatel.com/catalog/covers/benchmark-text2cad.svg",
      "demo": null,
      "licence_family": "other",
      "updated": null
    },
    {
      "type": "benchmark",
      "status": "reference",
      "name": "CADPrompt/CADCodeVerify",
      "url": "https://github.com/Kamel773/CAD_Code_Generation",
      "license": "none",
      "id": "benchmark-cadprompt-cadcodeverify",
      "kind": "benchmark",
      "areas": [
        "evaluation"
      ],
      "maturity": "idea",
      "what": "200 text prompts with expert CAD code and STL targets, and a VLM loop that checks and fixes generated code",
      "notes": "Licence: no licence file (GitHub detects none, checked 2026-11-01): all rights reserved, ideas only.",
      "cover": "https://cad-agent.valpatel.com/catalog/covers/benchmark-cadprompt-cadcodeverify.svg",
      "demo": null,
      "licence_family": "other",
      "updated": null
    },
    {
      "type": "benchmark",
      "status": "reference",
      "name": "CAD-Recode",
      "url": "https://github.com/filaPro/cad-recode",
      "license": "CC-BY-NC-4.0",
      "id": "benchmark-cad-recode",
      "kind": "benchmark",
      "areas": [
        "evaluation"
      ],
      "maturity": "idea",
      "what": "Model that reverse-engineers point clouds into CadQuery Python code, fine-tuned from Qwen2-1.5B",
      "notes": null,
      "cover": "https://cad-agent.valpatel.com/catalog/covers/benchmark-cad-recode.svg",
      "demo": null,
      "licence_family": "other",
      "updated": null
    },
    {
      "type": "benchmark",
      "status": "reference",
      "name": "cadrille",
      "url": "https://github.com/col14m/cadrille",
      "license": "Apache-2.0",
      "id": "benchmark-cadrille",
      "kind": "benchmark",
      "areas": [
        "evaluation"
      ],
      "maturity": "idea",
      "what": "Model that reconstructs CAD code from point clouds, images or text, trained with online RL",
      "notes": null,
      "cover": "https://cad-agent.valpatel.com/catalog/covers/benchmark-cadrille.svg",
      "demo": null,
      "licence_family": "permissive",
      "updated": null
    },
    {
      "type": "benchmark",
      "status": "reference",
      "name": "Text2CAD-Bench",
      "url": "https://arxiv.org/abs/2605.18430",
      "id": "benchmark-text2cad-bench",
      "kind": "benchmark",
      "areas": [
        "evaluation"
      ],
      "maturity": "idea",
      "what": "600 curated text-to-CAD tasks in four levels, from basic features to freeform and non-mechanical parts",
      "notes": "Data licence: unverified.",
      "cover": "https://cad-agent.valpatel.com/catalog/covers/benchmark-text2cad-bench.svg",
      "demo": null,
      "licence_family": "other",
      "updated": null
    },
    {
      "type": "benchmark",
      "status": "unverified",
      "name": "P3D-Bench",
      "url": "https://arxiv.org/abs/2606.11152",
      "id": "benchmark-p3d-bench",
      "kind": "benchmark",
      "areas": [
        "evaluation"
      ],
      "maturity": "idea",
      "what": "Text-to-3D, image-to-3D and assembly tasks for parametric 3D programs, scored on geometry and structure",
      "notes": "Listed from its paper or site; tasks, data and licence not yet checked.",
      "cover": "https://cad-agent.valpatel.com/catalog/covers/benchmark-p3d-bench.svg",
      "demo": null,
      "licence_family": "other",
      "updated": null
    },
    {
      "type": "benchmark",
      "status": "unverified",
      "name": "CADBench",
      "url": "https://arxiv.org/abs/2605.10873",
      "id": "benchmark-cadbench",
      "kind": "benchmark",
      "areas": [
        "evaluation"
      ],
      "maturity": "idea",
      "what": "18,000 samples testing CAD program generation from meshes and renders across six families and six metrics",
      "notes": "Listed from its paper or site; tasks, data and licence not yet checked.",
      "cover": "https://cad-agent.valpatel.com/catalog/covers/benchmark-cadbench.svg",
      "demo": null,
      "licence_family": "other",
      "updated": null
    },
    {
      "type": "benchmark",
      "status": "unverified",
      "name": "BenchCAD",
      "url": "https://arxiv.org/abs/2605.10865",
      "id": "benchmark-benchcad",
      "kind": "benchmark",
      "areas": [
        "evaluation"
      ],
      "maturity": "idea",
      "what": "17,900 checked CadQuery programs over 106 industrial part families, for CAD QA, image-to-code and editing",
      "notes": "Listed from its paper or site; tasks, data and licence not yet checked.",
      "cover": "https://cad-agent.valpatel.com/catalog/covers/benchmark-benchcad.svg",
      "demo": null,
      "licence_family": "other",
      "updated": null
    },
    {
      "type": "benchmark",
      "status": "unverified",
      "name": "RealCADBench",
      "url": "https://arxiv.org/abs/2609.03773",
      "id": "benchmark-realcadbench",
      "kind": "benchmark",
      "areas": [
        "evaluation"
      ],
      "maturity": "idea",
      "what": "12,632 tasks from real factory-automation design intents; models write FreeCAD Python for parts and assemblies",
      "notes": "Listed from its paper or site; tasks, data and licence not yet checked.",
      "cover": "https://cad-agent.valpatel.com/catalog/covers/benchmark-realcadbench.svg",
      "demo": null,
      "licence_family": "other",
      "updated": null
    },
    {
      "type": "benchmark",
      "status": "unverified",
      "name": "CAD Arena",
      "url": "https://cadarena.dev/",
      "license": "none",
      "id": "benchmark-cad-arena",
      "kind": "benchmark",
      "areas": [
        "evaluation"
      ],
      "maturity": "idea",
      "what": "Side-by-side comparison of text-to-CAD models on 20 prompts in four difficulty tiers, with 3D outputs",
      "notes": "Licence: a website; its home page states no licence or terms (checked 2026-11-01): ideas only. Listed from its paper or site; tasks and data not yet checked.",
      "cover": "https://cad-agent.valpatel.com/catalog/covers/benchmark-cad-arena.svg",
      "demo": null,
      "licence_family": "other",
      "updated": null
    }
  ]
}