feat: Blender 5.x API fixes + planner edit awareness

app/api/ai/chat/route.ts

@@ -24,6 +24,8 @@ import type {
 import { recordExecutionLog } from "@/lib/orchestration/monitor"
 import { buildSystemPrompt } from "@/lib/orchestration/prompts"
 import { searchFirecrawl, type FirecrawlSearchResult } from "@/lib/firecrawl"
+import { classifyStrategy } from "@/lib/orchestration/strategy-router"
+import { generateWorkflowProposal } from "@/lib/orchestration/workflow-advisor"
 import { z } from "zod"
 
 const MAX_HISTORY_MESSAGES = 12
@@ -148,45 +150,40 @@ function formatSceneSnapshot(payload: unknown): string | null {
   const objectCount = typeof scene.object_count === "number" ? scene.object_count : undefined
   const errorMessage = typeof scene.error === "string" ? (scene.error as string) : undefined
 
-  const objectList = Array.isArray(scene.objects) ? scene.objects.slice(0, 12) : []
-  const objects = objectList
-    .map((raw) => {
-      if (!raw || typeof raw !== "object") {
-        return "- (unknown object)"
+  const objectList = Array.isArray(scene.objects) ? scene.objects.slice(0, 30) : []
+  const structuredObjects = objectList
+    .filter((raw): raw is Record<string, unknown> => !!raw && typeof raw === "object")
+    .map((obj) => {
+      const entry: Record<string, unknown> = {
+        name: typeof obj.name === "string" ? obj.name : "(unnamed)",
+        type: typeof obj.type === "string" ? obj.type : "UNKNOWN",
       }
-      const obj = raw as Record<string, unknown>
-      const identifier = typeof obj.name === "string" ? obj.name : "(unnamed)"
-      const type = typeof obj.type === "string" ? obj.type : "UNKNOWN"
-      const locationArray = Array.isArray(obj.location) ? obj.location : []
-      const location = locationArray
-        .slice(0, 3)
-        .map((value) => (typeof value === "number" ? value.toFixed(2) : "?"))
-        .join(", ")
-      return `- ${identifier} [${type}] @ (${location})`
+      if (Array.isArray(obj.location)) {
+        entry.location = obj.location.slice(0, 3).map((v) => typeof v === "number" ? Math.round(v * 100) / 100 : 0)
+      }
+      if (Array.isArray(obj.dimensions)) {
+        entry.dimensions = obj.dimensions.slice(0, 3).map((v) => typeof v === "number" ? Math.round(v * 100) / 100 : 0)
+      }
+      return entry
     })
-    .join("\n")
 
-  const materials =
+  const materialsCount =
     typeof scene.materials_count === "number"
-      ? `Materials: ${scene.materials_count}`
+      ? scene.materials_count
       : undefined
 
-  let summary = `Scene: ${name}`
-  if (typeof objectCount === "number") {
-    summary += ` | Objects: ${objectCount}`
-  }
-  if (materials) {
-    summary += ` | ${materials}`
+  // Return structured JSON so the model can parse object names and locations precisely
+  const snapshot: Record<string, unknown> = {
+    scene: name,
+    object_count: objectCount,
+    materials_count: materialsCount,
+    objects: structuredObjects,
   }
   if (errorMessage) {
-    summary += ` | Error: ${errorMessage}`
+    snapshot.error = errorMessage
   }
 
-  if (objects) {
-    summary += `\nObjects:\n${objects}`
-  }
-
-  return summary
+  return JSON.stringify(snapshot, null, 2)
 }
 
 
@@ -501,94 +498,158 @@ export async function POST(req: Request) {
 
           const sceneSnapshotResult = await fetchSceneSummary()
 
+          // Strategy classification: determine procedural vs neural vs hybrid
+          const strategyDecision = await classifyStrategy(message, {
+            sceneContext: sceneSnapshotResult.summary ?? undefined,
+          })
+          send({
+            type: "agent:strategy_classification",
+            timestamp: new Date().toISOString(),
+            strategy: strategyDecision.strategy,
+            confidence: strategyDecision.confidence,
+            reasoning: strategyDecision.reasoning,
+            method: strategyDecision.classificationMethod,
+          })
+
           let executedCommands: ExecutedCommand[] = []
           let planningMetadata: PlanningMetadata | null = null
           let executionLogs: ExecutionLogEntry[] | undefined = undefined
           let planResult: PlanGenerationResult | null = null
 
-          try {
-            planResult = await planner.generatePlan(
-              message,
-              {
-                sceneSummary: sceneSnapshotResult.summary ?? undefined,
-                allowHyper3dAssets: assetConfig.allowHyper3d,
-                allowSketchfabAssets: assetConfig.allowSketchfab,
-                allowPolyHavenAssets: assetConfig.allowPolyHaven,
-                researchContext: researchContext?.promptContext,
-              },
-              llmProvider
-            )
+          // ── Neural/Hybrid → Guided Workflow (human-in-the-loop) ──
+          if (strategyDecision.strategy === "neural" || strategyDecision.strategy === "hybrid") {
+            try {
+              const workflowProposal = await generateWorkflowProposal(
+                message,
+                strategyDecision.strategy,
+                { sceneContext: sceneSnapshotResult.summary ?? undefined }
+              )
+
+              // Send the workflow proposal to the UI
+              send({
+                type: "agent:workflow_proposal",
+                timestamp: new Date().toISOString(),
+                proposal: workflowProposal,
+              })
+
+              // Build a lightweight planningMetadata for the conversation record
+              planningMetadata = {
+                planSummary: `Workflow proposed: ${workflowProposal.title} (${workflowProposal.steps.length} steps). Awaiting user action on each step.`,
+                planSteps: workflowProposal.steps.map((s) => ({
+                  stepNumber: s.stepNumber,
+                  action: s.recommendedTool === "neural" ? "neural_generate" : s.recommendedTool === "manual" ? "manual" : "execute_code",
+                  parameters: {
+                    category: s.category,
+                    tool: s.recommendedTool,
+                    neuralProvider: s.neuralProvider,
+                    workflowStepId: s.id,
+                  },
+                  rationale: s.toolReasoning,
+                  expectedOutcome: s.description,
+                })),
+                rawPlan: JSON.stringify(workflowProposal, null, 2),
+                retries: 0,
+                executionSuccess: true, // Proposal was successfully generated
+                sceneSnapshot: sceneSnapshotResult.summary,
+                strategyDecision,
+              }
+            } catch (workflowError) {
+              console.error("Workflow proposal failed, falling back to planner:", workflowError)
+              // Fall through to the procedural planner+executor below
+            }
+          }
 
-            if (planResult && planResult.plan) {
-              const executionResult = await planExecutor.executePlan(
-                planResult.plan,
+          // ── Procedural → Auto-pilot planner + executor (existing flow) ──
+          if (!planningMetadata) {
+
+            try {
+              planResult = await planner.generatePlan(
                 message,
                 {
-                  ...assetConfig,
-                  onStreamEvent: (event) => send(event),
+                  sceneSummary: sceneSnapshotResult.summary ?? undefined,
+                  allowHyper3dAssets: assetConfig.allowHyper3d,
+                  allowSketchfabAssets: assetConfig.allowSketchfab,
+                  allowPolyHavenAssets: assetConfig.allowPolyHaven,
+                  researchContext: researchContext?.promptContext,
+                  strategyDecision,
                 },
-                planResult.analysis,
                 llmProvider
               )
-              executionLogs = executionResult.logs
-              executedCommands = buildExecutedCommandsFromPlan(planResult.plan, executionResult)
-              planningMetadata = {
-                planSummary: planResult.plan.planSummary,
-                planSteps: planResult.plan.steps,
-                rawPlan: planResult.rawResponse,
-                retries: planResult.retries ?? 0,
-                executionSuccess: executionResult.success,
-                errors: planResult.errors,
-                executionLog: executionResult.logs,
+
+              if (planResult && planResult.plan) {
+                const executionResult = await planExecutor.executePlan(
+                  planResult.plan,
+                  message,
+                  {
+                    ...assetConfig,
+                    enableVisualFeedback: true,
+                    onStreamEvent: (event) => send(event),
+                    strategyDecision,
+                  },
+                  planResult.analysis,
+                  llmProvider
+                )
+                executionLogs = executionResult.logs
+                executedCommands = buildExecutedCommandsFromPlan(planResult.plan, executionResult)
+                planningMetadata = {
+                  planSummary: planResult.plan.planSummary,
+                  planSteps: planResult.plan.steps,
+                  rawPlan: planResult.rawResponse,
+                  retries: planResult.retries ?? 0,
+                  executionSuccess: executionResult.success,
+                  errors: planResult.errors,
+                  executionLog: executionResult.logs,
+                  sceneSnapshot: sceneSnapshotResult.summary,
+                  analysis: planResult.analysis,
+                  researchSummary: researchContext?.promptContext,
+                  researchSources: researchContext?.sources,
+                  strategyDecision,
+                }
+
+                if (!executionResult.success) {
+                  planningMetadata.executionSuccess = false
+                }
+              } else if (planResult) {
+                const previousLogs = executionLogs
+                planningMetadata = {
+                  planSummary: "Plan generation failed",
+                  planSteps: [],
+                  rawPlan: planResult.rawResponse,
+                  retries: planResult.retries ?? 0,
+                  executionSuccess: false,
+                  errors: planResult.errors,
+                  fallbackUsed: false,
+                  executionLog: previousLogs,
+                  sceneSnapshot: sceneSnapshotResult.summary,
+                  analysis: planResult.analysis,
+                  researchSummary: researchContext?.promptContext,
+                  researchSources: researchContext?.sources,
+                }
+                executedCommands = []
+              } else {
+                throw new Error("Planner returned no result")
+              }
+            } catch (error) {
+              console.error("Planning pipeline error:", error)
+              const messageText =
+                error instanceof Error ? error.message : "Unknown planning error"
+              planningMetadata = planningMetadata ?? {
+                planSummary: "Planner error",
+                planSteps: [],
+                rawPlan: "",
+                retries: 0,
+                executionSuccess: false,
+                errors: [messageText],
+                fallbackUsed: false,
+                executionLog: executionLogs,
                 sceneSnapshot: sceneSnapshotResult.summary,
-                analysis: planResult.analysis,
+                analysis: planResult?.analysis,
                 researchSummary: researchContext?.promptContext,
                 researchSources: researchContext?.sources,
               }
-
-              if (!executionResult.success) {
-                planningMetadata.executionSuccess = false
-              }
-            } else if (planResult) {
-            const previousLogs = executionLogs
-            planningMetadata = {
-              planSummary: "Plan generation failed",
-              planSteps: [],
-              rawPlan: planResult.rawResponse,
-              retries: planResult.retries ?? 0,
-              executionSuccess: false,
-              errors: planResult.errors,
-              fallbackUsed: false,
-              executionLog: previousLogs,
-              sceneSnapshot: sceneSnapshotResult.summary,
-              analysis: planResult.analysis,
-              researchSummary: researchContext?.promptContext,
-              researchSources: researchContext?.sources,
-            }
               executedCommands = []
-            } else {
-              throw new Error("Planner returned no result")
             }
-          } catch (error) {
-            console.error("Planning pipeline error:", error)
-            const messageText =
-              error instanceof Error ? error.message : "Unknown planning error"
-            planningMetadata = planningMetadata ?? {
-              planSummary: "Planner error",
-              planSteps: [],
-              rawPlan: "",
-              retries: 0,
-              executionSuccess: false,
-              errors: [messageText],
-              fallbackUsed: false,
-              executionLog: executionLogs,
-              sceneSnapshot: sceneSnapshotResult.summary,
-              analysis: planResult?.analysis,
-              researchSummary: researchContext?.promptContext,
-              researchSources: researchContext?.sources,
-            }
-            executedCommands = []
-          }
+          } // ← closing brace for procedural fallback block