"""
Askme Routes
Endpoints for Askme AI data assistant operations
"""
from fastapi import APIRouter, HTTPException, Header, Depends, UploadFile, File, Form
from fastapi.responses import JSONResponse
from fastapi.concurrency import run_in_threadpool
from fastapi.exceptions import RequestValidationError
from pydantic import BaseModel, Field, ConfigDict, ValidationError, field_validator
from typing import Optional, List, Dict, Any, Union, Tuple
from datetime import datetime, date
from sqlalchemy.orm import Session
import html as html_module
import json
import re
import ast
from types import SimpleNamespace
from app.services.askme_service import (
    get_askme_service,
    build_multi_agent_datasource_map,
    classify_multi_agent_strategy,
    execute_mixed_multi_agent_sync,
    normalize_request_datasource_override,
)
from app.services.multi_agent_pipeline import multi_agent_pipeline, merge_results
from app.services.llm_service import get_global_usage_totals, reset_global_usage_totals, _qwen_configured
import pandas as pd
import numpy as np
import plotly.graph_objects as go
import plotly.express as px

from app.services.agent_config import get_agent_config, resolve_file_capable_agent_id_for_user
from app.core.config import settings, get_base_url
from app.core.database import get_db, get_db_session
from app.core.dependencies import get_current_user
from app.routes.auth import get_user_agents
from askme_open_agent.exceptions import QueryTimeoutError

from app.models.agent import Training, AgentAdmin
from app.models.question import UserQuestionLog, Question
from app.models.user import User

from sqlalchemy import text, cast, Integer, or_
from textwrap import dedent

import asyncio
import functools
import json
import shutil
import os
import threading
import tempfile
import io
from concurrent.futures import ThreadPoolExecutor
from faster_whisper import WhisperModel

from app.services.talkback_orchestrator import (
    decide_uploaded_file_followup_action,
    run_talkback_pipeline,
    _resolve_talkback_effective_session_key,
    talkback_client_session_key_is_empty,
)
from app.services.log_field_serialization import (
    api_field_to_db_value as _shared_api_field_to_db_value,
    sql_to_db_value as _shared_sql_to_db_value,
)
from app.services.user_question_log_cache import (
    normalize_question_for_cache,
    try_merge_success_user_question_log,
)
# Dedicated executor for long-running askme operations (LLM + SQL validation).
# Keeps the default Starlette thread pool free for other endpoints so they don't get stuck
# when /get-alternate-prompts runs for 30-120+ seconds.
_LONG_RUNNING_EXECUTOR = ThreadPoolExecutor(max_workers=4, thread_name_prefix="askme-long")
import math

_DOC_ASSISTANT_CACHE: Dict[Tuple[str, str], Any] = {}
_DOC_ASSISTANT_CACHE_LOCK = threading.Lock()

# Import DocumentService for file operations (new modular structure)
# Supports both SQLAlchemy Session and db_file modes for backward compatibility
try:
    from app.services.document_service import (
        DocumentService,
        ContentExtractor,
        AIAnalyzer,
        FILE_SYS_METADATA_DB,
        FILE_VECTORS_DB_DIR,
        FILE_UPLOAD_DISK_ROOT,
        FILE_IMAGE_DISK_ROOT,
        FILE_EXTRACT_DISK_ROOT,
    )
    # Alias for backward compatibility with existing code
    DocumentAssistant = DocumentService
    DOCUMENT_ASSISTANT_AVAILABLE = True
except ImportError:
    DOCUMENT_ASSISTANT_AVAILABLE = False
    DocumentAssistant = None
    DocumentService = None
    ContentExtractor = None
    AIAnalyzer = None
    FILE_SYS_METADATA_DB = "file_sys_metadata.db"
    FILE_VECTORS_DB_DIR = "file_vectors_db"
    FILE_UPLOAD_DISK_ROOT = "uploaded_files_/file_data"
    FILE_IMAGE_DISK_ROOT = "document_images_/file_data"
    FILE_EXTRACT_DISK_ROOT = "extracted_images_/file_data"


def _get_or_create_doc_assistant(
    *,
    agent_id: Union[str, List[str]],
    api_key: Optional[str],
    upload_folder: str,
    image_folder: str,
    extracted_folder: str,
    db_file: str,
    chroma_path: str,
    claude_api_key: Optional[str] = None,
):
    ck = str(claude_api_key or "")
    # Normalize agent_id for cache key
    if isinstance(agent_id, list):
        aid_key = ",".join(sorted([str(a).strip() for a in agent_id if str(a).strip()]))
    else:
        aid_key = str(agent_id).strip()
        
    # Use agent_id in the cache key to ensure that when switching agents, 
    # we get a DocumentAssistant instance scoped to the correct agent_id(s).
    cache_key = (aid_key, str(db_file), str(chroma_path), str(api_key or ""), ck)
    with _DOC_ASSISTANT_CACHE_LOCK:
        cached = _DOC_ASSISTANT_CACHE.get(cache_key)
        if cached is not None:
            return cached
        assistant = DocumentAssistant(
            upload_folder=upload_folder,
            image_folder=image_folder,
            extracted_folder=extracted_folder,
            db_file=db_file,
            chroma_path=chroma_path,
            openai_api_key=str(api_key) if api_key else None,
            claude_api_key=ck or None,
            agent_id=agent_id,
        )
        _DOC_ASSISTANT_CACHE[cache_key] = assistant
        return assistant


async def _purge_talkback_sessions_older_than_24h() -> None:
    """Best-effort TTL cleanup for ``talkback_session_state`` (see ``purge_expired_talkback_sessions``)."""
    try:
        await run_in_threadpool(
            lambda: get_askme_service().purge_expired_talkback_sessions(24)
        )
    except Exception as exc:
        print(f"[talkback-session-ttl] purge on new thread failed: {exc}")


def clean_nan_values(obj):
    """
    Recursively clean NaN, Infinity, and -Infinity values from data structures
    to make them JSON serializable. Replaces NaN with None, and Infinity/-Infinity with None.
    Handles numpy/pandas scalars (e.g. np.float64(nan)) that standard json.dumps cannot serialize.
    """
    import math

import numpy as np


class NumpyEncoder(json.JSONEncoder):
    """Custom JSON encoder that converts numpy/pandas types to native Python types."""
    def default(self, obj):
        if isinstance(obj, (datetime, date)):
            return obj.isoformat()
        if isinstance(obj, np.generic):
            val = obj.item()
            # numpy NaN/Inf floats must become None for valid JSON
            if isinstance(val, float) and (math.isnan(val) or math.isinf(val)):
                return None
            return val
        if isinstance(obj, np.ndarray):
            return obj.tolist()
        # Handle pandas types
        if hasattr(pd, 'DataFrame') and isinstance(obj, pd.DataFrame):
            return obj.to_dict(orient='records')
        if hasattr(pd, 'Series') and isinstance(obj, pd.Series):
            return obj.tolist()
        if hasattr(pd, 'Timestamp') and isinstance(obj, pd.Timestamp):
            return obj.isoformat()
        if hasattr(pd, 'NaT') and obj is pd.NaT:
            return None
        return super().default(obj)


def _to_json_safe(obj):
    """
    Serialize obj to a JSON-safe Python structure.
    Two-phase: first sanitize with clean_nan_values (replaces NaN/Inf floats
    and numpy types), then run through json.dumps/loads for any remaining
    non-standard types (DataFrames, etc.).
    """
    cleaned = clean_nan_values(obj)
    return json.loads(json.dumps(cleaned, cls=NumpyEncoder))


def clean_nan_values(obj):
    """
    Recursively clean NaN, Infinity, and -Infinity values from data structures
    to make them JSON serializable. Replaces NaN with None, and Infinity/-Infinity with None.
    Handles numpy/pandas scalars (e.g. np.float64(nan)) that standard json.dumps cannot serialize.
    """
    if isinstance(obj, dict):
        return {
            # Convert numpy keys directly without circular _to_json_safe call
            (k.item() if isinstance(k, np.generic) else k): clean_nan_values(v)
            for k, v in obj.items()
        }
    elif isinstance(obj, (list, tuple)):
        return [clean_nan_values(item) for item in obj]
    elif isinstance(obj, np.generic):
        val = obj.item()
        if isinstance(val, float) and (math.isnan(val) or math.isinf(val)):
            return None
        return val
    elif isinstance(obj, np.ndarray):
        return clean_nan_values(obj.tolist())
    elif isinstance(obj, pd.DataFrame):
        return clean_nan_values(obj.to_dict(orient='records'))
    elif isinstance(obj, pd.Series):
        return clean_nan_values(obj.tolist())
    elif isinstance(obj, float):
        if math.isnan(obj) or math.isinf(obj):
            return None
        return obj
    elif hasattr(obj, "item"):
        # numpy/pandas scalar (e.g. np.float64(nan), np.int64)
        try:
            v = obj.item()
            if isinstance(v, float) and (math.isnan(v) or math.isinf(v)):
                return None
            return v
        except (ValueError, AttributeError, TypeError):
            return None
    else:
        return obj


def drop_null_keys(obj):
    """
    Recursively remove keys whose value is None from dicts.
    Leaves lists and other values unchanged (except dict values are processed).
    """
    if isinstance(obj, dict):
        return {
            k: drop_null_keys(v)
            for k, v in obj.items()
            if v is not None
        }
    if isinstance(obj, list):
        return [drop_null_keys(item) for item in obj]
    return obj


def _df_unmerged_multi_agent(agents: List[Dict[str, Any]], **extra: Any) -> Dict[str, Any]:
    """
    Standard ``details.df`` shape when multiple agents return separate tables (not merged).
    Top-level ``headers`` / ``data`` are empty; per-agent payloads live under ``agents``.
    """
    payload: Dict[str, Any] = {
        "headers": [],
        "data": [],
        "multi_agent": True,
        "merged": False,
        "agents": agents,
    }
    payload.update(extra)
    return payload


def _df_disjoint_tabular_list(agents: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
    """
    ``details.df`` as a list when the pipeline could not merge tabular results (disjoint schemas).

    Each item: ``agent_id``, ``headers``, ``data`` — used only for non-mergeable multi-frame
    outcomes (``merge_strategy == separate_frames``); merged / single-frame paths keep the
    usual ``{headers, data}`` object.
    """
    out: List[Dict[str, Any]] = []
    for a in agents:
        item: Dict[str, Any] = {
            "agent_id": str(a.get("agent_id") or ""),
            "headers": list(a.get("headers") or []),
            "data": clean_nan_values(a.get("data") or [])
            if a.get("data")
            else [],
            "chart_code": a.get("chart_code", "N"),
        }
        if a.get("api_name") is not None:
            item["api_name"] = a.get("api_name") or ""
        out.append(item)
    return out


def _df_api_multi_frame_payload(
    agents: List[Dict[str, Any]],
    *,
    fallback_agent_id: str,
    multi_agent_mixed: bool,
) -> Union[Dict[str, Any], List[Dict[str, Any]]]:
    """
    Multi-API ``details.df``: align with DB-style payloads — one dict with headers/data when
    a single frame; a list of dicts (like ``_df_disjoint_tabular_list``) when multiple frames.
    Avoids the legacy ``agents`` / empty top-level headers wrapper.
    """
    if not agents:
        return {
            "agent_id": fallback_agent_id,
            "headers": [],
            "data": [],
            "chart_code": "N",
            "merged": False,
            "multi_agent_mixed": multi_agent_mixed,
            "tabular_from_merge": False,
        }
    for a in agents:
        a.setdefault("chart_code", "N")
    if len(agents) == 1:
        a0 = agents[0]
        _data = (
            clean_nan_values(a0.get("data") or [])
            if a0.get("data")
            else []
        )
        out: Dict[str, Any] = {
            "agent_id": str(a0.get("agent_id") or fallback_agent_id),
            "headers": list(a0.get("headers") or []),
            "data": _data,
            "chart_code": a0.get("chart_code", "N"),
            "merged": True,
            "multi_agent_mixed": multi_agent_mixed,
            "tabular_from_merge": bool(_data),
        }
        if a0.get("api_name"):
            out["api_name"] = a0.get("api_name")
        return out
    return _df_disjoint_tabular_list(agents)


def _sub_question_map_from_tasks(
    sub_tasks: Optional[List[Dict[str, Any]]], default_q: str
) -> Dict[str, str]:
    out: Dict[str, str] = {}
    if not sub_tasks:
        return out
    for t in sub_tasks:
        if not isinstance(t, dict):
            continue
        aid = t.get("agent_id")
        if aid is None:
            continue
        sq = (t.get("sub_question") or "").strip()
        out[str(aid)] = sq if sq else default_q
    return out


# --- @askme prior-df resolution (mirrors app.routes.download helpers; no import from download) ---

def _parse_api_field_askme_prior(
    field_value: Union[str, List[str], None], default=None
) -> List[str]:
    if not field_value or field_value == "N":
        return default or []
    if isinstance(field_value, list):
        return field_value
    if isinstance(field_value, str):
        field_value = field_value.strip()
        if field_value.startswith("[") and field_value.endswith("]"):
            try:
                parsed = json.loads(field_value)
                if isinstance(parsed, list):
                    return [str(item) for item in parsed]
            except Exception:
                pass
        return [field_value]
    return [str(field_value)] if field_value else []


def _normalize_sql_string_for_askme_prior(sql_val: Any) -> Optional[str]:
    sql_list = _parse_api_field_askme_prior(sql_val)
    sql_str = sql_list[0] if sql_list else None
    if sql_str and isinstance(sql_str, str):
        sql_str = sql_str.strip() or None
    if not sql_str or sql_str == "N":
        return None
    return sql_str


def _meaningful_api_askme_prior(api_endpoint: Any, api_params: Any) -> bool:
    if api_endpoint is None or api_params is None:
        return False
    if isinstance(api_endpoint, str) and api_endpoint.strip().upper() in (
        "",
        "N",
        "NONE",
        "[]",
    ):
        return False
    if isinstance(api_params, str) and api_params.strip().upper() in (
        "",
        "N",
        "NONE",
        "[]",
    ):
        return False
    return True


async def _askme_prior_fetch_dataframe_from_saved_api(
    service: Any, ctx: Any, single_agent_id: str
) -> Tuple[Optional[pd.DataFrame], Optional[str]]:
    """
    Run saved API (single or multi-endpoint) and return a DataFrame — same as download-excel
    when hydrating @askme without a prior ``df``.
    """
    api_endpoints = _parse_api_field_askme_prior(getattr(ctx, "api_endpoint", None))
    api_params_list = _parse_api_field_askme_prior(getattr(ctx, "api_params", None))
    api_validators = _parse_api_field_askme_prior(
        getattr(ctx, "api_response_validator", None)
    )
    if not api_endpoints or not _meaningful_api_askme_prior(
        getattr(ctx, "api_endpoint", None), getattr(ctx, "api_params", None)
    ):
        return None, None
    my_question = getattr(ctx, "my_question", "") or ""

    try:
        if len(api_endpoints) > 1:
            all_apis = await run_in_threadpool(service.get_all_apis, single_agent_id)
            selected_apis = []
            for ep in api_endpoints:
                for api in all_apis:
                    if api.get("endpoint") == ep:
                        selected_apis.append(api)
                        break
            if not selected_apis:
                return None, f"API endpoints {api_endpoints} not found for agent '{single_agent_id}'"
            svc_api = service._ensure_api_chat_service()
            merge_result = await run_in_threadpool(
                svc_api.call_multiple_apis_and_merge,
                selected_apis=selected_apis,
                user_query=my_question,
            )
            if not merge_result.get("success"):
                return None, merge_result.get(
                    "message", "Failed to retrieve data from multiple APIs"
                )
            return merge_result.get("df"), None

        api_endpoint_str = api_endpoints[0]
        api_params_str = api_params_list[0] if api_params_list else ""
        api_response_validator_str = api_validators[0] if api_validators else ""

        try:
            extracted_params = (
                ast.literal_eval(api_params_str)
                if isinstance(api_params_str, str) and api_params_str.strip()
                else {}
            )
            if not isinstance(extracted_params, dict):
                extracted_params = {}
        except Exception:
            extracted_params = {}

        apis_list = await run_in_threadpool(service.get_all_apis, single_agent_id)
        api_info: Dict[str, Any] = {}
        for api in apis_list:
            if api.get("endpoint") == api_endpoint_str:
                api_info = api
                break

        if not api_info:
            return (
                None,
                f"API endpoint '{api_endpoint_str}' not found for agent '{single_agent_id}'",
            )

        is_sap = api_info.get("extra1", "N")
        if is_sap == "Y":
            sap_result = await run_in_threadpool(
                service.process_sap_query,
                service_root=api_endpoint_str,
                entity_set=api_info.get("base_params", ""),
                user_query=my_question,
                response_schema=api_info.get("response_schema", ""),
                transformation_code=api_response_validator_str or "",
                username="S4h_aa",
                password="Welcome1",
                verify_ssl=False,
                api_query_params=extracted_params,
            )
            return sap_result.get("df"), None

        _data, _data2, df, _data3 = await run_in_threadpool(
            service.call_api_and_transform,
            extracted_params,
            api_endpoint_str,
            api_info.get("response_schema", ""),
            api_response_validator_str or "",
            my_question,
        )
        return df, None
    except Exception as e:
        return None, f"Error calling API for prior data: {e}"


async def _askme_resolve_prior_df_for_analysis(
    service: Any, ctx: Any, single_agent_id: str
) -> Tuple[Any, Optional[str]]:
    """
    For @askme: use ``ctx.df`` if present, else run saved SQL, else saved API.
    Returns (input for ``_parse_df_input``, error_message).
    """
    parsed = service._parse_df_input(getattr(ctx, "df", None))
    if parsed and len(parsed) > 0:
        return getattr(ctx, "df", None), None

    sql_str = _normalize_sql_string_for_askme_prior(getattr(ctx, "sql", None))
    if sql_str:

        def _run_sql() -> Any:
            service._ensure_sql()
            if not service.is_sql_valid_cached(sql=sql_str):
                raise ValueError("Saved SQL is not valid")
            return service.run_sql_cached(sql=sql_str)

        try:
            prior_df = await run_in_threadpool(_run_sql)
        except ValueError as ve:
            return None, str(ve)
        except Exception as e:
            return None, str(e)

        if prior_df is None or getattr(prior_df, "empty", True):
            return (
                None,
                "No rows returned from saved SQL. Provide 'df' or fix the SQL.",
            )
        return prior_df.to_dict("records"), None

    api_df, api_err = await _askme_prior_fetch_dataframe_from_saved_api(
        service, ctx, single_agent_id
    )
    if api_err:
        return None, api_err
    if api_df is not None and not api_df.empty:
        return api_df.to_dict("records"), None

    return None, None


def _chart_code_exec_context(df_for_code: Any) -> Tuple[Dict[str, Any], Dict[str, Any]]:
    """pd/json/plotly/np namespace for ``exec`` of LLM chart code (avoids NameError on ``go``/``px``)."""
    import json as json_module

    exec_globals = {
        # IMPORTANT: include `df` in globals as well as locals. Comprehensions (list/dict/set/gen)
        # created inside exec() may resolve free variables from the *globals* mapping.
        "df": df_for_code,
        "pd": pd,
        "json": json_module,
        "json_module": json_module,
        "go": go,
        "px": px,
        "np": np,
    }
    local_vars = {
        "df": df_for_code,
        "pd": pd,
        "json": json_module,
        "json_module": json_module,
        "go": go,
        "px": px,
        "np": np,
    }
    return exec_globals, local_vars


def _chart_json_from_exec_locals(local_vars: Dict[str, Any]) -> Optional[Dict[str, Any]]:
    """LLM code may set ``chart_json`` (dict) or ``chart_fig`` (``go.Figure``)."""
    chart_json: Any = local_vars.get("chart_json")
    if chart_json is None and local_vars.get("chart_fig") is not None:
        try:
            chart_json = local_vars["chart_fig"].to_dict()
        except Exception:
            chart_json = None
    if chart_json is not None and not isinstance(chart_json, dict):
        return None
    return chart_json


def _generate_chart_code_for_records(
    service: Any,
    question_text: str,
    columns: List[str],
    records: List[Dict[str, Any]],
    chart_code_to_reuse: Optional[str],
) -> str:
    """
    Same behavior as ask chart block: optional reuse of saved Python chart code, else LLM.
    Returns executable code string or 'N'.
    """
    import json as json_module

    if not records or not columns or len(records) >= 101:
        return "N"
    try:
        if (
            chart_code_to_reuse
            and str(chart_code_to_reuse).strip()
            and chart_code_to_reuse != "N"
        ):
            json_code = chart_code_to_reuse
        else:
            json_code = service.generate_json_code_cached(
                question=question_text,
                columns=columns,
                model="gpt-4.1",
            )
        if not json_code or json_code == "N":
            return "N"
        df_for_code = pd.DataFrame(records)
        exec_globals, local_vars = _chart_code_exec_context(df_for_code)
        exec(json_code, exec_globals, local_vars)
        _chart_json = _chart_json_from_exec_locals(local_vars)
        if _chart_json is None:
            return "N"
        return json_code
    except Exception as e:
        print(f"_generate_chart_code_for_records: {e}")
        return "N"


def _populate_per_agent_and_merged_chart_codes(
    df_payload: Union[Dict[str, Any], List[Dict[str, Any]]],
    *,
    service: Any,
    question_default: str,
    chart_code_to_use: Optional[str],
    sub_tasks: Optional[List[Dict[str, Any]]] = None,
    root_chart_code_for_aid: Optional[str] = None,
    first_tabular_agent_id: Optional[str] = None,
) -> None:
    """
    Fill ``chart_code`` on each ``agents[]`` row (or list-shaped df) and on merged
    single-table payloads. *root_chart_code_for_aid* is the code already computed
    for *first_tabular_agent_id* in the main handler (avoids duplicate LLM).
    Reuse *chart_code_to_use* only for that first tabular agent when generating.
    """
    sub_map = _sub_question_map_from_tasks(sub_tasks, question_default)

    def q_for(aid: str) -> str:
        return sub_map.get(str(aid), question_default)

    def one_agent(ag: Dict[str, Any]) -> None:
        rec = ag.get("data") or []
        aid = str(ag.get("agent_id") or "")
        if not rec or len(rec) >= 101:
            ag["chart_code"] = "N"
            return
        cols = list(rec[0].keys()) if isinstance(rec[0], dict) else []
        if not cols:
            ag["chart_code"] = "N"
            return
        reuse = None
        if first_tabular_agent_id is not None and aid == first_tabular_agent_id:
            if root_chart_code_for_aid and root_chart_code_for_aid != "N":
                ag["chart_code"] = root_chart_code_for_aid
                return
            reuse = chart_code_to_use
        ag["chart_code"] = _generate_chart_code_for_records(
            service, q_for(aid), cols, rec, reuse
        )

    if isinstance(df_payload, list):
        for ag in df_payload:
            if isinstance(ag, dict):
                one_agent(ag)
        return

    if not isinstance(df_payload, dict):
        return

    agents = df_payload.get("agents")
    if isinstance(agents, list):
        for ag in agents:
            if isinstance(ag, dict):
                one_agent(ag)
        return

    rec = df_payload.get("data") or []
    if not rec or len(rec) >= 101:
        df_payload["chart_code"] = "N"
        return
    cols = list(rec[0].keys()) if isinstance(rec[0], dict) else []
    if not cols:
        df_payload["chart_code"] = "N"
        return
    if (
        root_chart_code_for_aid
        and root_chart_code_for_aid != "N"
    ):
        df_payload["chart_code"] = root_chart_code_for_aid
        return
    df_payload["chart_code"] = _generate_chart_code_for_records(
        service, question_default, cols, rec, chart_code_to_use
    )


def convert_image_path_to_url(image_path: str, agent_id: str, base_url: str = None) -> str:
    """
    Convert a local file path to a backend URL for serving images.
    
    Args:
        image_path: Local file path (e.g., "extracted_images_/file_data/.../image.png")
        agent_id: Agent ID
        base_url: Base URL of the backend server (optional, defaults to http://localhost:8000)
    
    Returns:
        URL to access the image through the backend API
    """
    if not image_path:
        return ""
    
    # Check if already a URL (prevent double conversion)
    if image_path.startswith("http://") or image_path.startswith("https://"):
        return image_path
    
    # Use provided base_url or default from initial_config.txt (BASE_URL + SCHEME)
    if base_url is None:
        base_url = get_base_url()
    
    image_path = str(image_path).strip().strip('"').strip("'")
    # Normalize path separators to forward slashes
    image_path = image_path.replace("\\", "/")
    
    # Construct the URL - simplified for the new API endpoint structure
    # Served by static_files router: /api/images/{file_path} (disk path unchanged)
    url = f"{base_url}/api/images/{image_path}"
    return url


def _absolutize_document_media_path(path: str, api_base: str) -> str:
    if not path or str(path).startswith("http"):
        return path
    rel = str(path).strip()
    if rel.startswith("/api/"):
        return f"{api_base.rstrip('/')}{rel}"
    return rel


def _convert_document_hit_image_fields(hit: dict, agent_id: str, api_base: str = None) -> None:
    """Absolutize runtime page-image API paths on search/find hits."""
    if not isinstance(hit, dict):
        return
    base = (api_base or get_base_url()).rstrip("/")
    path = hit.get("path")
    if path and not str(path).startswith("http"):
        hit["path"] = _absolutize_document_media_path(path, base)
    for img in hit.get("relevant_images") or []:
        if isinstance(img, dict) and img.get("path") and not str(img["path"]).startswith("http"):
            img["path"] = _absolutize_document_media_path(img["path"], base)


def _convert_document_page_images(page_images: dict, agent_id: str, api_base: str = None) -> dict:
    if not isinstance(page_images, dict):
        return page_images
    base = (api_base or get_base_url()).rstrip("/")
    converted: dict = {}
    for page_key, imgs in page_images.items():
        converted[page_key] = []
        for img in imgs or []:
            if isinstance(img, dict) and img.get("path"):
                row = dict(img)
                if not str(row["path"]).startswith("http"):
                    row["path"] = _absolutize_document_media_path(row["path"], base)
                converted[page_key].append(row)
            else:
                converted[page_key].append(img)
    return converted


def _attach_talkback_documents(details: dict, agent_id: Optional[str] = None) -> None:
    """Set ``details.documents`` with preview contract for talkback / file UI."""
    if not isinstance(details, dict):
        return
    if agent_id and not details.get("agent_id"):
        details["agent_id"] = str(agent_id)
    try:
        from app.utils.document_preview import collect_talkback_documents_from_details

        docs = collect_talkback_documents_from_details(details)
        if docs:
            details["documents"] = docs
    except Exception:
        pass


def convert_relevant_images_to_urls(
    images: list, agent_id: str, base_url: str = None
) -> list:
    """Convert relevant_images path fields to static API URLs."""
    converted = []
    for img in images or []:
        if isinstance(img, dict) and "path" in img:
            c = img.copy()
            c["path"] = convert_image_path_to_url(img["path"], agent_id, base_url=base_url)
            converted.append(c)
        else:
            converted.append(img)
    return converted


# Schemas
class GenerateSummaryRequest(BaseModel):
    agent_id: Union[str, List[str]] # Supports single agent (string) or multi-agent (list)
    question: str
    columns: Union[List[str], List[List[str]]]  # Column names from DataFrame
    df: Optional[Union[List[Dict[str, Any]], str]] = None  # DataFrame as list of dicts or JSON string
    model: Optional[str] = None


class GenerateFollowupRequest(BaseModel):
    agent_id: Union[str, List[str]]  # Supports single agent (string) or multi-agent (list)
    question: str
    sql: Union[str, List[str]]  # Single SQL or list of SQLs (one per agent for multi-agent)
    columns: Union[List[str], List[List[str]]]  # Column names from DataFrame
    n_questions: int = 5
    model: str = "gpt-4.1"
    data_source: str = "database"  # "Database" or "API"


class ContinuationTalkbackRequest(BaseModel):
    agent_id: Union[str, List[str]]  # Supports single agent (string) or multi-agent (list)
    question: str
    answer_context: str
    columns: Union[List[str], List[List[str]]] = []
    n_items: int = 3
    model: str = "gpt-4.1"

class TalkbackResponse(BaseModel):
    talkback: str


class ContextualQuestionRequest(BaseModel):
    agent_id: str
    previous_questions: List[str]
    current_question: str
    model: str = "gpt-4.1"


class RewrittenQuestionRequest(BaseModel):
    agent_id: str
    last_question: Optional[str] = None
    new_question: str
    model: str = "gpt-4.1"


class ErrorRecoveryRequest(BaseModel):
    agent_id: Union[str, List[str]] # Supports single agent (string) or multi-agent (list)
    failed_question: str
    data_source: str = "database"  # "Database" or "API"
    max_questions: int = 3
    model: str = "gpt-4.1"


class GenerateQuestionRequest(BaseModel):
    agent_id: str
    sql: str
    df: Union[List[Dict[str, Any]], str]  # DataFrame as list of dicts or JSON string
    model: str = "gpt-4.1"
 

class GenerateInitQuestionRequest(BaseModel):
    agent_id: str
    sql: str
    count: int = 5
    model: str = "gpt-4.1"


class GeneralQuestionRequest(BaseModel):
    agent_id: str
    query: str
    model: str = "gpt-4.1"


class CompareDataRequest(BaseModel):
    agent_id: str
    query: str
    df: Union[List[Dict[str, Any]], str]  # DataFrame as list of dicts or JSON string
    web_data: str
    model: str = "gpt-4.1"


class AskQuestionRequest(BaseModel):
    model_config = ConfigDict(populate_by_name=True, extra="ignore")

    user_id: str
    agent_id: Optional[Union[str, List[str]]] = None # Supports single agent (string) or multi-agent (list)
    question: Optional[str] = None
    my_question: Optional[str] = Field(None, alias="my_question")
    is_file: Optional[bool] = False # Explicit flag for file-related query
    question_id: Optional[int] = None  # ID of saved question in questions table
    datasource: Optional[str] = None  # e.g. "Database", "API" — set by talkback / client
    inContext: Optional[bool] = False
    previous_questions: Optional[List[str]] = None
    is_chart: Optional[bool] = False
    sql: Optional[str] = None
    api_endpoint: Optional[Union[str, List[str]]] = None
    api_params: Optional[Union[str, List[str]]] = None
    api_response_validator: Optional[Union[str, List[str]]] = None
    chart_json: Optional[str] = None
    code: Optional[str] = Field(None, alias="code")
    # List of rows, raw JSON string, or API envelope { "headers", "data", "agent_id", ... } from details.df
    df: Optional[Union[List[Dict[str, Any]], str, Dict[str, Any]]] = None
    is_dashboard: Optional[bool] = False  # When True, do not save to user_question_logs (dashboard requests)
    # Talkback prior-result LLM path: client opt-in (`search_web`) and prior rows (`df`) after user confirms.
    search_web: Optional[bool] = False
    # Talkback thread id: non-empty value reuses that thread; empty/absent → server mints one (see response ``session_key``).
    session_key: Optional[str] = None
    # Internal: set when talkback replays from ``user_question_log`` cache to avoid recursive cache lookup.
    user_question_log_cache_injected: bool = False
    # Internal: optional cached SQL summary blocks reused from user_question_log.
    sql_query_summaries_cached: Optional[Any] = None

    @field_validator("df", mode="before")
    @classmethod
    def _coerce_df_envelope(cls, v: Any) -> Any:
        """Accept list rows, JSON string, or {headers, data, ...} shape from multi-agent / details.df."""
        if v is None or isinstance(v, (list, str)):
            return v
        if isinstance(v, dict):
            data = v.get("data")
            if isinstance(data, list):
                return data
            return json.dumps(v)
        return v


# AskFileRequest schema removed - now using multipart/form-data with Form() parameters


def _request_implies_api_datasource(
    api_endpoint_to_use: Any, api_params_to_use: Any
) -> bool:
    if not api_endpoint_to_use or not api_params_to_use:
        return False
    ep = api_endpoint_to_use
    if isinstance(ep, list):
        ep = next((x for x in ep if x and str(x).strip().upper() not in ("N", "NONE")), None)
    if ep is None or (isinstance(ep, str) and ep.strip().upper() in ("N", "NONE", "[]", "")):
        return False
    pm = api_params_to_use
    if isinstance(pm, list):
        pm = next((x for x in pm if x is not None and str(x).strip()), None)
    if pm is None:
        return False
    if isinstance(pm, str) and not pm.strip():
        return False
    return True


def _normalize_datasource_str(val: Optional[Any]) -> Optional[str]:
    """Map client/talkback ``datasource`` field to canonical Database | API | File."""
    if val is None:
        return None
    s = str(val).strip()
    if not s:
        return None
    low = s.lower()
    if low in ("db", "database"):
        return "Database"
    if low == "api":
        return "API"
    if low in ("file", "document"):
        return "File"
    return None


def _infer_ask_datasource(
    datasource_to_use: Optional[str],
    api_endpoint_to_use: Any,
    api_params_to_use: Any,
    *,
    multi_agent_unset_ok: bool = False,
) -> Optional[str]:
    if datasource_to_use:
        return datasource_to_use
    if _request_implies_api_datasource(api_endpoint_to_use, api_params_to_use):
        return "API"
    if multi_agent_unset_ok:
        return None
    return "Database"


class GetItCheckedRequest(BaseModel):
    user_id: str
    agent_id: Union[str, List[str]]  # Supports single agent (string) or multi-agent (list)
    username: str
    question: str
    sql: Optional[Union[List[str], dict, str]] = None  # List[str] before str so list is validated first
    chart: Optional[str] = None
    start_time: str
    api_endpoint: Optional[Union[str, List[str]]] = None
    api_params: Optional[Union[str, List[str]]] = None
    api_response_validator: Optional[Union[str, List[str]]] = None
    is_error: bool = False


router = APIRouter(prefix="/askme", tags=["Askme AI Assistant"])

def _normalize_agent_ids(agent_id_value: Optional[Union[str, List[str]]]) -> List[str]:
    """Normalize agent_id input to a flat list of non-empty strings."""
    if not agent_id_value:
        return []

    if isinstance(agent_id_value, list):
        out: List[str] = []
        for item in agent_id_value:
            if item is None:
                continue
            text = str(item).strip()
            if not text:
                continue
            if text.startswith("[") and text.endswith("]"):
                try:
                    parsed = json.loads(text)
                    if isinstance(parsed, list):
                        out.extend([str(v).strip() for v in parsed if v is not None and str(v).strip()])
                        continue
                except json.JSONDecodeError:
                    pass
            out.append(text)
        return out

    if isinstance(agent_id_value, str):
        text = agent_id_value.strip()
        if not text:
            return []
        if text.startswith("[") and text.endswith("]"):
            try:
                parsed = json.loads(text)
                if isinstance(parsed, list):
                    return [str(v).strip() for v in parsed if v is not None and str(v).strip()]
            except json.JSONDecodeError:
                pass
        return [text]

    text = str(agent_id_value).strip()
    return [text] if text else []


def _parse_multi_datasource(raw: str) -> List[str]:
    """Parse 'database,api' or '["database","api"]' into a normalized list."""
    text = (raw or "").strip()
    if not text:
        return []
    if text.startswith("[") and text.endswith("]"):
        try:
            parsed = json.loads(text)
            if isinstance(parsed, list):
                return [str(x).strip().upper() for x in parsed if str(x).strip()]
        except json.JSONDecodeError:
            pass
    return [p.strip().upper() for p in text.split(",") if p.strip()]


def _resolve_authorized_agent_ids(
    requested_agent_id: Optional[Union[str, List[str]]],
    current_user_id: str
) -> List[str]:
    """
    Resolve final agent ids for talkback ask route.
    - If request contains agent_id(s): return intersection with user's authorized agents.
    - If request omits agent_id: return all authorized agents for the user.
    """
    requested_ids = _normalize_agent_ids(requested_agent_id)

    with get_db_session() as db:
        user_agents = get_user_agents(current_user_id, db)

    authorized_ids: List[str] = []
    for agent in user_agents:
        aid = agent.get("agent_id")
        if aid is None:
            continue
        aid_str = str(aid).strip()
        if aid_str:
            authorized_ids.append(aid_str)

    if not requested_ids:
        return authorized_ids

    authorized_set = set(authorized_ids)
    return [aid for aid in requested_ids if aid in authorized_set]


def format_llm_response_as_html(text: str) -> str:
    """
    Format LLM response text as safe HTML for frontend display.
    Escapes HTML entities, converts Markdown-style formatting (**bold**, ## headers,
    - lists) to HTML, and newlines to <br/> so the content can be rendered with innerHTML.
    """
    if not text or not isinstance(text, str):
        return ""
    escaped = html_module.escape(text)
    # Convert **bold** to <strong>
    escaped = re.sub(r"\*\*(.+?)\*\*", r"<strong>\1</strong>", escaped, flags=re.DOTALL)
    # Convert *italic* to <em> (single asterisk, not part of **)
    escaped = re.sub(r"(?<!\*)\*([^*]+?)\*(?!\*)", r"<em>\1</em>", escaped)
    # Process line-by-line for headers and lists
    lines = escaped.split("\n")
    out = []
    in_list = False
    for line in lines:
        stripped = line.strip()
        # Headers (###, ##, # at start of line)
        if re.match(r"^####\s", stripped):
            h4_text = re.sub(r'^####\s*', '', stripped)
            out.append(f"<h4 class=\"askme-h4\">{h4_text}</h4>")
            in_list = False
        elif re.match(r"^###\s", stripped):
            h3_text = re.sub(r'^###\s*', '', stripped)
            out.append(f"<h3 class=\"askme-h3\">{h3_text}</h3>")
            in_list = False
        elif re.match(r"^##\s", stripped):
            h2_text = re.sub(r'^##\s*', '', stripped)
            out.append(f"<h2 class=\"askme-h2\">{h2_text}</h2>")
            in_list = False
        elif re.match(r"^#\s", stripped):
            h1_text = re.sub(r'^#\s*', '', stripped)
            out.append(f"<h1 class=\"askme-h1\">{h1_text}</h1>")
            in_list = False
        # Unordered list: line starts with - or * (with space)
        elif re.match(r"^[-*]\s", stripped):
            if not in_list:
                out.append("<ul class=\"askme-ul\">")
                in_list = True
            li_content = re.sub(r"^[-*]\s*", "", stripped)
            out.append(f"<li class=\"askme-li\">{li_content}</li>")
        else:
            if in_list:
                out.append("</ul>")
                in_list = False
            if stripped:
                out.append(f"<p class=\"askme-p\">{stripped}</p>")
            else:
                out.append("<br/>")
    if in_list:
        out.append("</ul>")
    html_content = "\n".join(out)
    return f'<div class="askme-response askme-text-content">{html_content}</div>'


def normalize_analysis_text_for_html(text: str) -> str:
    """Display-only cleanup for protocol markers leaked by LLM responses."""
    if not text or not isinstance(text, str):
        return ""
    cleaned = text.strip()
    cleaned = re.sub(
        r"^\s*OUTPUT_TYPE\s*:\s*[A-Z_]+\s*\n?",
        "",
        cleaned,
        flags=re.IGNORECASE,
    )
    cleaned = re.sub(r"^\s*TEXT\s*:\s*\n?", "", cleaned, flags=re.IGNORECASE)
    return cleaned.strip()


def get_mail_content(user_info, question, sql, agent, timestamp, type):
    """
    Generate email subject and HTML body for user question issues.
    
    Args:
        user_info: User information string
        question: Question text
        sql: SQL query
        agent: Agent name
        timestamp: Timestamp string
        type: Type of issue ("D" for needs checking, "E" for error)
    
    Returns:
        Tuple of (subject, html_body)
    """
    subject_map = {
        "D": f"Action Required: User Not Getting Correct Answer â€” {timestamp}",
        "E": f"Action Required: User Getting Error â€” {timestamp}",
    }
    subject = subject_map.get(type, f"Action Required â€” {timestamp}")

    # HTML version (email clients won't collapse this)
    html_body = dedent(f"""\
    <p>Hello {agent} Admin,</p>
    <p>{user_info} has reported that they are not getting {'proper answers to their prompt' if type == 'D' else 'answers to their prompt'}.</p>
    <p>Here are the details for your reference:</p>
    <ul>
      <li><strong>User prompt:</strong> {question}</li>
      <li><strong>Associated SQL:</strong> {sql}</li>
      <li><strong>Agent Name:</strong> {agent}</li>
      <li><strong>Time of Issue:</strong> {timestamp}</li>
    </ul>
    <p>Please investigate this issue at your earliest convenience.</p>
    <p>Thanks,<br>askme360 Team</p>
    """).strip()

    return subject, html_body


def _serialize_log_sequence(parts: List[Any]) -> str:
    """
    Store one or many values in a single log column: one value as JSON or plain str;
    multiple values as a JSON array (avoids ambiguous comma-splitting for SQL dicts).
    Used for SQL fields — plain string for a single SQL matches coerce_sql_field_for_request.
    """
    if not parts:
        return ""
    if len(parts) == 1:
        x = parts[0]
        if isinstance(x, (dict, list)):
            return json.dumps(x, default=str)
        return str(x) if x is not None else ""
    return json.dumps(list(parts), default=str)


def _serialize_api_log_field(parts: List[Any]) -> str:
    """
    Serialize API log fields (endpoint / params / validator) as a JSON array — even
    for a single value.  This mirrors ``_api_field_to_db_value(_to_list(val, n))``
    used by ``save-prompt`` / ``multi-agent-save-prompt`` so the stored format is
    always identical regardless of which path created the log row.
    Empty list → empty string (no API data).
    """
    if not parts:
        return ""
    cleaned: List[Any] = []
    for x in parts:
        if isinstance(x, (dict, list)):
            cleaned.append(json.dumps(x, default=str))
        else:
            cleaned.append(str(x) if x is not None else "")
    return json.dumps(cleaned)


def _has_meaningful_saved_sql(sql_val: Any) -> bool:
    """True when a saved question row carries SQL worth treating as DB-side replay."""
    if sql_val is None:
        return False
    if isinstance(sql_val, str):
        s = sql_val.strip().upper()
        if not s or s in ("N", "NONE", "[]", "{}"):
            return False
        return True
    if isinstance(sql_val, (dict, list)):
        return bool(sql_val)
    return True


def _has_meaningful_saved_api(saved: Dict[str, Any]) -> bool:
    """True when saved api_endpoint + api_params are present (lists or JSON arrays)."""
    ep = saved.get("api_endpoint")
    pr = saved.get("api_params")
    if ep is None or pr is None:
        return False
    if isinstance(ep, str) and ep.strip().upper() in ("", "N", "NONE", "[]"):
        return False
    if isinstance(pr, str) and pr.strip().upper() in ("", "N", "NONE", "[]"):
        return False
    return True


def _has_meaningful_request_api(api_endpoint: Any, api_params: Any) -> bool:
    """Same rules as saved-question API fields, for raw request payload."""
    return _has_meaningful_saved_api(
        {"api_endpoint": api_endpoint, "api_params": api_params}
    )


def _format_api_fields_from_merge_result(api_meta: Any) -> Tuple[str, str, str]:
    """api_endpoints / api_params / api_validators from call_multiple_apis_and_merge result."""
    if not api_meta or not isinstance(api_meta, dict):
        return "", "", ""

    def norm(v: Any) -> str:
        if v is None:
            return ""
        if isinstance(v, list):
            return _serialize_log_sequence(v)
        return _serialize_log_sequence([v])

    return (
        norm(api_meta.get("api_endpoints")),
        norm(api_meta.get("api_params")),
        norm(api_meta.get("api_validators")),
    )


def _format_sql_for_mixed_db(gen_sql: Any) -> str:
    """SQL side of a mixed DB+API run (empty if no DB agents ran)."""
    if gen_sql is None:
        return ""
    if isinstance(gen_sql, (dict, list)):
        return json.dumps(gen_sql, default=str)
    return str(gen_sql)


def _sql_arg_for_db_log(generated_sql: Any) -> Any:
    """SQL column for database-only executions (passed into create_user_question_log)."""
    if generated_sql is None:
        return "N"
    if isinstance(generated_sql, str) and not generated_sql.strip():
        return "N"
    return generated_sql


def _traces_and_subtasks_from_merge_sql_blob(
    blob: Any,
    *,
    fallback_agent_id: str,
    default_question: str,
) -> Optional[Tuple[List[Dict[str, Any]], List[Dict[str, Any]]]]:
    """Parse LLM merge JSON ``{question, agent_results:[{agent_id, sql, ...}]}`` into traces + DB sub_tasks."""
    parsed: Optional[Dict[str, Any]] = None
    if isinstance(blob, dict):
        parsed = blob
    elif isinstance(blob, str):
        s = blob.strip()
        if not s.startswith("{"):
            return None
        try:
            parsed = json.loads(s)
        except Exception:
            return None
    if not isinstance(parsed, dict) or "agent_results" not in parsed:
        return None
    base_q = str(parsed.get("question") or default_question or "").strip()
    traces: List[Dict[str, Any]] = []
    sub_tasks: List[Dict[str, Any]] = []
    for r in parsed.get("agent_results") or []:
        if not isinstance(r, dict):
            continue
        aid = str(r.get("agent_id") or fallback_agent_id)
        sq = (str(r.get("sub_question") or "").strip()) or base_q or default_question
        sp = r.get("sql")
        if sp is None:
            sp = ""
        elif not isinstance(sp, str):
            sp = json.dumps(sp, default=str)
        traces.append({"agent_id": aid, "sub_question": sq, "sql": sp})
        sub_tasks.append(
            {"sub_question": sq, "agent_id": aid, "datasource": "DATABASE"}
        )
    return (traces, sub_tasks) if traces else None


def _merge_info_from_multi_agent_sql_blob(blob: Any) -> Optional[Dict[str, Any]]:
    """
    Normalize multi-agent SQL blobs into ``run_sql_cached`` merge_info.

    Accepted shapes:
      - {"agent_results":[{"agent_id","sql",...}]}
      - {"queries": {...}, "execution_order": [...]}
      - [{"agent_id","sql",...}, ...]  # new saved questions format
      - JSON string of any of the above
    """
    parsed: Any = blob
    if isinstance(parsed, str):
        s = parsed.strip()
        if not s.startswith(("{", "[")):
            return None
        try:
            parsed = json.loads(s)
        except Exception:
            return None

    if isinstance(parsed, dict):
        if "queries" in parsed and isinstance(parsed.get("queries"), dict):
            qdict = parsed.get("queries") or {}
            queries: Dict[str, str] = {}
            for k, v in qdict.items():
                k_str = str(k).strip()
                if not k_str:
                    continue
                v_str = v if isinstance(v, str) else json.dumps(v, default=str)
                if not str(v_str).strip():
                    continue
                queries[k_str] = str(v_str)
            if not queries:
                return None
            order_raw = parsed.get("execution_order") or list(queries.keys())
            execution_order = [
                str(x).strip()
                for x in (order_raw if isinstance(order_raw, list) else list(queries.keys()))
                if str(x).strip() in queries
            ]
            if not execution_order:
                execution_order = list(queries.keys())
            return {
                "queries": queries,
                "execution_order": execution_order,
                "merge_key": parsed.get("merge_key", "id"),
                "merge_strategy": parsed.get("merge_strategy", "independent"),
            }
        if "agent_results" in parsed:
            parsed = parsed.get("agent_results") or []
        else:
            return None

    if isinstance(parsed, list):
        queries: Dict[str, str] = {}
        execution_order: List[str] = []
        for idx, item in enumerate(parsed):
            if not isinstance(item, dict):
                continue
            aid = str(
                item.get("agent_id")
                or item.get("id")
                or item.get("agent")
                or f"agent{idx + 1}"
            ).strip()
            sql_part = item.get("sql")
            if sql_part is None:
                continue
            sql_text = sql_part if isinstance(sql_part, str) else json.dumps(sql_part, default=str)
            if not str(sql_text).strip():
                continue
            queries[aid] = str(sql_text)
            if aid not in execution_order:
                execution_order.append(aid)
        if not queries:
            return None
        return {
            "queries": queries,
            "execution_order": execution_order or list(queries.keys()),
            "merge_key": "id",
            "merge_strategy": "independent",
        }
    return None


def _expand_sql_field_into_trace_rows(
    sql_raw: Any,
    *,
    fallback_agent_id: str,
    default_question: str,
    _depth: int = 0,
) -> List[Dict[str, Any]]:
    """
    Turn a ``sql`` field (plain SQL string, JSON string, or dict) into one or more
    ``{agent_id, sub_question, sql}`` rows with **executable SQL only** in ``sql``.
    """
    if _depth > 8:
        return [
            {
                "agent_id": fallback_agent_id,
                "sub_question": default_question,
                "sql": str(sql_raw) if sql_raw is not None else "",
            }
        ]
    if sql_raw is None:
        return [
            {
                "agent_id": fallback_agent_id,
                "sub_question": default_question,
                "sql": "",
            }
        ]
    if isinstance(sql_raw, dict):
        if "agent_results" in sql_raw:
            exp = _traces_and_subtasks_from_merge_sql_blob(
                sql_raw,
                fallback_agent_id=fallback_agent_id,
                default_question=default_question,
            )
            return list(exp[0]) if exp else []
        if "queries" in sql_raw:
            out: List[Dict[str, Any]] = []
            for k, sql_part in (sql_raw.get("queries") or {}).items():
                ks = str(k)
                aid = ks[5:] if ks.startswith("agent") else ks
                out.extend(
                    _expand_sql_field_into_trace_rows(
                        sql_part,
                        fallback_agent_id=str(aid),
                        default_question=default_question,
                        _depth=_depth + 1,
                    )
                )
            return out
        return [
            {
                "agent_id": fallback_agent_id,
                "sub_question": default_question,
                "sql": json.dumps(sql_raw, default=str),
            }
        ]
    if isinstance(sql_raw, list):
        out: List[Dict[str, Any]] = []
        for item in sql_raw:
            if isinstance(item, dict):
                aid = str(item.get("agent_id") or fallback_agent_id).strip() or fallback_agent_id
                sq = (
                    str(item.get("sub_question") or "").strip()
                    or default_question
                )
                sql_part = item.get("sql")
                if sql_part is None:
                    continue
                out.extend(
                    _expand_sql_field_into_trace_rows(
                        sql_part,
                        fallback_agent_id=aid,
                        default_question=sq,
                        _depth=_depth + 1,
                    )
                )
            else:
                out.extend(
                    _expand_sql_field_into_trace_rows(
                        item,
                        fallback_agent_id=fallback_agent_id,
                        default_question=default_question,
                        _depth=_depth + 1,
                    )
                )
        return out
    if isinstance(sql_raw, str):
        s = sql_raw.strip()
        if s.startswith(("{", "[")):
            try:
                parsed = json.loads(s)
            except Exception:
                # Some rerun paths can carry Python-repr payloads (single quotes).
                # Parse them as a compatibility fallback and continue normal flattening.
                try:
                    parsed = ast.literal_eval(s)
                except Exception:
                    return [
                        {
                            "agent_id": fallback_agent_id,
                            "sub_question": default_question,
                            "sql": sql_raw,
                        }
                    ]
            return _expand_sql_field_into_trace_rows(
                parsed,
                fallback_agent_id=fallback_agent_id,
                default_question=default_question,
                _depth=_depth + 1,
            )
        return [
            {
                "agent_id": fallback_agent_id,
                "sub_question": default_question,
                "sql": sql_raw,
            }
        ]
    return [
        {
            "agent_id": fallback_agent_id,
            "sub_question": default_question,
            "sql": json.dumps(sql_raw, default=str),
        }
    ]


def _flatten_sql_traces_list(traces: Optional[List[Any]]) -> List[Dict[str, Any]]:
    """Expand merge / queries JSON embedded in ``sql_traces[].sql`` into plain-SQL traces."""
    out: List[Dict[str, Any]] = []
    for t in traces or []:
        if not isinstance(t, dict):
            continue
        aid = str(t.get("agent_id") or "")
        sq = str(t.get("sub_question") or "")
        rows = _expand_sql_field_into_trace_rows(
            t.get("sql"),
            fallback_agent_id=aid,
            default_question=sq,
        )
        out.extend(rows)
    return out


def _infer_mixed_datasource_execution(sub_tasks: Optional[List[Dict[str, Any]]]) -> bool:
    types = set()
    for st in sub_tasks or []:
        types.add(str(st.get("datasource") or "").upper())
    return "DATABASE" in types and "API" in types


def _details_sql_envelope_db(
    *,
    agent_id: str,
    sub_question: str,
    sql_text: Any,
    training_driven_pipeline: bool = False,
) -> str:
    """JSON string for ``details['sql']``; ``sql_traces[].sql`` is always plain SQL when possible."""
    st = sql_text if sql_text is not None else ""
    if not isinstance(st, str):
        st = json.dumps(st, default=str)
    rows = _expand_sql_field_into_trace_rows(
        st,
        fallback_agent_id=str(agent_id),
        default_question=sub_question,
    )
    sub_out = [
        {
            "sub_question": r["sub_question"],
            "agent_id": r["agent_id"],
            "datasource": "DATABASE",
        }
        for r in rows
    ]
    mixed = _infer_mixed_datasource_execution(sub_out)
    payload: Dict[str, Any] = {
        "training_driven_pipeline": training_driven_pipeline,
        "mixed_datasource_execution": mixed,
        "sub_tasks": sub_out,
        "sql_traces": rows,
        "api_traces": [],
    }
    return json.dumps(payload, default=str)


def _details_sql_envelope_from_db_generated(
    generated_sql: Any,
    question: str,
    fallback_agent_id: str,
    *,
    training_driven_pipeline: bool = False,
) -> str:
    """Build training-style ``details['sql']`` JSON from legacy ``generated_sql`` (str or structured dict)."""
    if isinstance(generated_sql, dict) and "agent_results" in generated_sql:
        traces: List[Dict[str, Any]] = []
        sub_tasks: List[Dict[str, Any]] = []
        for r in generated_sql.get("agent_results") or []:
            if not isinstance(r, dict):
                continue
            aid = str(r.get("agent_id") or fallback_agent_id)
            sq = (r.get("sub_question") or "").strip() or question
            sql_part = r.get("sql") or ""
            for er in _expand_sql_field_into_trace_rows(
                sql_part, fallback_agent_id=aid, default_question=sq
            ):
                traces.append(er)
                sub_tasks.append(
                    {
                        "sub_question": er["sub_question"],
                        "agent_id": er["agent_id"],
                        "datasource": "DATABASE",
                    }
                )
        mixed = _infer_mixed_datasource_execution(sub_tasks)
        payload = {
            "training_driven_pipeline": training_driven_pipeline,
            "mixed_datasource_execution": mixed,
            "sub_tasks": sub_tasks,
            "sql_traces": traces,
            "api_traces": [],
        }
        return json.dumps(payload, default=str)
    if isinstance(generated_sql, str):
        try:
            parsed = json.loads(generated_sql)
            if isinstance(parsed, dict) and "agent_results" in parsed:
                exp = _traces_and_subtasks_from_merge_sql_blob(
                    parsed,
                    fallback_agent_id=fallback_agent_id,
                    default_question=question,
                )
                if exp:
                    trs, sts = exp
                    mixed = _infer_mixed_datasource_execution(sts)
                    payload = {
                        "training_driven_pipeline": training_driven_pipeline,
                        "mixed_datasource_execution": mixed,
                        "sub_tasks": sts,
                        "sql_traces": trs,
                        "api_traces": [],
                    }
                    return json.dumps(payload, default=str)
            if isinstance(parsed, dict) and "queries" in parsed:
                traces = _flatten_sql_traces_list(
                    [
                        {
                            "agent_id": str(
                                k[5:]
                                if str(k).startswith("agent")
                                else str(k)
                            ),
                            "sub_question": question,
                            "sql": v,
                        }
                        for k, v in (parsed.get("queries") or {}).items()
                    ]
                )
                sub_tasks = [
                    {
                        "sub_question": tr["sub_question"],
                        "agent_id": tr["agent_id"],
                        "datasource": "DATABASE",
                    }
                    for tr in traces
                ]
                mixed = _infer_mixed_datasource_execution(sub_tasks)
                payload = {
                    "training_driven_pipeline": training_driven_pipeline,
                    "mixed_datasource_execution": mixed,
                    "sub_tasks": sub_tasks,
                    "sql_traces": traces,
                    "api_traces": [],
                }
                return json.dumps(payload, default=str)
            if isinstance(parsed, list):
                traces = _flatten_sql_traces_list(
                    [
                        {
                            "agent_id": str((x or {}).get("agent_id") or fallback_agent_id),
                            "sub_question": str((x or {}).get("sub_question") or question),
                            "sql": (x or {}).get("sql"),
                        }
                        for x in parsed
                        if isinstance(x, dict)
                    ]
                )
                if traces:
                    sub_tasks = [
                        {
                            "sub_question": tr["sub_question"],
                            "agent_id": tr["agent_id"],
                            "datasource": "DATABASE",
                        }
                        for tr in traces
                    ]
                    mixed = _infer_mixed_datasource_execution(sub_tasks)
                    payload = {
                        "training_driven_pipeline": training_driven_pipeline,
                        "mixed_datasource_execution": mixed,
                        "sub_tasks": sub_tasks,
                        "sql_traces": traces,
                        "api_traces": [],
                    }
                    return json.dumps(payload, default=str)
        except Exception:
            pass
        return _details_sql_envelope_db(
            agent_id=fallback_agent_id,
            sub_question=question,
            sql_text=generated_sql,
            training_driven_pipeline=training_driven_pipeline,
        )
    return _details_sql_envelope_db(
        agent_id=fallback_agent_id,
        sub_question=question,
        sql_text=str(generated_sql),
        training_driven_pipeline=training_driven_pipeline,
    )


def _api_result_dict_for_envelope(
    api_endpoint: Any,
    api_params: Any,
    api_response_validator: Any,
) -> Dict[str, Any]:
    return {
        "api_endpoint": api_endpoint if api_endpoint is not None else "",
        "api_params": api_params,
        "api_response_validator": api_response_validator if api_response_validator is not None else "",
    }


def _details_sql_envelope_api(
    *,
    sub_tasks: List[Dict[str, Any]],
    api_traces: List[Dict[str, Any]],
    sql_traces: Optional[List[Dict[str, Any]]] = None,
    training_driven_pipeline: bool = False,
) -> str:
    sql_flat = _flatten_sql_traces_list(sql_traces or [])
    mixed = _infer_mixed_datasource_execution(sub_tasks)
    payload: Dict[str, Any] = {
        "training_driven_pipeline": training_driven_pipeline,
        "mixed_datasource_execution": mixed,
        "sub_tasks": sub_tasks,
        "sql_traces": sql_flat,
        "api_traces": api_traces,
    }
    return json.dumps(payload, default=str)


def _details_sql_envelope_mixed(
    gen_sql: Any,
    question: str,
    primary_agent_id: str,
    agent_ids: List[str],
    datasource_map: Dict[str, Dict[str, Any]],
    api_merge_meta: Dict[str, Any],
    *,
    training_driven_pipeline: bool = False,
) -> str:
    """
    Same JSON shape as other multi-agent paths: sql_traces + api_traces + sub_tasks,
    plus mixed_datasource_execution for DB+API runs.
    """
    sql_part = json.loads(
        _details_sql_envelope_from_db_generated(
            gen_sql if gen_sql is not None else "",
            question,
            primary_agent_id,
            training_driven_pipeline=training_driven_pipeline,
        )
    )
    sub_tasks: List[Dict[str, Any]] = []
    for aid in agent_ids:
        row = datasource_map.get(aid) or {}
        ds = str(row.get("type", "DATABASE")).upper()
        if ds == "DATABASE":
            sub_tasks.append(
                {
                    "sub_question": question,
                    "agent_id": str(aid),
                    "datasource": "DATABASE",
                }
            )
        elif ds == "API":
            sub_tasks.append(
                {
                    "sub_question": question,
                    "agent_id": str(aid),
                    "datasource": "API",
                }
            )
        elif ds == "FILE":
            sub_tasks.append(
                {
                    "sub_question": question,
                    "agent_id": str(aid),
                    "datasource": "FILE",
                }
            )
        else:
            sub_tasks.append(
                {
                    "sub_question": question,
                    "agent_id": str(aid),
                    "datasource": ds,
                }
            )

    api_traces: List[Dict[str, Any]] = []
    meta = api_merge_meta or {}
    details = meta.get("api_details") or []
    for d in details:
        if not isinstance(d, dict):
            continue
        aid = str(d.get("agent_id") or "")
        api_traces.append(
            {
                "agent_id": aid,
                "sub_question": question,
                "result": _api_result_dict_for_envelope(
                    d.get("endpoint"),
                    d.get("params"),
                    d.get("validator"),
                ),
            }
        )

    sql_traces_out = _flatten_sql_traces_list(sql_part.get("sql_traces") or [])
    payload: Dict[str, Any] = {
        "training_driven_pipeline": training_driven_pipeline,
        "mixed_datasource_execution": True,
        "sub_tasks": sub_tasks,
        "sql_traces": sql_traces_out,
        "api_traces": api_traces,
    }
    return json.dumps(payload, default=str)


def _log_fields_from_training_pipeline(pipe: Dict[str, Any]) -> Tuple[str, str, str, str]:
    """
    Build sql + API log columns from training pipeline sql_traces / api_traces.
    """
    sql_traces = pipe.get("sql_traces") or []
    api_traces = pipe.get("api_traces") or []

    # Preserve per-agent question split for reuse cache:
    # user_question_log.sql should be sql_traces list[{"agent_id","sql","sub_question"}].
    sql_trace_rows = _flatten_sql_traces_list(sql_traces)

    ep_parts: List[str] = []
    pr_parts: List[Any] = []
    val_parts: List[str] = []

    for t in api_traces:
        if not isinstance(t, dict):
            continue
        res = t.get("result")
        if not isinstance(res, dict):
            continue
        # Prefer the actual URL endpoint; api_used is the human-readable name and must NOT be stored here
        ep = res.get("api_endpoint") or res.get("endpoint")
        if isinstance(ep, list):
            for url in ep:
                ep_parts.append(str(url) if url else "")
        else:
            ep_parts.append(str(ep) if ep else "")
        _pr = res.get("api_params")
        if _pr is None:
            _pr = res.get("params")
        pr_parts.append(_pr if _pr is not None else "")
        val = res.get("api_response_validator")
        if val is None:
            val = res.get("code")
        val_parts.append("" if val is None else str(val))

    if sql_trace_rows:
        sql_out = json.dumps(sql_trace_rows, default=str)
    else:
        sql_out = "N"
    api_ep  = _serialize_api_log_field(ep_parts)    # API: always JSON array to match save-prompt
    api_pr  = _serialize_api_log_field(pr_parts)
    api_val = _serialize_api_log_field(val_parts)
    return sql_out, api_ep, api_pr, api_val


def _enrich_sub_tasks_with_api_traces(
    sub_tasks: Optional[List[Dict[str, Any]]],
    api_traces: Optional[List[Dict[str, Any]]],
) -> List[Dict[str, Any]]:
    """
    Add api_endpoint / api_params / api_response_validator to API sub_tasks using the
    same fields as create_user_question_log (_log_fields_from_training_pipeline).
    """
    if not sub_tasks:
        return []
    out: List[Dict[str, Any]] = [dict(t) for t in sub_tasks]
    if not api_traces:
        return out
    for t in api_traces:
        if not isinstance(t, dict):
            continue
        aid = str(t.get("agent_id") or "")
        sq = (t.get("sub_question") or "").strip()
        res = t.get("result")
        if not isinstance(res, dict):
            continue
        ep = res.get("api_endpoint") or res.get("endpoint")
        if isinstance(ep, list):
            ep_s = json.dumps(ep, default=str) if ep else ""
        else:
            ep_s = str(ep) if ep else ""
        pr = res.get("api_params")
        if pr is None:
            pr = res.get("params")
        if pr is None:
            pr_s = ""
        elif isinstance(pr, (dict, list)):
            pr_s = json.dumps(pr, default=str)
        else:
            pr_s = str(pr)
        val = res.get("api_response_validator")
        if val is None:
            val = res.get("code")
        val_s = "" if val is None else str(val)
        for row in out:
            if str(row.get("agent_id") or "") != aid:
                continue
            if (row.get("sub_question") or "").strip() != sq:
                continue
            if str(row.get("datasource") or "").upper() != "API":
                continue
            row["api_endpoint"] = ep_s
            row["api_params"] = pr_s
            row["api_response_validator"] = val_s
            break
    return out


def create_user_question_log(
    user_id: str,
    agent_id: str,
    question: str,
    sql: str,
    chart: str,
    web: str,
    executed: str,
    start_time: str,
    api_endpoint: str = '',
    api_params: str = '',
    api_response_validator: str = '',
    dataframe: str = "N",
    question_normalized: Optional[str] = None,
    sql_query_summaries: Optional[Any] = None,
    db: Session = None  # Optional for backward compatibility - will be ignored
) -> Optional[int]:
    """
    Create a new user question log entry.
    Uses lazy database session - acquires connection only when needed.
    Uses retry logic for database locks.
    
    Args:
        user_id: User ID
        agent_id: Agent ID
        question: Question text
        sql: SQL query (or "N" if not applicable)
        chart: Chart code (or "N" if not applicable)
        web: Web search flag ("Y" or "N")
        executed: Execution status ("Y", "N", "E" for error, "D" for needs checking)
        start_time: Start time in format "%Y-%m-%d %H:%M:%S"
        api_endpoint: API endpoint (optional)
        api_params: API parameters (optional)
        api_response_validator: API response validator (optional)
        dataframe: DataFrame columns as string (optional, default "N")
        question_normalized: Cache key override; when omitted, derived from ``question``.
        db: DEPRECATED - ignored, kept for backward compatibility
    
    Returns:
        Optional[int]: ID of the created log entry if successful, None otherwise
    """
    import time
    from sqlalchemy.exc import OperationalError
    from app.core.database import SessionLocal

    def _extract_sql_traces_only(sql_payload: Any) -> Any:
        """
        user_question_log requirement: persist only ``sql_traces`` shape when available.
        Returns list[trace] / scalar payload for fallback.
        """
        def _agent_ids_from_arg(agent_val: Any) -> List[str]:
            if isinstance(agent_val, list):
                return [str(x).strip() for x in agent_val if str(x).strip()]
            s = str(agent_val or "").strip()
            if not s:
                return []
            if s.startswith("[") and s.endswith("]"):
                try:
                    parsed = json.loads(s)
                    if isinstance(parsed, list):
                        return [str(x).strip() for x in parsed if str(x).strip()]
                except Exception:
                    pass
            return [s]

        def _traces_from_sql_list(sql_list: List[Any]) -> Optional[List[Dict[str, Any]]]:
            rows: List[Dict[str, Any]] = []
            agent_ids = _agent_ids_from_arg(agent_id)
            for idx, item in enumerate(sql_list):
                if item is None:
                    continue
                if isinstance(item, dict) and item.get("sql") is not None:
                    aid = str(item.get("agent_id") or (agent_ids[idx] if idx < len(agent_ids) else ""))
                    sq = str(item.get("sub_question") or question or "").strip()
                    rows.extend(
                        _expand_sql_field_into_trace_rows(
                            item.get("sql"),
                            fallback_agent_id=aid,
                            default_question=sq,
                        )
                    )
                    continue
                rows.extend(
                    _expand_sql_field_into_trace_rows(
                        item,
                        fallback_agent_id=agent_ids[idx] if idx < len(agent_ids) else "",
                        default_question=str(question or "").strip(),
                    )
                )
            return rows if rows else None

        def _from_agent_results(agent_results: Any) -> Optional[List[Dict[str, Any]]]:
            if not isinstance(agent_results, list):
                return None
            rows: List[Dict[str, Any]] = []
            for r in agent_results:
                if not isinstance(r, dict):
                    continue
                sql_part = r.get("sql")
                if sql_part is None:
                    continue
                rows.extend(
                    _expand_sql_field_into_trace_rows(
                        sql_part,
                        fallback_agent_id=str(r.get("agent_id") or ""),
                        default_question=str(r.get("sub_question") or question or "").strip(),
                    )
                )
            return rows if rows else None

        def _collapse_single_agent_sql(payload: Any) -> Any:
            """
            Keep historical single-agent behavior:
            store plain SQL string instead of a one-item trace array.
            """
            if not isinstance(payload, list):
                return payload
            traces = _flatten_sql_traces_list(payload)
            if len(traces) != 1:
                return payload
            agent_ids = _agent_ids_from_arg(agent_id)
            if len(agent_ids) > 1:
                return payload
            only_sql = traces[0].get("sql")
            if isinstance(only_sql, str) and only_sql.strip():
                return only_sql
            return payload

        if isinstance(sql_payload, dict):
            if isinstance(sql_payload.get("sql_traces"), list):
                traces = _flatten_sql_traces_list(sql_payload.get("sql_traces"))
                return _collapse_single_agent_sql(traces)
            from_agents = _from_agent_results(sql_payload.get("agent_results"))
            if from_agents is not None:
                return _collapse_single_agent_sql(from_agents)
            return sql_payload

        if isinstance(sql_payload, list):
            from_list = _traces_from_sql_list(sql_payload)
            return _collapse_single_agent_sql(
                from_list if from_list is not None else sql_payload
            )

        if isinstance(sql_payload, str):
            s = sql_payload.strip()
            if (s.startswith("{") and s.endswith("}")) or (s.startswith("[") and s.endswith("]")):
                try:
                    parsed = json.loads(s)
                except Exception:
                    return sql_payload
                return _extract_sql_traces_only(parsed)
            return sql_payload

        return sql_payload

    max_retries = 5
    retry_delay = 0.1  # Start with 100ms delay
    
    for attempt in range(max_retries):
        # Create a new session for each attempt (lazy connection)
        current_db = SessionLocal()
        try:
            current_time = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
            # start_time must parse or the whole insert is skipped (silent None).
            # Accept common variants; fall back to 0s duration.
            time_diff_seconds = 0.0
            if start_time:
                st = str(start_time).strip()
                start_dt = None
                for fmt in (
                    "%Y-%m-%d %H:%M:%S",
                    "%Y-%m-%d %H:%M:%S.%f",
                    "%Y-%m-%dT%H:%M:%S",
                    "%Y-%m-%dT%H:%M:%S.%f",
                ):
                    try:
                        start_dt = datetime.strptime(st, fmt)
                        break
                    except ValueError:
                        continue
                if start_dt is not None:
                    end_dt = datetime.strptime(current_time, "%Y-%m-%d %H:%M:%S")
                    time_diff_seconds = (end_dt - start_dt).total_seconds()
            
            # Ensure question is never None (database constraint)
            question_value = question if question is not None else ""
            
            # user_question_log stores sql_traces-only shape when available.
            sql_payload = _extract_sql_traces_only(sql)
            sql_value = _shared_sql_to_db_value(sql_payload, none_default=None)
            if sql_value is None or (isinstance(sql_value, str) and not sql_value.strip()):
                sql_value = "N"
            # Coerce agent_id to str: single id or JSON array of ids when list is passed.
            if agent_id is None:
                agent_id_value = ""
            elif isinstance(agent_id, list):
                agent_id_value = json.dumps(agent_id)
            else:
                agent_id_value = str(agent_id)
            
            # Normalize so reads filtered by str(request.user_id) match inserts
            user_id_norm = str(user_id).strip() if user_id is not None else ""

            ep = _shared_api_field_to_db_value(api_endpoint, none_default="") or ""
            pr = _shared_api_field_to_db_value(api_params, none_default="") or ""
            val = _shared_api_field_to_db_value(api_response_validator, none_default="") or ""

            qn_store_src = (
                question_normalized
                if question_normalized is not None
                else normalize_question_for_cache(question_value)
            )
            qn_store = (qn_store_src or "").strip() or None

            # Always insert a fresh log row for each run (no merge/update of older rows).
            exec_val = executed if executed is not None else "N"

            # Default token values at insert time so successful rows are never left null
            # when a later async token update is skipped by a specific branch.
            if question_value:
                input_tokens_value = max(1, len(question_value) // 4)
            else:
                input_tokens_value = 0
            output_tokens_value = 0
            total_tokens_value = input_tokens_value + output_tokens_value

            log_entry = UserQuestionLog(
                user_id=user_id_norm,
                agent_id=agent_id_value,
                question=question_value,
                sql=sql_value,
                chart=chart if chart is not None else "N",
                web=web if web is not None else "N",
                executed=exec_val,
                execution_time=str(time_diff_seconds),
                api_endpoint=ep,
                api_params=pr,
                api_response_validator=val,
                question_normalized=qn_store,
                input_tokens=input_tokens_value,
                output_tokens=output_tokens_value,
                total_tokens=total_tokens_value,
                sql_query_summaries=(
                    json.dumps(sql_query_summaries, default=str)
                    if isinstance(sql_query_summaries, (dict, list))
                    else (str(sql_query_summaries) if sql_query_summaries is not None else None)
                ),
                created_at=current_time,
                updated_at=current_time
            )
            current_db.add(log_entry)
            current_db.commit()
            current_db.refresh(log_entry)
            return log_entry.id
            
        except OperationalError as e:
            current_db.rollback()
            if "database is locked" in str(e).lower() and attempt < max_retries - 1:
                time.sleep(retry_delay * (2 ** attempt))
                continue
            else:
                print(f"[LOG] Error creating user question log (OperationalError): {e}")
                return None
        except Exception as e:
            current_db.rollback()
            print(f"[LOG] Error creating user question log: {type(e).__name__}: {e}")
            return None
        finally:
            # Always close the session
            try:
                current_db.close()
            except Exception:
                pass
    
    # If we've exhausted all retries
    print("Error creating user question log: Database is locked after multiple retries.")
    return None


def calculate_tokens(text: str) -> int:
    """Approximate token count with a simple 1 token ~= 4 chars rule."""
    if not text:
        return 0
    return max(1, len(text) // 4)


def _extract_answer_text(response_details: Any) -> str:
    """
    Normalize the final answer text used for token estimation.
    Priority: details.text_analysis -> details.talkback -> empty string.
    """
    if not isinstance(response_details, dict):
        return ""
    for key in ("text_analysis", "talkback"):
        value = response_details.get(key)
        if isinstance(value, str) and value.strip():
            return value.strip()
    return ""


def update_user_question_log_tokens(
    log_id: int,
    input_tokens: int,
    output_tokens: int,
    total_tokens: int,
) -> None:
    """Best-effort token update for an existing user question log row."""
    from app.core.database import SessionLocal

    db = SessionLocal()
    try:
        row = db.query(UserQuestionLog).filter(UserQuestionLog.id == int(log_id)).first()
        if row is None:
            return
        row.input_tokens = int(input_tokens or 0)
        row.output_tokens = int(output_tokens or 0)
        row.total_tokens = int(total_tokens or 0)
        row.updated_at = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
        db.commit()
    except Exception as e:
        db.rollback()
        print(f"[LOG] Error updating token metrics for log_id={log_id}: {e}")
    finally:
        try:
            db.close()
        except Exception:
            pass


def update_user_question_log_sql_query_summaries(
    log_id: int,
    sql_query_summaries: Any,
) -> None:
    """Best-effort sql_query_summaries update for an existing user question log row."""
    from app.core.database import SessionLocal

    db = SessionLocal()
    try:
        row = db.query(UserQuestionLog).filter(UserQuestionLog.id == int(log_id)).first()
        if row is None:
            return
        row.sql_query_summaries = (
            json.dumps(sql_query_summaries, default=str)
            if isinstance(sql_query_summaries, (dict, list))
            else (str(sql_query_summaries) if sql_query_summaries is not None else None)
        )
        row.updated_at = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
        db.commit()
    except Exception as e:
        db.rollback()
        print(f"[LOG] Error updating sql_query_summaries for log_id={log_id}: {e}")
    finally:
        try:
            db.close()
        except Exception:
            pass


 


@router.post("/generate-summary")
async def generate_summary(
    request: GenerateSummaryRequest,
    current_user: User = Depends(get_current_user)
):
    """
    Generate a summary of the results of a SQL query (single-agent).

    Use agent_id as a single string. If a list is provided, the first entry is used.
    Send columns as: df.columns.tolist()
    """
    try:
        # Single-agent: extract agent_id as a plain string
        if isinstance(request.agent_id, list):
            agent_id = request.agent_id[0] if request.agent_id else ""
        else:
            agent_id = request.agent_id

        if not agent_id:
            return {
                "success": False,
                "message": "No agent ID provided",
                "details": {"question": request.question, "summary": ""}
            }

        # Normalize columns: handle List[str] or List[List[str]]
        columns_to_use = []
        if request.columns and any(isinstance(c, list) for c in request.columns):
            seen = set()
            for item in request.columns:
                if isinstance(item, list):
                    for col in item:
                        if col not in seen:
                            columns_to_use.append(col)
                            seen.add(col)
                else:
                    if item not in seen:
                        columns_to_use.append(item)
                        seen.add(item)
        else:
            columns_to_use = request.columns

        def get_service_safe():
            try:
                return get_askme_service(agent_id=agent_id)
            except Exception as e:
                return None

        service = await run_in_threadpool(get_service_safe)
        if not service:
            return {
                "success": False,
                "message": f"Agent configuration not found for agent_id: {agent_id}",
                "details": {"question": request.question, "summary": ""}
            }

        summary = await run_in_threadpool(
            service.generate_summary_cached,
            question=request.question,
            columns=columns_to_use,
            df=request.df,
            use_qwen=settings.default_use_qwen_for_summary_and_web()
        )
        return clean_nan_values({
            "success": True,
            "message": "Summary generated successfully",
            "details": {
                "question": request.question,
                "summary": summary,
                "summary_html": format_llm_response_as_html(summary)
            }
        })
    except Exception as e:
        return clean_nan_values({
            "success": False,
            "message": f"Error generating summary: {str(e)}",
            "details": {"question": request.question, "summary": ""}
        })


@router.post("/multi-agent-generate-summary")
async def multi_agent_generate_summary(
    request: GenerateSummaryRequest,
    current_user: User = Depends(get_current_user)
):
    """
    Generate a combined summary of results from multiple agents (multi-agent).

    Use agent_id as a list of agent ID strings.
    Send columns as:
      - List[str]       â€” same flat columns applied to every agent
      - List[List[str]] â€” one inner list per agent (positionally matched)
    """
    try:
        def _parse_df_blocks_for_summary(df_val: Any) -> List[Dict[str, Any]]:
            parsed = df_val
            if isinstance(df_val, str):
                s = df_val.strip()
                if not s:
                    return []
                try:
                    parsed = json.loads(s)
                except Exception:
                    return []
            if isinstance(parsed, dict):
                parsed = [parsed]
            if not isinstance(parsed, list):
                return []

            blocks: List[Dict[str, Any]] = []
            for idx, item in enumerate(parsed):
                if not isinstance(item, dict):
                    continue
                rows_raw = item.get("data")
                if not isinstance(rows_raw, list):
                    continue
                rows = [r for r in rows_raw if isinstance(r, dict)]
                if not rows:
                    continue

                raw_headers = item.get("headers")
                headers: List[str] = []
                if isinstance(raw_headers, list):
                    headers = [str(h) for h in raw_headers if h is not None and str(h).strip()]
                if not headers:
                    seen = set()
                    for row in rows:
                        for k in row.keys():
                            ks = str(k)
                            if ks not in seen:
                                seen.add(ks)
                                headers.append(ks)

                blocks.append(
                    {
                        "agent_id": str(item.get("agent_id") or item.get("source_agent") or f"block_{idx+1}"),
                        "headers": headers,
                        "data": rows,
                    }
                )
            return blocks

        # Normalize agent_id to list
        if isinstance(request.agent_id, list):
            agent_ids = request.agent_id
        elif isinstance(request.agent_id, str) and request.agent_id.startswith('[') and request.agent_id.endswith(']'):
            try:
                agent_ids = json.loads(request.agent_id)
                if not isinstance(agent_ids, list):
                    agent_ids = [request.agent_id]
            except Exception:
                agent_ids = [request.agent_id]
        elif isinstance(request.agent_id, str):
            agent_ids = [request.agent_id]
        else:
            agent_ids = list(request.agent_id) if request.agent_id else []

        if not agent_ids:
            return {
                "success": False,
                "message": "No agent IDs provided",
                "details": {"question": request.question, "summary": ""}
            }

        df_blocks = _parse_df_blocks_for_summary(request.df)
        df_headers_by_agent = {str(b.get("agent_id")): (b.get("headers") or []) for b in df_blocks}

        # Build per-agent columns list for generate_summary_multi_agent
        raw_columns = request.columns or []
        has_nested = any(isinstance(c, list) for c in raw_columns)

        per_agent_columns = []
        agent_names = []
        
        for idx, aid in enumerate(agent_ids):
            if has_nested:
                # Each agent gets its own inner list (by position); fall back to flat if short
                agent_cols = raw_columns[idx] if idx < len(raw_columns) and isinstance(raw_columns[idx], list) else []
            else:
                # Flat list â€” same columns for all agents
                agent_cols = raw_columns

            # Auto-derive columns from df blocks when payload columns are empty/missing.
            if (not agent_cols) and df_headers_by_agent:
                derived = df_headers_by_agent.get(str(aid), [])
                if not derived and idx < len(df_blocks):
                    derived = df_blocks[idx].get("headers", [])
                agent_cols = derived or []
            
            # Fetch agent name for better summary description
            def get_name(agent_id):
                try:
                    cfg = get_agent_config(agent_id=agent_id)
                    return cfg.get("agent_name", agent_id) if cfg else agent_id
                except Exception:
                    return agent_id
            
            a_name = await run_in_threadpool(get_name, aid)
            agent_names.append(a_name)
            
            per_agent_columns.append({
                "agent_id": aid, 
                "agent_name": a_name,
                "columns": agent_cols
            })

        # Use the first agent's service to call generate_summary_multi_agent_cached
        primary_agent_id = agent_ids[0]

        def get_service_safe():
            try:
                return get_askme_service(agent_id=primary_agent_id)
            except Exception:
                return None

        service = await run_in_threadpool(get_service_safe)
        if not service:
            return {
                "success": False,
                "message": f"Agent configuration not found for agent_id: {primary_agent_id}",
                "details": {"question": request.question, "summary": ""}
            }

        summary = await run_in_threadpool(
            service.generate_summary_multi_agent_cached,
            question=request.question,
            columns=per_agent_columns,
            df=request.df,
            use_qwen=settings.default_use_qwen_for_summary_and_web(),
            model=request.model,
        )
        return clean_nan_values({
            "success": True,
            "message": "Multi-agent summary generated successfully",
            "details": {
                "question": request.question,
                "summary": summary,
                "summary_html": format_llm_response_as_html(summary),
                "agents": agent_names
            }
        })
    except Exception as e:
        return clean_nan_values({
            "success": False,
            "message": f"Error generating multi-agent summary: {str(e)}",
            "details": {"question": request.question, "summary": ""}
        })



@router.post("/generate-followup-questions")
async def generate_followup_questions(
    request: GenerateFollowupRequest,
    current_user: User = Depends(get_current_user)
):
    """
    Generate a list of followup questions that you can ask Askme360 (single-agent).

    For Database data source: Returns validated followup questions with SQL
    For API data source: Returns simple followup questions without SQL validation

    Send columns as: df.columns.tolist()
    """
    try:
        # Use agent_id directly as a single string (single-agent endpoint)
        agent_id = request.agent_id if isinstance(request.agent_id, str) else request.agent_id[0] if request.agent_id else ""

        if not agent_id:
            return {
                "success": False,
                "message": "No agent ID provided",
                "details": {
                    "question": request.question,
                    "followup_questions": [],
                    "data_source": request.data_source
                }
            }

        # Normalize columns: handle List[str] or List[List[str]]
        columns_to_use = []
        if request.columns and any(isinstance(c, list) for c in request.columns):
            seen = set()
            for item in request.columns:
                if isinstance(item, list):
                    for col in item:
                        if col not in seen:
                            columns_to_use.append(col)
                            seen.add(col)
                else:
                    if item not in seen:
                        columns_to_use.append(item)
                        seen.add(item)
        else:
            columns_to_use = request.columns

        def get_service_safe():
            try:
                return get_askme_service(agent_id=agent_id)
            except Exception as e:
                print(f"Error getting service for agent {agent_id}: {e}")
                return None

        service = await run_in_threadpool(get_service_safe)
        if not service:
            return {
                "success": False,
                "message": f"Agent configuration not found for agent_id: {agent_id}",
                "details": {
                    "question": request.question,
                    "followup_questions": [],
                    "data_source": request.data_source
                }
            }

        followup_questions = []

        if request.data_source.upper() == "API":
            # For API: simple followup, no validation
            questions = await run_in_threadpool(
                service.generate_followup_cached,
                question=request.question,
                sql="",
                columns=columns_to_use
            )
            for question in (questions[:request.n_questions] if questions else []):
                followup_questions.append({
                    "question": question,
                    "sql": "",
                    "api_endpoint": "",
                    "api_params": "",
                    "api_response_validator": "",
                    "agent_id": agent_id
                })

        elif request.data_source.lower() == "database":
            # For Database: validated followups with SQL (sql can be str or list; use first if list)
            sql_to_use = request.sql[0] if isinstance(request.sql, list) else request.sql
            validated_followups = await run_in_threadpool(
                service.generate_validated_followup_questions,
                question=request.question,
                sql=sql_to_use,
                df=pd.DataFrame(columns=columns_to_use),
                max_questions=min(request.n_questions, 5)
            )
            for followup in validated_followups:
                followup_questions.append({
                    "question": followup.get("question", ""),
                    "sql": followup.get("sql", ""),
                    "api_endpoint": "",
                    "api_params": "",
                    "api_response_validator": "",
                    "agent_id": agent_id
                })

        if not followup_questions:
            return {
                "success": False,
                "message": "Failed to generate any followup questions",
                "details": {
                    "question": request.question,
                    "followup_questions": [],
                    "data_source": request.data_source
                }
            }

        return {
            "success": True,
            "message": "Follow-up questions generated successfully",
            "details": {
                "question": request.question,
                "followup_questions": followup_questions,
                "data_source": request.data_source
            }
        }

    except Exception as e:
        import traceback
        traceback.print_exc()
        return {
            "success": False,
            "message": f"Error generating followup questions: {str(e)}",
            "details": {
                "question": request.question,
                "followup_questions": [],
                "data_source": request.data_source if hasattr(request, 'data_source') else "Unknown"
            }
        }


def _df_payload_to_reconcile_prev_df_str(
    df_val: Optional[Union[List[Dict[str, Any]], str, Dict[str, Any]]],
) -> Optional[str]:
    """Serialize AskQuestionRequest.df for POST /reconcile-style prev_df (JSON string)."""
    if df_val is None:
        return None
    if isinstance(df_val, str):
        return df_val
    if isinstance(df_val, dict):
        return json.dumps(df_val)
    return json.dumps(df_val)


def _build_talkback_flat_form_dict(
    *,
    user_id: str,
    agent_id: Optional[str] = None,
    question: Optional[str] = None,
    my_question: Optional[str] = None,
    question_id: Optional[str] = None,
    datasource: Optional[str] = None,
    in_context: Optional[str] = None,
    previous_questions: Optional[str] = None,
    is_chart: Optional[str] = None,
    sql: Optional[str] = None,
    api_endpoint: Optional[str] = None,
    api_params: Optional[str] = None,
    api_response_validator: Optional[str] = None,
    chart_json: Optional[str] = None,
    code: Optional[str] = None,
    df: Optional[str] = None,
    is_dashboard: Optional[str] = None,
    search_web: Optional[str] = None,
    session_key: Optional[str] = None,
    is_file: Optional[str] = None,
) -> Dict[str, str]:
    """Collect non-None talkback multipart parts into a flat string dict (OpenAPI field names)."""
    flat: Dict[str, str] = {"user_id": user_id}
    if agent_id is not None:
        flat["agent_id"] = agent_id
    if question is not None:
        flat["question"] = question
    if my_question is not None:
        flat["my_question"] = my_question
    if question_id is not None:
        flat["question_id"] = question_id
    if datasource is not None:
        flat["datasource"] = datasource
    if in_context is not None:
        flat["inContext"] = in_context
    if previous_questions is not None:
        flat["previous_questions"] = previous_questions
    if is_chart is not None:
        flat["is_chart"] = is_chart
    if sql is not None:
        flat["sql"] = sql
    if api_endpoint is not None:
        flat["api_endpoint"] = api_endpoint
    if api_params is not None:
        flat["api_params"] = api_params
    if api_response_validator is not None:
        flat["api_response_validator"] = api_response_validator
    if chart_json is not None:
        flat["chart_json"] = chart_json
    if code is not None:
        flat["code"] = code
    if df is not None:
        flat["df"] = df
    if is_dashboard is not None:
        flat["is_dashboard"] = is_dashboard
    if search_web is not None:
        flat["search_web"] = search_web
    if session_key is not None:
        flat["session_key"] = session_key
    if is_file is not None:
        flat["is_file"] = is_file
    return flat


def _flat_form_strings_to_ask_question_request(flat: Dict[str, str]) -> AskQuestionRequest:
    """
    Build AskQuestionRequest from multipart form string fields (same keys as JSON AskQuestionRequest).
    """
    payload: Dict[str, Any] = {}
    bool_keys = frozenset({"inContext", "is_chart", "is_dashboard", "search_web", "is_file"})
    json_heavy_keys = frozenset(
        {
            "df",
            "previous_questions",
            "api_endpoint",
            "api_params",
            "api_response_validator",
        }
    )

    for key, raw in flat.items():
        if key in bool_keys:
            if not (raw or "").strip():
                continue
            payload[key] = raw.strip().lower() in ("1", "true", "yes", "on")
            continue
        if key == "question_id":
            st = raw.strip()
            if not st:
                continue
            try:
                payload[key] = int(st)
            except ValueError:
                payload[key] = raw
            continue
        if key == "agent_id":
            st = raw.strip()
            if not st:
                continue
            if st.startswith("["):
                try:
                    payload[key] = json.loads(st)
                except json.JSONDecodeError:
                    payload[key] = st
            else:
                payload[key] = st
            continue
        st = raw.strip() if raw else ""
        if not st:
            continue
        if key in json_heavy_keys:
            if st.startswith(("[", "{")):
                try:
                    payload[key] = json.loads(st)
                except json.JSONDecodeError:
                    payload[key] = raw
            else:
                payload[key] = st
            continue
        payload[key] = st

    try:
        return AskQuestionRequest.model_validate(payload)
    except ValidationError as e:
        raise RequestValidationError(e.errors(), body=payload) from e


async def _run_talkback_direct_pipeline(
    *,
    request: AskQuestionRequest,
    current_user: Any,
) -> Optional[Dict[str, Any]]:
    """
    Direct bypass disabled: always keep full talkback orchestrator flow.
    """
    _ = request
    _ = current_user
    print("🔍 [talkback:direct] direct bypass disabled -> full orchestrator pipeline")
    return None


@router.post("/talkback-ask-question")
async def ask_question_dispatcher(
    user_id: str = Form(..., description="User ID"),
    agent_id: Optional[str] = Form(
        None, description="Agent id or JSON array string for multiple agents"
    ),
    question: Optional[str] = Form(None),
    my_question: Optional[str] = Form(None),
    question_id: Optional[str] = Form(None),
    datasource: Optional[str] = Form(None),
    in_context: Optional[str] = Form(None, alias="inContext"),
    previous_questions: Optional[str] = Form(None),
    is_chart: Optional[str] = Form(None),
    sql: Optional[str] = Form(None),
    api_endpoint: Optional[str] = Form(None),
    api_params: Optional[str] = Form(None),
    api_response_validator: Optional[str] = Form(None),
    chart_json: Optional[str] = Form(None),
    code: Optional[str] = Form(None),
    df: Optional[str] = Form(
        None,
        description="Prior rows JSON or string; required when `file` is sent (reconcile path)",
    ),
    is_dashboard: Optional[str] = Form(None),
    search_web: Optional[str] = Form(None),
    session_key: Optional[str] = Form(
        None,
        description=(
            "Talkback thread id. Non-empty: reuse that thread's history (hashed with your user id). "
            "Empty or omitted: server creates a new thread id (returned as ``session_key`` in the JSON response)."
        ),
    ),
    is_file: Optional[str] = Form(None),
    file: Optional[UploadFile] = File(
        None,
        description="Optional spreadsheet (xlsx/csv); when set, runs reconcile like POST /reconcile",
    ),
    current_user: User = Depends(get_current_user),
):
    """
    Unified talkback route:
    - Resolves user-authorized agent_id(s) when omitted
    - Filters requested agent_id(s) by user authorization
    - Classifies intent and suggested agent(s) via LLM (per-agent training context)
    - Optionally narrows dispatch to suggested agents when non-empty
    - Dispatches via the multi-agent ask handler (one or many `agent_id`s)
    - Response includes intent_routing (classification JSON) when classification ran
    - Response includes ``session_key``: echo of the request value when you sent a non-empty thread id;
      otherwise the server-minted thread id to send on follow-up requests for the same conversation.

    Prior-result AI / web (@askme / @web): Users must type ``@askme`` or ``@web`` in ``question`` for
    those modes (same as ``/single-agent-ask-question`` / ``/multi-agent-ask-question``). The server
    does not infer these tags from keywords. After consent, resubmit with ``search_web: true``,
    prior rows in ``df``, and a ``question`` that still includes ``@askme`` or ``@web`` as needed.

    File reconcile / compare-with-upload: when `intent_routing.prior_df_followup` is `file_reconcile`,
    the response is a clarification turn (not `llm_prior_df` consent). Integrators should direct users
    to `POST /askme/reconcile` with `prev_df` from the last result and an xlsx/csv `file`, then
    continue in Talkback with merged tabular data in `df` if further @askme-style analysis is needed.

    **Request body:** `multipart/form-data` — same field names as AskQuestionRequest (`user_id`, `agent_id`,
    `question`, `inContext`, `df`, …) as form fields, plus optional `file` as binary (same style as
    `POST /askme/reconcile`). Optional ``session_key`` selects the talkback thread (non-empty reuses;
    empty or omitted mints a new id, returned in the response). When `file` is present, prior rows must be in `df` and `agent_id` set.
    If routing classifies `prior_df_followup` as `file_reconcile`, the same `reconcile()` handler as
    `POST /askme/reconcile` is used (not duplicate merge logic).
    """
    flat = _build_talkback_flat_form_dict(
        user_id=user_id,
        agent_id=agent_id,
        question=question,
        my_question=my_question,
        question_id=question_id,
        datasource=datasource,
        in_context=in_context,
        previous_questions=previous_questions,
        is_chart=is_chart,
        sql=sql,
        api_endpoint=api_endpoint,
        api_params=api_params,
        api_response_validator=api_response_validator,
        chart_json=chart_json,
        code=code,
        df=df,
        is_dashboard=is_dashboard,
        search_web=search_web,
        session_key=session_key,
        is_file=is_file,
    )
    request = _flat_form_strings_to_ask_question_request(flat)
    if talkback_client_session_key_is_empty(request):
        await _purge_talkback_sessions_older_than_24h()
    question_text = (
        request.question or getattr(request, "my_question", None) or ""
    ).strip()

    def _attach_question_field(payload: Any) -> Any:
        if isinstance(payload, dict) and not payload.get("question"):
            payload["question"] = question_text
        return payload

    # Fast path: explicit agent_id means user-selected dispatch, so skip LLM routing/orchestration.
    direct = await _run_talkback_direct_pipeline(
        request=request,
        current_user=current_user,
    )
    if direct is not None:
        print("⚡ [talkback:dispatcher] direct pipeline result returned (bypassed orchestrator)")
        return _attach_question_field(direct)

    if file is not None:
        requester_id = str(getattr(current_user, "id", None) or request.user_id)
        agent_ids = _resolve_authorized_agent_ids(request.agent_id, requester_id)
        if not agent_ids:
            return _attach_question_field({
                "success": False,
                "message": "No authorized agents are available for this request.",
                "details": {},
            })
        prev_df_str = _df_payload_to_reconcile_prev_df_str(request.df)

        async def _run_uploaded_file_analysis() -> Any:
            uid = requester_id
            file_agent_id = resolve_file_capable_agent_id_for_user(uid, agent_ids)
            if not file_agent_id:
                return _attach_question_field({
                    "success": False,
                    "message": "No file-capable agents are available for this request.",
                    "details": {
                        "needs_clarification": True,
                        "clarification_message": (
                            "Please select an agent that supports file/document datasource."
                        ),
                        "error_code": "file_agent_required",
                    },
                })
            file_result = await _ask_file_core(
                user_id=uid,
                agent_id=str(file_agent_id),
                question=question_text,
                file_content=await file.read(),
                filename=getattr(file, "filename", None),
                current_user=current_user,
                skip_log=False,
            )
            return _attach_question_field(file_result)

        question_norm = str(question_text or "").strip().lower()
        if question_norm == "analyze uploaded file":
            question_text = "@analyze uploaded file"
            setattr(request, "question", question_text)
            return await _run_uploaded_file_analysis()
        if question_norm == "compare with previous question":
            if prev_df_str is None or not str(prev_df_str).strip():
                return _attach_question_field({
                    "success": False,
                    "message": "df (prior tabular rows) is required when uploading a file; pass the last answer table in the `df` field",
                    "details": {},
                })
            q = (question_text or "").strip() or "Reconcile prior result with uploaded file"
            return _attach_question_field(await reconcile(
                prev_df=prev_df_str,
                question=q,
                user_id=requester_id,
                agent_id=agent_ids[0],
                file=file,
                current_user=current_user,
            ))

        # Uploaded file + explicit document command should use ask-file flow,
        # not reconcile (which requires prior df rows).
        parsed_cmd = _detect_file_commands(question_text)
        file_followup_action = decide_uploaded_file_followup_action(
            question_text=question_text,
            has_file_command=bool(parsed_cmd.get("has_file_cmd")),
            has_valid_session_key=bool(str(getattr(request, "session_key", "") or "").strip()),
            has_prev_df=bool(prev_df_str and str(prev_df_str).strip()),
        )
        if file_followup_action == "clarify_reconcile_vs_analyze":
            return _attach_question_field({
                "success": True,
                "message": "Clarification needed",
                "details": {
                    "needs_clarification": True,
                    "consent_type": "file_upload_action",
                    "clarification_type": "file_upload_action_selection",
                    "clarification_message": (
                        "Choose how to proceed with the uploaded file "
                        "Please select one option to continue."
                    ),
                    "clarification_options": [
                        {
                            "id": "reconcile",
                            "label": "Compare with previous question",
                        },
                        {
                            "id": "analyze",
                            "label": "Analyze uploaded file",
                        },
                    ],
                },
            })
        if file_followup_action == "auto_analyze":
            return await _run_uploaded_file_analysis()
        if parsed_cmd.get("has_file_cmd"):
            return await _run_uploaded_file_analysis()
        if prev_df_str is None or not str(prev_df_str).strip():
            return _attach_question_field({
                "success": False,
                "message": "df (prior tabular rows) is required when uploading a file; pass the last answer table in the `df` field",
                "details": {},
            })
        pipeline_out = await run_talkback_pipeline(
            request=request,
            current_user=current_user,
            resolve_authorized_agent_ids=_resolve_authorized_agent_ids,
            get_askme_service=get_askme_service,
            run_in_threadpool=run_in_threadpool,
            execute=multi_agent_ask_question,
        )
        ir = pipeline_out.get("intent_routing") if isinstance(pipeline_out, dict) else None
        prior = str((ir or {}).get("prior_df_followup", "none")).lower()
        if prior == "file_reconcile":
            q = (request.question or "").strip() or "Reconcile prior result with uploaded file"
            inferred_user_id = str(getattr(current_user, "id", None) or request.user_id)
            return _attach_question_field(await reconcile(
                prev_df=prev_df_str,
                question=q,
                user_id=inferred_user_id,
                agent_id=agent_ids[0],
                file=file,
                current_user=current_user,
            ))
        return _attach_question_field(pipeline_out)
    return _attach_question_field(await run_talkback_pipeline(
        request=request,
        current_user=current_user,
        resolve_authorized_agent_ids=_resolve_authorized_agent_ids,
        get_askme_service=get_askme_service,
        run_in_threadpool=run_in_threadpool,
        execute=multi_agent_ask_question,
    ))


@router.post("/single-agent-ask-question")
async def ask_question(
    request: AskQuestionRequest,
    current_user: User = Depends(get_current_user)
):
    """
    Ask a question to the agent.
    Returns SQL, DataFrame, and optionally chart data.
    
    Body:
        agent_id: Agent ID (required)
        question: Question to ask (required)
        in_context: Use context from previous questions (optional)
        is_chart: Generate chart data (optional)
    """
    # Track start time for logging
    question_start_time = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
    skip_user_question_log = bool(getattr(request, "is_dashboard", False))

    # Initialize variables for logging
    generated_sql = None
    api_endpoint = None
    api_params = None
    api_response_validator = None
    api_name = ""
    
    try:
        # Check if question_id is provided (support both question_id and legacy quesion_id typo)
        question_id = request.question_id
        
        # Initialize variables that may be overridden by saved question
        question_to_use = request.question
        sql_to_use = request.sql
        api_endpoint_to_use = request.api_endpoint
        api_params_to_use = request.api_params
        api_response_validator_to_use = request.api_response_validator
        chart_code_to_use = request.chart_json
        datasource_to_use = (request.datasource or "").strip() or None
        if isinstance(datasource_to_use, str):
            ds_up = datasource_to_use.upper()
            if ds_up in ("DB", "DATABASE"):
                datasource_to_use = "Database"
            elif ds_up == "API":
                datasource_to_use = "API"
            elif ds_up == "FILE":
                datasource_to_use = "File"
        
        auth_uid = str(getattr(current_user, "id", None) or request.user_id).strip()
        user_id = str(request.user_id).strip()
        if user_id != auth_uid and auth_uid != "1":
            return {
                "success": False,
                "message": f"Not authorized to run saved questions for user_id {user_id}",
                "details": {},
            }
        
        # If question_id is provided, fetch saved question information
        if question_id:
            # Use lazy session for DB lookup - connection is released immediately after query
            def fetch_saved_question():
                with get_db_session() as db:
                    if auth_uid == "1":
                        qry = db.query(Question).filter(Question.id == question_id)
                    else:
                        qry = db.query(Question).filter(
                            Question.id == question_id,
                            Question.user_id == user_id,
                        )
                    q = qry.first()
                    if q:
                        # Return dict to avoid DetachedInstanceError when session closes
                        return {
                            "question": q.question,
                            "sql": q.sql,
                            "api_endpoint": q.api_endpoint,
                            "api_params": q.api_params,
                            "api_response_validator": q.api_response_validator,
                            "chart": q.chart
                        }
                    return None
                    
            saved_question = await run_in_threadpool(fetch_saved_question)
                
            if not saved_question:
                return {
                    "success": False,
                    "message": f"Saved question with id {question_id} not found for user_id {user_id} and agent_id {request.agent_id}",
                    "details": {}
                }
            
            # Use saved question information to populate variables
            # Only override if the field is not already provided in the request
            if not question_to_use or question_to_use == "":
                question_to_use = saved_question.get("question")
            
            # Override SQL if not provided in request
            if not sql_to_use and saved_question.get("sql"):
                sql_to_use = saved_question.get("sql")
            
            # Override API fields if not provided in request
            if not api_endpoint_to_use and saved_question.get("api_endpoint"):
                api_endpoint_to_use = saved_question.get("api_endpoint")
            
            if not api_params_to_use and saved_question.get("api_params"):
                api_params_to_use = saved_question.get("api_params")
            
            if not api_response_validator_to_use and saved_question.get("api_response_validator"):
                api_response_validator_to_use = saved_question.get("api_response_validator")
            
            # Override chart_json if not provided in request
            if not chart_code_to_use and saved_question.get("chart"):
                chart_code_to_use = saved_question.get("chart")
            
            # Determine datasource from saved question if not provided
            if not datasource_to_use:
                # If API fields are present, assume API datasource
                if saved_question.get("api_endpoint") and saved_question.get("api_params"):
                    datasource_to_use = "API"
                # Otherwise, assume Database
                else:
                    datasource_to_use = "Database"

        request.question = question_to_use

        ds_req = _normalize_datasource_str(getattr(request, "datasource", None))
        if ds_req is not None:
            datasource_to_use = ds_req

        datasource_to_use = _infer_ask_datasource(
            datasource_to_use,
            api_endpoint_to_use,
            api_params_to_use,
            multi_agent_unset_ok=False,
        )
        # Follow-ups often send tabular `df` without API fields; inferring Database is wrong
        # for API-only (or file-only) agents. Multi-agent path uses build_multi_agent_datasource_map;
        # single-agent must align here.
        _aids_inf = _normalize_agent_ids(
            [request.agent_id] if not isinstance(request.agent_id, list) else request.agent_id
        )
        if len(_aids_inf) == 1:
            try:
                from app.services.agent_config import get_agent_by_id, ordered_available_datasources

                _row_inf = get_agent_by_id(str(_aids_inf[0]))
                _avail_inf = ordered_available_datasources(_row_inf) if _row_inf else []
                _aup_inf = [str(x).strip().upper() for x in _avail_inf]
                if "DATABASE" not in _aup_inf:
                    if "API" in _aup_inf:
                        datasource_to_use = "API"
                    elif "FILE" in _aup_inf and len(_aup_inf) == 1:
                        datasource_to_use = "File"
            except Exception:
                pass

        # Get agent config
        def get_agent_config_safe():
            try:
                return get_agent_config(agent_id=request.agent_id)
            except Exception as e:
                print(f"Error fetching agent config: {e}")
                return None

        agent_config = await run_in_threadpool(get_agent_config_safe)
        if not agent_config:
            # Log error for agent not found
            if not skip_user_question_log:
                await run_in_threadpool(
                    create_user_question_log,
                    user_id=user_id,
                    agent_id=request.agent_id,
                    question=question_to_use,
                    sql="N",
                    chart="N",
                    web="N",
                    executed="N",
                    start_time=question_start_time,
                    api_endpoint="",
                    api_params="",
                    api_response_validator=""
                )
            return {
                "success": False,
                "message": f"Agent '{request.agent_id}' not found",
                "details": {}
            }
        
        # Get service for this agent
        def get_service_safe():
            try:
                return get_askme_service(agent_id=request.agent_id)
            except Exception as e:
                print(f"Error getting askme service: {e}")
                return None

        service = await run_in_threadpool(get_service_safe)
        if not service:
            return {
                "success": False,
                "message": f"Agent '{request.agent_id}' not found",
                "details": {}
            }
        reset_global_usage_totals()
        
        # Generate contextual question if inContext is True
        question_to_process = question_to_use
        if request.inContext and request.previous_questions:
            try:
                question_to_process = await run_in_threadpool(
                    service.generate_contextual_question,
                    previous_questions=request.previous_questions,
                    current_question=question_to_use
                )
                print(f"Original question: {question_to_use}")
                print(f"Generated contextual question: {question_to_process}")
            except Exception as e:
                print(f"Error generating contextual question: {e}")
                # Fall back to original question if contextual generation fails
                question_to_process = question_to_use
        
        _sql_preloaded_single = sql_to_use and str(sql_to_use).strip().upper() not in (
            "N",
            "NONE",
            "[]",
        )
        if (
            not question_id
            and not _sql_preloaded_single
            and "@askme" not in (question_to_use or "")
            and "@web" not in (question_to_use or "")
        ):
            from app.services.agent_config import get_agent_by_id, ordered_available_datasources

            _row_sg = get_agent_by_id(str(request.agent_id))
            _avail_sg = ordered_available_datasources(_row_sg) if _row_sg else []
            if len(_avail_sg) > 1:
                try:
                    _mr_pipe = int(
                        getattr(service, "_max_row_count", "5000") or 5000
                    )
                except (TypeError, ValueError):
                    _mr_pipe = 5000

                def _run_sg_pipeline():
                    return multi_agent_pipeline(
                        question_to_process,
                        [str(request.agent_id)],
                        sql_to_use=None,
                        max_rows=_mr_pipe,
                    )

                pipe = await run_in_threadpool(_run_sg_pipeline)
                if pipe.get("used_pipeline") and pipe.get("success"):
                    merged_df = pipe.get("df")
                    file_results = pipe.get("file_results") or []
                    merge_meta = pipe.get("merge_meta") or {}
                    has_tabular = merged_df is not None and not getattr(
                        merged_df, "empty", True
                    )
                    sub_tasks = pipe.get("sub_tasks") or []
                    sql_traces = pipe.get("sql_traces") or []
                    api_traces_sg = pipe.get("api_traces") or []
                    _st_sg = [dict(x) for x in (sub_tasks or []) if isinstance(x, dict)]
                    sql_for_log = json.dumps(
                        {
                            "training_driven_pipeline": True,
                            "mixed_datasource_execution": _infer_mixed_datasource_execution(
                                _st_sg
                            ),
                            "sub_tasks": clean_nan_values(sub_tasks),
                            "sql_traces": _flatten_sql_traces_list(sql_traces),
                            "api_traces": clean_nan_values(api_traces_sg),
                        },
                        default=str,
                    )
                    df_records = []
                    formatted_columns = []
                    if has_tabular:
                        df_records = clean_nan_values(
                            merged_df.to_dict("records")
                        )
                        columns = list(merged_df.columns)
                        formatted_columns = [
                            str(c).replace("_", " ").capitalize() for c in columns
                        ]
                    response_details = {
                        "sql": sql_for_log,
                        "df": {
                            "agent_id": str(request.agent_id),
                            "headers": formatted_columns,
                            "data": df_records,
                            "chart_code": "N",
                            "multi_agent_mixed": True,
                            "tabular_from_merge": has_tabular,
                            "training_driven_pipeline": True,
                        },
                        "file_results": clean_nan_values(file_results),
                        "sub_tasks": clean_nan_values(sub_tasks),
                        "multi_agent_datasource_map": clean_nan_values(
                            pipe.get("multi_agent_datasource_map") or {}
                        ),
                        "raw_decomposition": clean_nan_values(
                            pipe.get("raw_decomposition") or []
                        ),
                        "datasource_map_list": clean_nan_values(
                            pipe.get("datasource_map_list") or []
                        ),
                        "api_traces": clean_nan_values(pipe.get("api_traces") or []),
                        "question": question_to_process,
                        "original_question": question_to_use
                        if request.inContext
                        else None,
                        "api_endpoint": "",
                        "api_params": "",
                        "api_response_validator": "",
                        "merge_meta": merge_meta,
                    }
                    _attach_talkback_documents(
                        response_details,
                        agent_id=str(request.agent_id) if getattr(request, "agent_id", None) else None,
                    )
                    if not skip_user_question_log:
                        await run_in_threadpool(
                            create_user_question_log,
                            user_id=user_id,
                            agent_id=request.agent_id,
                            question=question_to_use,
                            sql=sql_for_log,
                            chart="N",
                            web="N",
                            executed="Y",
                            start_time=question_start_time,
                            api_endpoint="",
                            api_params="",
                            api_response_validator="",
                        )
                    try:
                        await run_in_threadpool(
                            service.merge_sql_query_summaries_into_details_cached,
                            response_details,
                            question_to_process,
                        )
                    except Exception as _sq_sum_tr_exc:
                        print(f"sql_query_summaries (training pipeline single): {_sq_sum_tr_exc}")
                    return JSONResponse(
                        content=_to_json_safe(
                            {
                                "success": True,
                                "message": "Question processed successfully (training-driven pipeline)",
                                "details": response_details,
                            }
                        )
                    )

        # Handle @askme analysis requests
        if "@askme" in question_to_use and settings.IS_ASKME_ACTIVE:
            query = question_to_use.replace("@askme", "").strip()
            
            if not query:
                if not skip_user_question_log:
                    await run_in_threadpool(
                        create_user_question_log,
                        user_id=user_id,
                        agent_id=request.agent_id,
                        question=question_to_use,
                        sql="N",
                        chart="N",
                        web="N",
                        executed="N",
                        start_time=question_start_time,
                        api_endpoint="",
                        api_params="",
                        api_response_validator=""
                    )
                return {
                    "success": False,
                    "message": "Please provide your analysis request after @askme (e.g., 'build a forecast', 'do trend analysis', 'detailed summary')",
                    "details": {"question": request.question}
                }
            
            # Get dataframe - either from request or need to run a query first
            analysis_df = None
            
            # Option 1: DataFrame provided in request
            if request.df:
                # Parse DF input
                request.df = service._parse_df_input(request.df)
                
            if request.df and len(request.df) > 0:
                analysis_df = pd.DataFrame(request.df)
            else:
                # Option 2: Same as download-excel @askme — hydrate from request/saved SQL, else API
                # (save-prompt stores SQL/chart; ask with question_id has no `df` until we run them).
                _ctx_askme = SimpleNamespace(
                    my_question=question_to_use,
                    sql=sql_to_use,
                    code=chart_code_to_use,
                    api_endpoint=api_endpoint_to_use,
                    api_params=api_params_to_use,
                    api_response_validator=api_response_validator_to_use,
                    question_id=str(question_id) if question_id else None,
                    df=None,
                )
                _aid_askme = (
                    str(request.agent_id[0])
                    if isinstance(request.agent_id, list) and request.agent_id
                    else str(request.agent_id)
                )
                prior_input, prior_err = await _askme_resolve_prior_df_for_analysis(
                    service, _ctx_askme, _aid_askme
                )
                if prior_err:
                    if not skip_user_question_log:
                        await run_in_threadpool(
                            create_user_question_log,
                            user_id=user_id,
                            agent_id=request.agent_id,
                            question=question_to_use,
                            sql="N",
                            chart="N",
                            web="N",
                            executed="N",
                            start_time=question_start_time,
                            api_endpoint="",
                            api_params="",
                            api_response_validator="",
                        )
                    return {
                        "success": False,
                        "message": prior_err,
                        "details": {"question": request.question},
                    }
                if prior_input:
                    _parsed_askme = service._parse_df_input(prior_input)
                    if _parsed_askme and len(_parsed_askme) > 0:
                        analysis_df = pd.DataFrame(_parsed_askme)
                if analysis_df is None or (
                    hasattr(analysis_df, "empty") and analysis_df.empty
                ):
                    if not skip_user_question_log:
                        await run_in_threadpool(
                            create_user_question_log,
                            user_id=user_id,
                            agent_id=request.agent_id,
                            question=question_to_use,
                            sql="N",
                            chart="N",
                            web="N",
                            executed="N",
                            start_time=question_start_time,
                            api_endpoint="",
                            api_params="",
                            api_response_validator="",
                        )
                    return {
                        "success": False,
                        "message": "No previous data found. Please provide a dataframe in the 'df' field or run a query first to get data for analysis.",
                        "details": {"question": request.question},
                    }
            
            # Generate analysis based on user request
            try:
                analysis_result = await run_in_threadpool(service.ask_data_analysis_cached, question=query, df=analysis_df, use_qwen=settings.default_use_qwen_for_summary_and_web())
                
                is_chart_available = "N"
                result_df = None
                chart_fig = None
                analysis_text = None
                # Extract text analysis if available
                if analysis_result.get('text'):
                    analysis_text = analysis_result['text']
                    print("analysis_text", analysis_text)
                elif not analysis_result.get('has_dataframe') and not analysis_result.get('has_chart'):
                    # Fallback if nothing else was detected
                    analysis_text = str(analysis_result.get('text', ''))
                    print("Analysis text", analysis_text)
                
                
                # Get result dataframe if available
                if analysis_result.get('dataframe') is not None:
                    result_df = analysis_result['dataframe']
                    # Convert to records format
                    chart_df = analysis_result.get('dataframe')
                    chart_columns = list(chart_df.columns) if hasattr(chart_df, 'columns') else []
                    if chart_df is not None and len(chart_df) > 0 and len(chart_df) < 101:
                        json_code = await run_in_threadpool(
                            service.generate_json_code_cached,
                            question=question_to_process,
                            columns=chart_columns,
                            model="gpt-4.1",
                            use_qwen=settings.default_use_qwen_for_summary_and_web(),
                        )
                        
                        if json_code and json_code != "N":
                            try:
                                # Validate syntax before execution
                                compile(json_code, '<string>', 'exec')
                                
                                df_for_code = pd.DataFrame(chart_df)
                                exec_globals, local_vars = _chart_code_exec_context(
                                    df_for_code
                                )
                                exec(json_code, exec_globals, local_vars)
                                
                                # chart_json dict, or chart_fig from go.Figure() (common from Phi/small LLMs)
                                chart_json = _chart_json_from_exec_locals(
                                    local_vars
                                ) or {}
                                
                                # Ensure it has Plotly structure (data and layout)
                                if "data" not in chart_json or "layout" not in chart_json:
                                    chart_json = {}
                                else:
                                    # Ensure layout has title and axes
                                    if "title" not in chart_json.get("layout", {}):
                                        chart_json.setdefault("layout", {})["title"] = query[:50] if len(query) > 50 else query
                                    if "xaxis" not in chart_json.get("layout", {}):
                                        chart_json.setdefault("layout", {})["xaxis"] = {"title": chart_columns[0] if len(chart_columns) > 0 else ""}
                                    if "yaxis" not in chart_json.get("layout", {}) and len(chart_columns) > 1:
                                        chart_json.setdefault("layout", {})["yaxis"] = {"title": chart_columns[1] if len(chart_columns) > 1 else ""}
                                
                                if chart_json is not None:
                                    chart_fig = chart_json
                                    is_chart_available = json_code
                            except SyntaxError as syntax_err:
                                print(f"Chart code has syntax error: {syntax_err}")
                                chart_json = {}
                                chart_fig = chart_json
                                is_chart_available = "N"
                            except Exception as exec_err:
                                print(f"Error executing chart code: {exec_err}")
                                chart_json = {}
                                chart_fig = chart_json
                                is_chart_available = "N"
                        else:
                            # json_code is None or "N", create default chart
                            chart_json = {}
                            chart_fig = chart_json
                            is_chart_available = "N"
                
                    if hasattr(result_df, 'to_dict'):
                        df_records = result_df.to_dict('records')
                        # Clean NaN values to make JSON serializable
                        df_records = clean_nan_values(df_records)
                        columns = list(result_df.columns)
                    else:
                        df_records = []
                        columns = []
                else:
                    df_records = []
                    columns = []
                
                # Build response
                response_details = {
                    "question": request.question,
                    "analysis_query": query,
                    "sql": "N",
                    "api_endpoint": "",
                    "api_params": "",
                    "api_response_validator": ""
                }
                
                # Add text analysis (raw and HTML for frontend)
                if analysis_text:
                    response_details["text_analysis"] = analysis_text
                    response_details["text_analysis_html"] = format_llm_response_as_html(
                        normalize_analysis_text_for_html(analysis_text)
                    )
                
                # Add chart if available
                if chart_fig is not None:
                    # chart_fig is already chart_json (dict) from the execution above
                    try:
                        if isinstance(chart_fig, dict):
                            # Clean NaN values to make JSON serializable
                            chart_fig = clean_nan_values(chart_fig)
                            response_details["chart"] = chart_fig
                            response_details["chart_code"] = is_chart_available
                        else:
                            # Fallback: if it's still a plotly figure, convert to dict
                            chart_json = chart_fig.to_dict()
                            # Clean NaN values to make JSON serializable
                            chart_json = clean_nan_values(chart_json)
                            response_details["chart"] = chart_json
                            response_details["chart_code"] = is_chart_available
                    except Exception as e:
                        print(f"Error processing chart: {e}")
                
                # Add dataframe if available (fallback to input df if result_df is None)
                if result_df is not None and len(df_records) > 0:
                    formatted_columns = [col.replace('_', ' ').capitalize() for col in columns]
                    response_details["df"] = {
                        "agent_id": str(request.agent_id),
                        "headers": formatted_columns,
                        "data": df_records,
                        "chart_code": is_chart_available,
                    }
                elif analysis_df is not None and len(analysis_df) > 0:
                    # Fallback: Provide input df if no new df was generated
                    try:
                        fallback_records = analysis_df.to_dict('records')
                        fallback_records = clean_nan_values(fallback_records)
                        # Ensure we get columns from analysis_df
                        fallback_cols = [col.replace('_', ' ').capitalize() for col in analysis_df.columns]
                        response_details["df"] = {
                            "agent_id": str(request.agent_id),
                            "headers": fallback_cols,
                            "data": fallback_records,
                        }
                    except Exception as df_err:
                        print(f"Error in @askme DF fallback: {df_err}")
                
                # Check if any analysis was generated (flags and/or actual text / dataframe)
                _txt = (analysis_text or analysis_result.get("text") or "").strip()
                _has_any = (
                    analysis_result.get("has_text")
                    or analysis_result.get("has_dataframe")
                    or analysis_result.get("has_chart")
                    or bool(_txt)
                    or (analysis_result.get("dataframe") is not None)
                )
                if not _has_any:
                    if not skip_user_question_log:
                        await run_in_threadpool(
                            create_user_question_log,
                            user_id=user_id,
                            agent_id=request.agent_id,
                            question=question_to_use,
                            sql="N",
                            chart="N",
                            web="N",
                            executed="N",
                            start_time=question_start_time,
                            api_endpoint="",
                            api_params="",
                            api_response_validator=""
                        )
                    return {
                        "success": False,
                        "message": "Could not generate analysis. Please try rephrasing your request.",
                        "details": response_details
                    }
                
                # Log successful execution
                if not skip_user_question_log:
                    await run_in_threadpool(
                        create_user_question_log,
                        user_id=user_id,
                        agent_id=request.agent_id,
                        question=question_to_use,
                        sql="N",
                        chart=is_chart_available,
                        web="N",
                        executed="Y",
                        start_time=question_start_time,
                        api_endpoint="",
                        api_params="",
                        api_response_validator=""
                    )
                
                try:
                    # Force to string for safe slicing
                    analysis_str = str(analysis_text) if analysis_text else ""
                    summary_text = analysis_str[:1000] if analysis_str else "Data analysis completed."
                    talkback_df = None
                    if result_df is not None and hasattr(result_df, "empty") and not result_df.empty:
                        talkback_df = result_df
                    elif analysis_df is not None and hasattr(analysis_df, "empty") and not analysis_df.empty:
                        talkback_df = analysis_df
                    talkback = await run_in_threadpool(
                        service.generate_continuation_talkback_cached,
                        question=question_to_process,
                        answer_context=summary_text,
                        columns=columns if 'columns' in locals() else [],
                        n_items=3,
                        df=talkback_df,
                        use_qwen=settings.default_use_qwen_for_summary_and_web(),
                    )
                    response_details["talkback"] = talkback
                except Exception as e:
                    print(f"Error generating talkback for @askme: {e}")


                return {
                    "success": True,
                    "message": "Analysis completed successfully",
                    "details": response_details
                }
                
            except Exception as e:
                import traceback
                error_msg = f"Error performing analysis: {str(e)}"
                print(f"Error in @askme analysis: {traceback.format_exc()}")
                
                if not skip_user_question_log:
                    await run_in_threadpool(
                        create_user_question_log,
                        user_id=user_id,
                        agent_id=request.agent_id,
                        question=question_to_use,
                        sql="N",
                        chart="N",
                        web="N",
                        executed="N",
                        start_time=question_start_time,
                        api_endpoint="",
                        api_params="",
                        api_response_validator=""
                    )
                
                return {
                    "success": False,
                    "message": error_msg,
                    "details": {"question": request.question, "error": str(e)}
                }
        elif "@web" in question_to_use and settings.IS_WEB_ACTIVE:
            query = question_to_use.replace("@web", "").strip()
            
            if not query:
                if not skip_user_question_log:
                    await run_in_threadpool(
                        create_user_question_log,
                        user_id=user_id,
                        agent_id=request.agent_id,
                        question=question_to_use,
                        sql="N",
                        chart="N",
                        web="N",
                        executed="N",
                        start_time=question_start_time,
                        api_endpoint="",
                        api_params="",
                        api_response_validator=""
                    )
                return {
                    "success": False,
                    "message": "Please provide your search request after @web",
                    "details": {"question": request.question}
                }
            
            # Get dataframe - either from request or need to run a query first
            analysis_df = None
            
            # Option 1: DataFrame provided in request
            if request.df:
                # Parse DF input
                request.df = service._parse_df_input(request.df)
                
            if request.df and len(request.df) > 0:
                analysis_df = pd.DataFrame(request.df)
            else:
                # Standalone @web: Tavily + optional compare — no prior table required
                analysis_df = pd.DataFrame()

            # Generate analysis based on user request
            try:
                analysis_result = await run_in_threadpool(
                    service.search_and_format_response,
                    user_query=query,
                    last_response=analysis_df, use_qwen=settings.default_use_qwen_for_summary_and_web()
                )
                
                # Handle both string and dict responses
                if isinstance(analysis_result, str):
                    # Wrap string response in expected format
                    analysis_text = analysis_result
                    has_text = True
                elif isinstance(analysis_result, dict):
                    # Handle dict response (legacy format)
                    analysis_text = analysis_result.get('text', analysis_result.get('formatted_response', ''))
                    has_text = analysis_result.get('has_text', bool(analysis_text))
                else:
                    analysis_text = str(analysis_result)
                    has_text = True
                
                is_chart_available = "N"
                result_df = None
                chart_fig = None
                
                # Build response
                response_details = {
                    "question": request.question,
                    "analysis_query": query,
                    "sql": "N",
                    "api_endpoint": "",
                    "api_params": "",
                    "api_response_validator": ""
                }
                
                # Add text analysis (raw and HTML for frontend)
                if analysis_text:
                    response_details["text_analysis"] = analysis_text
                    response_details["text_analysis_html"] = format_llm_response_as_html(
                        normalize_analysis_text_for_html(analysis_text)
                    )
                
                # Check if any analysis was generated
                if not has_text or not analysis_text:
                    if not skip_user_question_log:
                        await run_in_threadpool(
                            create_user_question_log,
                            user_id=user_id,
                            agent_id=request.agent_id,
                            question=question_to_use,
                            sql="N",
                            chart="N",
                            web="Y",
                            executed="N",
                            start_time=question_start_time,
                            api_endpoint="",
                            api_params="",
                            api_response_validator=""
                        )
                    return {
                        "success": False,
                        "message": "Could not generate web search results. Please try rephrasing your request.",
                        "details": response_details
                    }
                
                # Log successful execution
                if not skip_user_question_log:
                    await run_in_threadpool(
                        create_user_question_log,
                        user_id=user_id,
                        agent_id=request.agent_id,
                        question=question_to_use,
                        sql="N",
                        chart=is_chart_available,
                        web="Y",
                        executed="Y",
                        start_time=question_start_time,
                        api_endpoint="",
                        api_params="",
                        api_response_validator=""
                    )
                
                try:
                    # Force to string for safe slicing
                    analysis_str = str(analysis_text) if analysis_text else ""
                    summary_text = analysis_str[:1000] if analysis_str else "Web search results found."
                    tb_cols_web = (
                        list(analysis_df.columns)
                        if analysis_df is not None and hasattr(analysis_df, "columns")
                        else []
                    )
                    talkback = await run_in_threadpool(
                        service.generate_continuation_talkback_cached,
                        question=question_to_use,
                        answer_context=summary_text,
                        columns=tb_cols_web,
                        n_items=3,
                        df=analysis_df if analysis_df is not None and len(analysis_df) > 0 else None,
                        use_qwen=settings.default_use_qwen_for_summary_and_web(),
                    )
                    response_details["talkback"] = talkback
                except Exception as e:
                    print(f"Error generating talkback for web: {e}")

                if analysis_df is not None and len(analysis_df) > 0:
                    df_records_web = clean_nan_values(analysis_df.to_dict("records"))
                    cols_web = list(analysis_df.columns)
                    formatted_cols_web = [
                        str(c).replace("_", " ").capitalize() for c in cols_web
                    ]
                    response_details["df"] = {
                        "agent_id": str(request.agent_id),
                        "headers": formatted_cols_web,
                        "data": df_records_web,
                        "chart_code": is_chart_available,
                    }

                return {
                    "success": True,
                    "message": "Web search completed successfully",
                    "details": response_details
                }
                
            except Exception as e:
                import traceback
                error_msg = f"Error performing analysis: {str(e)}"
                print(f"Error in @web analysis: {traceback.format_exc()}")
                
                if not skip_user_question_log:
                    await run_in_threadpool(
                        create_user_question_log,
                        user_id=user_id,
                        agent_id=request.agent_id,
                        question=question_to_use,
                        sql="N",
                        chart="N",
                        web="N",
                        executed="N",
                        start_time=question_start_time,
                        api_endpoint="",
                        api_params="",
                        api_response_validator=""
                    )
                
                return {
                    "success": False,
                    "message": error_msg,
                    "details": {"question": request.question, "error": str(e)}
                }

        
        df_records = []
        columns = []
        chart_code = "N"
        
        # Handle API datasource
        if datasource_to_use and datasource_to_use.upper() == "API":
            # Check if API parameters are already provided (reuse saved API parameters)
            if api_endpoint_to_use and api_params_to_use and api_response_validator_to_use:
                print("API parameters present - Reusing saved API call")
                
                # Reuse the provided API parameters directly
                api_endpoint = api_endpoint_to_use
                api_params = api_params_to_use
                api_response_validator = api_response_validator_to_use
                
                # Parse the saved params (they're stored as strings)
                import ast
                if isinstance(api_params, str) and api_params.strip():
                    try:
                        extracted_params = ast.literal_eval(api_params)
                    except:
                        extracted_params = {}
                else:
                    extracted_params = {}
                
                # Get api_info to determine if SAP (only when reusing parameters)
                apis_list = await run_in_threadpool(service.get_all_apis, request.agent_id)
                api_info = {}
                for data in apis_list:
                    if data.get("endpoint") == api_endpoint:
                        api_info = data
                        break
                
                is_sap = api_info.get("extra1", "N") if api_info else "N"
                api_name = api_info.get("name", "") if api_info else ""
                print(f"Is sap: {is_sap}")
                
                # Unified API call handling (for reused parameters - always single API)
                try:
                    if is_sap == "Y":
                        # SAP Flow
                        sap_result = await run_in_threadpool(
                            service.process_sap_query,
                            service_root=api_endpoint,
                            entity_set=api_info.get("base_params"),
                            user_query=question_to_process,
                            response_schema=api_info.get("response_schema"),
                            transformation_code=api_response_validator,
                            username="S4h_aa",
                            password="Welcome1",
                            verify_ssl=False,
                            api_query_params=extracted_params
                        )
                        df = sap_result.get("df")
                    else:
                        # Non-SAP API Flow
                        print("Reusing saved NON-SAP API call")
                        data, data2, df, data3 = await run_in_threadpool(
                            service.call_api_and_transform,
                            extracted_params,
                            api_endpoint,
                            "", 
                            api_response_validator, 
                            question_to_process
                        )
                    
                    if df is None:
                        if not skip_user_question_log:
                            await run_in_threadpool(
                                create_user_question_log,
                                user_id=user_id,
                                agent_id=request.agent_id,
                                question=question_to_use,
                                sql="N", chart="N", web="N", executed="N",
                                start_time=question_start_time,
                                api_endpoint=api_endpoint,
                                api_params=str(api_params),
                                api_response_validator=api_response_validator
                            )
                        return {"success": False, "message": "No results returned from API", "details": {"question": request.question}}
                except Exception as e:
                    if not skip_user_question_log:
                        await run_in_threadpool(
                            create_user_question_log,
                            user_id=user_id,
                            agent_id=request.agent_id,
                            question=question_to_use,
                            sql="N", chart="N", web="N", executed="N",
                            start_time=question_start_time,
                            api_endpoint=api_endpoint,
                            api_params=str(api_params),
                            api_response_validator=api_response_validator
                        )
                    return {"success": False, "message": f"Error calling API: {str(e)}", "details": {"question": request.question, "error": str(e)}}
            else:
                # Generate new API call
                print("Generating new API call")
                apis_list = await run_in_threadpool(service.get_all_apis, request.agent_id)
                
                if not apis_list:
                    if not skip_user_question_log:
                        await run_in_threadpool(
                            create_user_question_log, user_id=user_id, agent_id=request.agent_id,
                            question=question_to_use, sql="N", chart="N", web="N", executed="N",
                            start_time=question_start_time, api_endpoint="", api_params="", api_response_validator=""
                        )
                    return {"success": False, "message": "No APIs configured for this agent", "details": {"question": request.question}}
                
                # Check if multi-API logic is needed
                if len(apis_list) > 1:
                    print("Multi-API mode detected for single agent")
                    print("Selected apis", apis_list)
                    svc = service._ensure_api_chat_service()
                    selected_apis = await run_in_threadpool(svc._select_api_from_candidates, question_to_process, apis_list)
                    
                    if not selected_apis:
                        if not skip_user_question_log:
                            await run_in_threadpool(
                                create_user_question_log, user_id=user_id, agent_id=request.agent_id,
                                question=question_to_use, sql="N", chart="N", web="N", executed="N",
                                start_time=question_start_time, api_endpoint="", api_params="", api_response_validator=""
                            )
                        return {"success": False, "message": "No matching API found for this query", "details": {"question": request.question}}
                    
                    merge_result = await run_in_threadpool(svc.call_multiple_apis_and_merge, selected_apis=selected_apis, user_query=question_to_process)
                    
                    if not merge_result or not merge_result.get("success"):
                        return {"success": False, "message": merge_result.get("message", "No results returned from APIs") if merge_result else "No matching APIs found", "details": {}}
                    
                    df = merge_result.get("df")
                    api_details = merge_result.get("api_details", [])
                    api_endpoints_list = merge_result.get("api_endpoints", [])
                    api_params_list = merge_result.get("api_params", [])
                    api_validators_list = merge_result.get("api_validators", [])
                    
                    # Log question
                    if not skip_user_question_log:
                        await run_in_threadpool(
                            create_user_question_log, user_id=user_id, agent_id=request.agent_id, question=question_to_use,
                            sql="N", chart="N", web="N", executed="Y", start_time=question_start_time,
                            api_endpoint=json.dumps(api_endpoints_list), api_params=json.dumps(api_params_list), api_response_validator=json.dumps(api_validators_list)
                        )
                    
                    # Process and format multi-API response (align df + sql with multi-agent API shape)
                    _api_traces_ma: List[Dict[str, Any]] = []
                    for i, ep in enumerate(api_endpoints_list):
                        pr = api_params_list[i] if i < len(api_params_list) else ""
                        val = api_validators_list[i] if i < len(api_validators_list) else ""
                        _api_traces_ma.append(
                            {
                                "agent_id": str(request.agent_id),
                                "sub_question": question_to_process,
                                "result": _api_result_dict_for_envelope(ep, pr, val),
                            }
                        )
                    _sub_tasks_ma = [
                        {
                            "sub_question": question_to_process,
                            "agent_id": str(request.agent_id),
                            "datasource": "API",
                        }
                    ]
                    _sql_ma = _details_sql_envelope_api(
                        sub_tasks=_sub_tasks_ma,
                        api_traces=_api_traces_ma,
                        training_driven_pipeline=False,
                    )
                    if isinstance(df, list) and df and isinstance(df[0], pd.DataFrame):
                        agents_resp = []
                        for i, sub_df in enumerate(df):
                            sub_cols = list(sub_df.columns)
                            sub_records = clean_nan_values(sub_df.to_dict("records"))
                            agents_resp.append({
                                "agent_id": request.agent_id,
                                "headers": [str(c).replace('_', ' ').capitalize() for c in sub_cols],
                                "data": sub_records,
                                "api_name": api_details[i].get("name") if i < len(api_details) else "",
                                "chart_code": "N",
                            })
                        df_response = _df_api_multi_frame_payload(
                            agents_resp,
                            fallback_agent_id=str(request.agent_id),
                            multi_agent_mixed=False,
                        )
                    else:
                        sub_cols = list(df.columns) if df is not None and hasattr(df, "columns") else []
                        df_response = (
                            {
                                "agent_id": str(request.agent_id),
                                "headers": [str(c).replace('_', ' ').capitalize() for c in sub_cols],
                                "data": clean_nan_values(df.to_dict("records")),
                                "chart_code": "N",
                                "multi_agent_mixed": True,
                                "tabular_from_merge": bool(sub_cols),
                            }
                            if df is not None
                            else {
                                "agent_id": str(request.agent_id),
                                "headers": [],
                                "data": [],
                                "chart_code": "N",
                                "multi_agent_mixed": True,
                                "tabular_from_merge": False,
                            }
                        )

                    response_details = {
                        "sql": _sql_ma,
                        "df": df_response,
                        "question": question_to_process,
                        "original_question": question_to_use if request.inContext else None,
                        "api_endpoint": api_endpoints_list,
                        "api_params": api_params_list,
                        "api_response_validator": api_validators_list,
                    }
                    return JSONResponse(content=_to_json_safe({"success": True, "message": "Question processed successfully", "details": response_details}))
                else:
                    # Single-API mode (current existing logic)
                    api_info = await run_in_threadpool(
                        service.select_appropriate_api,
                        user_query=question_to_process,
                        available_apis=apis_list,
                        agent_id=request.agent_id
                    )
                    
                    if not api_info:
                        if not skip_user_question_log:
                            await run_in_threadpool(
                                create_user_question_log, user_id=user_id, agent_id=request.agent_id,
                                question=question_to_use, sql="N", chart="N", web="N", executed="N",
                                start_time=question_start_time, api_endpoint="", api_params="", api_response_validator=""
                            )
                        return {"success": False, "message": "No matching API found for this query", "details": {"question": request.question}}
                    
                    is_sap = api_info.get("extra1", "N")
                    try:
                        if is_sap == "Y":
                            sap_result = await run_in_threadpool(
                                service.process_sap_query,
                                service_root=api_info.get("endpoint"),
                                entity_set=api_info.get("base_params"),
                                user_query=question_to_process,
                                response_schema=api_info.get("response_schema"),
                                transformation_code="",
                                username="S4h_aa",
                                password="Welcome1",
                                verify_ssl=False
                            )
                            df = sap_result.get("df")
                            api_endpoint = api_info.get("endpoint", "")
                            api_params = str(sap_result.get("query_params"))
                            api_response_validator = sap_result.get("code", "")
                        else:
                            extracted_params = await run_in_threadpool(service.extract_api_params, question_to_process, api_info)
                            data, data2, df, data3 = await run_in_threadpool(
                                service.call_api_and_transform,
                                extracted_params,
                                api_info.get("endpoint"),
                                api_info.get("response_schema"),
                                "",
                                question_to_process
                            )
                            api_endpoint = api_info.get("endpoint", "")
                            api_params = str(extracted_params)
                            api_response_validator = data3
                        
                        api_name = api_info.get("name", "") if api_info else ""
                            
                        if df is None:
                            if not skip_user_question_log:
                                await run_in_threadpool(
                                    create_user_question_log, user_id=user_id, agent_id=request.agent_id,
                                    question=question_to_use, sql="N", chart="N", web="N", executed="N",
                                    start_time=question_start_time, api_endpoint=api_endpoint,
                                    api_params=api_params, api_response_validator=api_response_validator
                                )
                            return {"success": False, "message": "No results returned from API", "details": {"question": request.question}}
                        # Cap rows by agent max_row_count (from agent config)
                        try:
                            _max_rows = getattr(service, "_max_row_count", "1000")
                            api_max_rows = int(_max_rows) if _max_rows is not None else 1000
                        except (TypeError, ValueError):
                            api_max_rows = 1000
                        if hasattr(df, "head"):
                            df = df.head(api_max_rows)
                    except Exception as e:
                        if not skip_user_question_log:
                            await run_in_threadpool(
                                create_user_question_log, user_id=user_id, agent_id=request.agent_id,
                                question=question_to_use, sql="N", chart="N", web="N", executed="N",
                                start_time=question_start_time, api_endpoint=api_endpoint,
                                api_params=api_params, api_response_validator=api_response_validator
                            )
                        return {"success": False, "message": f"Error calling API: {str(e)}", "details": {"question": request.question, "error": str(e)}}
        
        # Handle Database datasource (original flow)
        else:
            # Ensure Oracle connection for this agent
            service._ensure_sql()
            
            # Check if SQL is already provided (reuse saved SQL)
            if sql_to_use:
                print("SQL Present - Reusing saved SQL")
                print("sql_to_use", sql_to_use)
                generated_sql = sql_to_use
            else:
                # Generate SQL
                print("Creating SQL.....")
                generated_sql = await run_in_threadpool(service.generate_sql_cached, question=question_to_process)
            
            if not generated_sql:
                # Log error for SQL generation failure
                if not skip_user_question_log:
                    await run_in_threadpool(
                        create_user_question_log,
                        user_id=user_id,
                        agent_id=request.agent_id,
                        question=question_to_use,
                        sql="N",
                        chart="N",
                        web="N",
                        executed="N",
                        start_time=question_start_time,
                        api_endpoint="",
                        api_params="",
                        api_response_validator=""
                    )
                return {
                    "success": False,
                    "message": "Failed to generate SQL for the question",
                    "details": {"question": request.question}
                }
            print("generated_sql", generated_sql)
            # Validate SQL
            is_valid = await run_in_threadpool(service.is_sql_valid_cached, sql=generated_sql)
            if not is_valid:
                # Log error for invalid SQL
                if not skip_user_question_log:
                    await run_in_threadpool(
                        create_user_question_log,
                        user_id=user_id,
                        agent_id=request.agent_id,
                        question=question_to_use,
                        sql=generated_sql,
                        chart="N",
                        web="N",
                        executed="N",
                        start_time=question_start_time,
                        api_endpoint="",
                        api_params="",
                        api_response_validator=""
                    )
                _d_inv: Dict[str, Any] = {
                    "question": request.question,
                    "sql": _details_sql_envelope_db(
                        agent_id=str(request.agent_id),
                        sub_question=question_to_process,
                        sql_text=generated_sql,
                        training_driven_pipeline=False,
                    ),
                }
                _d_inv["talkback"] = await _talkback_for_db_sql_error(
                    service,
                    generated_sql,
                    no_rows=False,
                    question=_pick_question_for_error_talkback(
                        question_to_process, question_to_use, getattr(request, "question", None)
                    ),
                )
                return {
                    "success": False,
                    "message": (
                        "We could not use that as a valid query. Rephrase your question, or use one of "
                        "the suggested questions."
                    ),
                    "details": _d_inv,
                }
            
            # Run SQL
            try:
                df = await run_in_threadpool(service.run_sql_cached, sql=generated_sql)
                
                # Check if SQL returned no data
                if df is None:
                    # Log error for no results from SQL
                    if not skip_user_question_log:
                        await run_in_threadpool(
                            create_user_question_log,
                            user_id=user_id,
                            agent_id=request.agent_id,
                            question=question_to_use,
                            sql=generated_sql,
                            chart="N",
                            web="N",
                            executed="N",
                            start_time=question_start_time,
                            api_endpoint="",
                            api_params="",
                            api_response_validator=""
                        )
                    _d_empty: Dict[str, Any] = {
                        "question": request.question,
                        "sql": _details_sql_envelope_db(
                            agent_id=str(request.agent_id),
                            sub_question=question_to_process,
                            sql_text=generated_sql,
                            training_driven_pipeline=False,
                        ),
                    }
                    _d_empty["talkback"] = await _talkback_for_db_sql_error(
                        service,
                        generated_sql,
                        no_rows=True,
                        question=_pick_question_for_error_talkback(
                            question_to_process, question_to_use, getattr(request, "question", None)
                        ),
                    )
                    return {
                        "success": False,
                        "message": "No results returned for that query. Try rephrasing your question.",
                        "details": _d_empty,
                    }
            except Exception as e:
                if isinstance(e, QueryTimeoutError):
                    if not skip_user_question_log:
                        await run_in_threadpool(
                            create_user_question_log,
                            user_id=user_id,
                            agent_id=request.agent_id,
                            question=question_to_use,
                            sql=generated_sql,
                            chart="N",
                            web="N",
                            executed="N",
                            start_time=question_start_time,
                            api_endpoint="",
                            api_params="",
                            api_response_validator=""
                        )
                    return {
                        "success": False,
                        "message": "The query took too long and was stopped. Try a more specific question.",
                        "details": {
                            "question": request.question,
                            "error": str(e),
                        },
                    }
                # Log error for SQL execution failure
                if not skip_user_question_log:
                    await run_in_threadpool(
                        create_user_question_log,
                        user_id=user_id,
                        agent_id=request.agent_id,
                        question=question_to_use,
                        sql=generated_sql,
                        chart="N",
                        web="N",
                        executed="N",
                        start_time=question_start_time,
                        api_endpoint="",
                        api_params="",
                        api_response_validator=""
                    )
                return {
                    "success": False,
                    "message": f"Error running SQL: {str(e)}",
                    "details": {
                        "sql": _details_sql_envelope_db(
                            agent_id=str(request.agent_id),
                            sub_question=question_to_process,
                            sql_text=generated_sql,
                            training_driven_pipeline=False,
                        ),
                        "error": str(e),
                    },
                }
        
        # Convert DataFrame or List to list of dicts
        df_records = []
        columns = []
        
        # Handle DataFrame (standard case)
        if hasattr(df, 'to_dict') and hasattr(df, 'columns'):
            if not df.empty:
                df_records = df.to_dict('records')
                # Clean NaN values to make JSON serializable
                df_records = clean_nan_values(df_records)
                columns = list(df.columns)
        
        # Handle List (fallback or alternative return type)
        elif isinstance(df, list):
            df_records = df
            # Clean NaN values to make JSON serializable
            df_records = clean_nan_values(df_records)
            if df and isinstance(df[0], dict):
                columns = list(df[0].keys())

        formatted_columns = [str(col).replace('_', ' ').capitalize() for col in columns]

        if datasource_to_use and datasource_to_use.upper() == "API":
            _api_tr = [
                {
                    "agent_id": str(request.agent_id),
                    "sub_question": question_to_process,
                    "result": _api_result_dict_for_envelope(
                        api_endpoint, api_params, api_response_validator
                    ),
                }
            ]
            _sub_api = [
                {
                    "sub_question": question_to_process,
                    "agent_id": str(request.agent_id),
                    "datasource": "API",
                }
            ]
            details_sql_str = _details_sql_envelope_api(
                sub_tasks=_sub_api,
                api_traces=_api_tr,
                training_driven_pipeline=False,
            )
        else:
            details_sql_str = _details_sql_envelope_db(
                agent_id=str(request.agent_id),
                sub_question=question_to_process,
                sql_text=generated_sql if generated_sql else "",
                training_driven_pipeline=False,
            )

        response_details = {
            "sql": details_sql_str,
            "df": {
                "agent_id": str(request.agent_id),
                "headers": formatted_columns,
                "data": df_records,
                "api_name": api_name,
            },
            "question": question_to_process,
            "original_question": question_to_use if request.inContext else None,
            "api_endpoint": api_endpoint if api_endpoint else "",
            "api_params": api_params if api_params else "",
            "api_response_validator": api_response_validator if api_response_validator else "",
        }
        
        # Generate chart only when tabular result has fewer than 100 rows (no is_chart flag).
        should_generate_chart = bool(
            df_records and len(columns) > 0 and len(df_records) < 101
        )
        
        if should_generate_chart:
            # Generate JSON chart structure
            try:
                # Import required modules first
                import json as json_module
                # Check if chart_json is provided as a string (Python code or JSON)
                if chart_code_to_use and chart_code_to_use.strip() and chart_code_to_use != "N":
                    print("Reusing provided chart JSON code")
                    json_code = chart_code_to_use
                    df_for_code = pd.DataFrame(df_records)
                    # Execute the generated code in a safe namespace
                    exec_globals, local_vars = _chart_code_exec_context(df_for_code)
                    exec(json_code, exec_globals, local_vars)

                    # Extract the chart_json variable created by the code
                    chart_json = _chart_json_from_exec_locals(local_vars)
                else:
                    # Generate new Python code if chart_json is not provided
                    # Only generate if we have dataframe data
                    if df_records and len(columns) > 0 and len(df_records) < 101:
                        json_code = service.generate_json_code_cached(
                            question=question_to_process,
                            columns=columns,
                            model="gpt-4.1"
                        )
                        
                        df_for_code = pd.DataFrame(df_records)
                        # Execute the generated code in a safe namespace
                        exec_globals, local_vars = _chart_code_exec_context(df_for_code)
                        exec(json_code, exec_globals, local_vars)
                        
                        # Extract the chart_json variable created by the code
                        chart_json = _chart_json_from_exec_locals(local_vars)
                    else:
                        chart_json = None
                        json_code = "N"
                
                # # Validate and ensure the structure matches Plotly format
                # if chart_json is None or not isinstance(chart_json, dict):
                #     # Fallback: create default Plotly structure if we have data
                #     if df_records and len(columns) > 0 and len(df_records) < 101:
                #         df_for_code = pd.DataFrame(df_records)
                #         chart_json = service._create_default_plotly_json(df_for_code, columns, question_to_process)
                #         json_code = "N"  # Default chart, no code
                #     else:
                #         # No data and no valid chart_json - skip chart
                #         chart_json = None
                #         json_code = "N"
                # else:
                #     # Ensure it has Plotly structure (data and layout)
                #     if "data" not in chart_json or "layout" not in chart_json:
                #         if df_records and len(columns) > 0 and len(df_records) < 101:
                #             df_for_code = pd.DataFrame(df_records)
                #             chart_json = service._create_default_plotly_json(df_for_code, columns, question_to_process)
                #             json_code = "N"
                #         else:
                #             chart_json = None
                #             json_code = "N"
                #     else:
                #         # Ensure layout has title and axes
                #         if "title" not in chart_json.get("layout", {}):
                #             chart_json.setdefault("layout", {})["title"] = question_to_process[:50] if len(question_to_process) > 50 else question_to_process
                #         if "xaxis" not in chart_json.get("layout", {}):
                #             chart_json.setdefault("layout", {})["xaxis"] = {"title": columns[0] if len(columns) > 0 else ""}
                #         if "yaxis" not in chart_json.get("layout", {}) and len(columns) > 1:
                #             chart_json.setdefault("layout", {})["yaxis"] = {"title": columns[1] if len(columns) > 1 else ""}
                
                # Add chart to response if we have valid chart_json
                if chart_json is not None:
                    # Clean NaN values to make JSON serializable
                    chart_json = clean_nan_values(chart_json)
                    response_details["chart"] = chart_json
                    response_details["chart_code"] = json_code 
                    chart_code = json_code
                else:
                    chart_code = "N"
                
            except Exception as e:
                # If JSON code generation/execution fails, fall back to default Plotly structure
                print(f"Error generating chart: {e}")
                import traceback
                print(traceback.format_exc())
                if df_records and len(columns) > 0 and len(df_records) < 101:
                    df_for_code = pd.DataFrame(df_records)
                    chart_json = {}
                    # Clean NaN values to make JSON serializable
                    chart_json = clean_nan_values(chart_json)
                    response_details["chart"] = chart_json
                    response_details["chart_error"] = str(e)
                    chart_code = "N"  # Chart generation failed
                else:
                    chart_code = "N"
        
        if isinstance(response_details.get("df"), dict):
            response_details["df"]["chart_code"] = chart_code

        _cached_sq_single = getattr(request, "sql_query_summaries_cached", None)
        if _cached_sq_single:
            response_details["sql_query_summaries"] = _cached_sq_single
        else:
            try:
                await run_in_threadpool(
                    service.merge_sql_query_summaries_into_details_cached,
                    response_details,
                    question_to_process,
                )
            except Exception as _sq_sum_exc:
                print(f"sql_query_summaries (single-agent): {_sq_sum_exc}")
        
        # Generate summary
        # if df_records:
        #     summary = service.generate_summary(
        #         question=question_to_use,
        #         df=df_records,
        #         model="gpt-4.1"
        #     )
        #     response_details["summary"] = summary
        
        # Log successful execution
        dataframe_str = str(columns) if columns else "N"
        _sql_for_log_col = details_sql_str if details_sql_str else (generated_sql if generated_sql else "N")
        log_id = None
        if not skip_user_question_log:
            log_id = await run_in_threadpool(
                create_user_question_log,
                user_id=request.user_id,
                agent_id=request.agent_id,
                question=question_to_use,
                sql=_sql_for_log_col,
                chart=chart_code,
                web="N",
                executed="Y",
                start_time=question_start_time,
                api_endpoint=api_endpoint if api_endpoint else "",
                api_params=api_params if api_params else "",
                api_response_validator=api_response_validator if api_response_validator else "",
                sql_query_summaries=response_details.get("sql_query_summaries"),
                dataframe=dataframe_str
            )
        
        try:
            context_str = f"Columns: {str(columns)}" if columns else "No data returned."
            if df_records and len(df_records) > 0 and (settings.default_use_qwen_for_summary_and_web() and _qwen_configured()):
                # Convert records to string for safe slicing
                records_str = str(df_records)
                context_str += f"\nSample data: {records_str[:500]}"
            talkback_tb_df = None
            if hasattr(df, "to_dict") and hasattr(df, "columns") and not getattr(df, "empty", True):
                talkback_tb_df = df
            elif isinstance(df, list) and df and isinstance(df[0], dict):
                try:
                    talkback_tb_df = pd.DataFrame(df)
                except Exception:
                    talkback_tb_df = None
            _sql_talkback = (
                generated_sql
                if generated_sql and str(generated_sql).strip()
                else details_sql_str
            )
            talkback = await run_in_threadpool(
                service.generate_continuation_talkback_cached,
                question=question_to_use,
                answer_context=context_str,
                columns=columns,
                n_items=3,
                df=talkback_tb_df,
                sql=_sql_talkback,
                api_endpoint=api_endpoint,
                api_params=api_params,
                api_response_validator=api_response_validator,
                use_qwen=settings.default_use_qwen_for_summary_and_web(),
            )
            response_details["talkback"] = talkback
        except Exception as e:
            print(f"Error generating talkback: {e}")

        if not skip_user_question_log and log_id:
            _totals = get_global_usage_totals()
            _input_tokens = int(_totals.get("input_tokens", 0) or 0)
            _output_tokens = int(_totals.get("output_tokens", 0) or 0)
            _total_tokens = int(_totals.get("total_tokens", (_input_tokens + _output_tokens)) or 0)
            asyncio.create_task(
                run_in_threadpool(
                    update_user_question_log_tokens,
                    log_id,
                    _input_tokens,
                    _output_tokens,
                    _total_tokens,
                )
            )

        return {
            "success": True,
            "message": "Question processed successfully",
            "details": response_details
        }

    except Exception as e:
        # Log exception error
        try:
            if not skip_user_question_log:
                await run_in_threadpool(
                    create_user_question_log,
                    user_id=user_id,
                    agent_id=request.agent_id,
                    question=question_to_use,
                    sql=generated_sql if generated_sql else "N",
                    chart="N",
                    web="N",
                    executed="N",
                    start_time=question_start_time,
                    api_endpoint=api_endpoint if api_endpoint else "",
                    api_params=api_params if api_params else "",
                    api_response_validator=api_response_validator if api_response_validator else ""
                )
        except:
            pass  # Don't fail if logging fails
        return {
            "success": False,
            "message": f"Error processing question: {str(e)}",
            "details": {"error": str(e)}
        }



_FILE_CMD_RE = re.compile(r"(?<![.\w@])(@file|@find|@analyze|@analyse)(?![\w.])", re.IGNORECASE)
_PRIOR_CMD_RE = re.compile(r"(?<![.\w@])(@askme|@web)(?![\w.])", re.IGNORECASE)


def _detect_file_commands(question: str) -> Dict[str, Any]:
    text = str(question or "")
    file_hits = [m.group(1).lower() for m in _FILE_CMD_RE.finditer(text)]
    file_cmds = set(file_hits)
    prior_cmds = set(m.group(1).lower() for m in _PRIOR_CMD_RE.finditer(text))
    has_file_cmd = bool(file_cmds)
    has_conflict = bool(has_file_cmd and prior_cmds)

    file_clause = ""
    non_file_clause = text.strip()
    first_file_match = _FILE_CMD_RE.search(text)
    if first_file_match:
        file_clause = text[first_file_match.start() :].strip()
        non_file_clause = text[: first_file_match.start()].strip()

    return {
        "has_file_cmd": has_file_cmd,
        "has_non_file_clause": bool(non_file_clause),
        "has_conflict": has_conflict,
        "file_command": (file_hits[0] if file_hits else None),
        "file_clause": file_clause,
        "non_file_clause": non_file_clause,
    }


async def _ask_file_core(
    user_id: str,
    agent_id: Union[str, List[str]],
    question: Optional[str],
    file_content: Optional[bytes] = None,
    filename: Optional[str] = None,
    *,
    current_user: Optional[User],
    skip_log: bool = False,
) -> Dict[str, Any]:
    """
    Shared entrypoint for talkback/orchestrator to invoke ask-file behavior.
    Keeps endpoint semantics by delegating to `ask_file`.
    """
    _ = skip_log  # The route currently owns logging behavior.
    upload: Optional[UploadFile] = None
    if file_content is not None and filename:
        upload = UploadFile(filename=filename, file=io.BytesIO(file_content))
    return await ask_file(
        user_id=str(user_id),
        agent_id=json.dumps(agent_id) if isinstance(agent_id, list) else str(agent_id),
        question=question,
        datasource="file",
        file=upload,
        current_user=current_user,  # type: ignore[arg-type]
    )


@router.post("/ask-file")
async def ask_file(
    user_id: str = Form(...),
    agent_id: str = Form(...),
    question: Optional[str] = Form(None),
    datasource: str = Form(...),
    file: Optional[UploadFile] = File(None),
    current_user: User = Depends(get_current_user),
):
    """
    Ask a file-based question to the agent.
    Handles document operations (@file, @analyze, @find).
   
    Form Data (multipart/form-data):
        user_id: User ID (required)
        agent_id: Agent ID (required)
        question: Question with @file, @analyze, or @find prefix (optional)
        datasource: Data source type (optional)
        file: File upload (optional) - when provided, file is analyzed using @analyze
    """
    import io
    import uuid
   
    # Track start time for logging
    question_start_time = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
    temp_file_path = None
   
    try:
        print(f"[FILE] /ask-file start: user_id={user_id}, agent_id={agent_id}, datasource={datasource}")
        # Validate datasource - only "file" is accepted
        if not datasource or datasource.lower() != "file":
            return {
                "success": False,
                "message": f"Invalid datasource: '{datasource}'. Only 'file' datasource is supported for this API.",
                "details": {"datasource": datasource}
            }

        auth_uid = str(getattr(current_user, "id", None) or user_id or "").strip()
        
        # Parse agent_id input (could be a string, a comma-separated list, or a JSON array)
        raw_agent_id = str(agent_id).strip()
        preferred_agent_ids = []
        if raw_agent_id.startswith("[") and raw_agent_id.endswith("]"):
            try:
                import json
                preferred_agent_ids = json.loads(raw_agent_id)
                if not isinstance(preferred_agent_ids, list):
                    preferred_agent_ids = [str(preferred_agent_ids)]
            except:
                preferred_agent_ids = [raw_agent_id]
        elif "," in raw_agent_id:
            preferred_agent_ids = [a.strip() for a in raw_agent_id.split(",") if a.strip()]
        else:
            preferred_agent_ids = [raw_agent_id]

        from app.services.agent_config import resolve_all_file_capable_agent_ids_for_user
        file_agent_ids = resolve_all_file_capable_agent_ids_for_user(
            auth_uid, [str(aid).strip() for aid in preferred_agent_ids]
        )
        
        if not file_agent_ids:
            return {
                "success": False,
                "message": "No file-capable agents are available for this request.",
                "details": {
                    "needs_clarification": True,
                    "clarification_message": (
                        "Please select an agent that supports file/document datasource."
                    ),
                    "error_code": "file_agent_required",
                },
            }
        
        # We use all resolved agents for retrieval, but we need one agent's config 
        # (e.g. for OpenAI API keys, chroma_path). We take the first one.
        # Since document storage is shared (file_sys_metadata.db / file_vectors_db),
        # as long as the agent has the FILE datasource, they can access the shared store.
        agent_id = file_agent_ids[0] if len(file_agent_ids) == 1 else file_agent_ids
        primary_agent_id = file_agent_ids[0]
        
        # Get parameters from request
        question_to_use = question or ""
       
        # Initialize flags
        is_uploaded_file = file is not None
        uploaded_filename = file.filename if file else None
        uploaded_extension = uploaded_filename.split('.')[-1].lower() if uploaded_filename else None
        print(f"[FILE] /ask-file request parsed: uploaded={is_uploaded_file}, filename={uploaded_filename}, question={question_to_use}")
        if is_uploaded_file:
            supported_extensions = ['pdf', 'docx', 'doc', 'xlsx', 'xls', 'csv', 'txt', 'png', 'jpg', 'jpeg']
            if not uploaded_extension or uploaded_extension not in supported_extensions:
                return {
                    "success": False,
                    "message": f"Unsupported file type: {uploaded_extension or 'unknown'}. Supported types: {', '.join(supported_extensions)}",
                    "details": {"filename": uploaded_filename, "extension": uploaded_extension}
                }
        # Resolve prefix logic
        if is_uploaded_file:
            # When file is uploaded, default to @analyze
            if question_to_use:
                # Also handle @analyse (British spelling)
                has_prefix = any(prefix in question_to_use.lower() for prefix in ["@file", "@analyze", "@analyse", "@find"])
                if not has_prefix:
                    question_to_use = f"@analyze {question_to_use}"
                # Normalize @analyse to @analyze
                question_to_use = question_to_use.replace("@analyse", "@analyze")
            else:
                question_to_use = "@analyze"
        else:
            # Original flow: default to @file if no prefix
            if question_to_use:
                has_prefix = any(prefix in question_to_use.lower() for prefix in ["@file", "@analyze", "@analyse", "@find"])
                if not has_prefix:
                    question_to_use = f"@file {question_to_use}"
                # Normalize @analyse to @analyze
                question_to_use = question_to_use.replace("@analyse", "@analyze")
       
        # Get agent config
        def get_agent_config_safe():
            try:
                # Use primary_agent_id for config fetching as agent_id might be a list
                return get_agent_config(agent_id=primary_agent_id)
            except Exception as e:
                print(f"Error fetching agent config for {primary_agent_id}: {e}")
                return None

        agent_config = await run_in_threadpool(get_agent_config_safe)
        if not agent_config:
            await run_in_threadpool(
                create_user_question_log,
                user_id=user_id,
                agent_id=agent_id,
                question=question_to_use,
                sql="N",
                chart="N",
                web="N",
                executed="N",
                start_time=question_start_time,
                api_endpoint="",
                api_params="",
                api_response_validator=""
            )
            return {
                "success": False,
                "message": f"Agent '{agent_id}' not found",
                "details": {}
            }
       
        # Handle document operations - check if module is available
        if not DOCUMENT_ASSISTANT_AVAILABLE:
            await run_in_threadpool(
                create_user_question_log,
                user_id=user_id,
                agent_id=agent_id,
                question=question,
                sql="N",
                chart="N",
                web="N",
                executed="N",
                start_time=question_start_time,
                api_endpoint="",
                api_params="",
                api_response_validator=""
            )
            return {
                "success": False,
                "message": "Document assistant module is not available",
                "details": {"question": question}
            }
       
        # Check if OpenAI API key is available
        api_key = agent_config.get("api_key") if agent_config else None
        if not api_key:
            await run_in_threadpool(
                create_user_question_log,
                user_id=user_id,
                agent_id=agent_id,
                question=question or "",
                sql="N",
                chart="N",
                web="N",
                executed="N",
                start_time=question_start_time,
                api_endpoint="",
                api_params="",
                api_response_validator=""
            )
            return {
                "success": False,
                "message": "OpenAI API key not configured for this agent (required for embeddings / file RAG; Claude use_claude does not replace this)",
                "details": {"question": question}
            }

        # Resolve AskmeService by agent_id only. Do not use Depends(get_askme_service) here:
        # that dependency exposes get_askme_service's optional `config: dict` parameter to
        # multipart parsing; clients sending config= or config="" get 422 (dict_type).
        def get_service_safe():
            try:
                return get_askme_service(agent_id=agent_id)
            except Exception as e:
                print(f"Error getting askme service: {e}")
                return None

        service = await run_in_threadpool(get_service_safe)
        if not service:
            await run_in_threadpool(
                create_user_question_log,
                user_id=user_id,
                agent_id=agent_id,
                question=question or "",
                sql="N",
                chart="N",
                web="N",
                executed="N",
                start_time=question_start_time,
                api_endpoint="",
                api_params="",
                api_response_validator="",
            )
            return {
                "success": False,
                "message": f"Could not load Askme service for agent '{agent_id}'",
                "details": {},
            }
       
        # ==========================================
        # Initialize DocumentAssistant
        # Initialize DocumentAssistant
        # ==========================================
        try:
            # Shared document store (same DB + Chroma + disk roots for all FILE-capable agents)
            def init_doc_assistant():
                return _get_or_create_doc_assistant(
                    agent_id=agent_id,
                    api_key=str(api_key) if api_key else None,
                    upload_folder=FILE_UPLOAD_DISK_ROOT,
                    image_folder=FILE_IMAGE_DISK_ROOT,
                    extracted_folder=FILE_EXTRACT_DISK_ROOT,
                    db_file=FILE_SYS_METADATA_DB,
                    chroma_path=FILE_VECTORS_DB_DIR,
                    claude_api_key=str(agent_config.get("claude_api_key") or ""),
                )

            doc_assistant = await run_in_threadpool(init_doc_assistant)
            print("[FILE] /ask-file document assistant initialized")
            # ==========================================
            # BRANCH 1: File uploaded - Save, Ingest, and Analyze
            # BRANCH 1: File uploaded - Save, Ingest, and Analyze
            # ==========================================
            if is_uploaded_file and file:
                try:
                    print("[FILE] /ask-file branch=upload+analyze")
                    # Read uploaded file content
                    file_content = await file.read()
                    file_bytes = io.BytesIO(file_content)
                    file_bytes.name = uploaded_filename
                    
                    # -----------------------------------------------
                    # STEP A: Ingest into ChromaDB + SQLite via the SAME
                    #         pipeline used by training.py upload-files
                    # -----------------------------------------------
                    ingestion_result = None
                    ask_file_folder = "ask_file_uploads"
                    saved_file_path = ""
                    try:
                        # Register the folder in the DB so get-all-files-folders can find it
                        # (same as scan_folder_path / create_folder does)
                        await run_in_threadpool(
                            doc_assistant.db_service.save_folder,
                            "Ask File Uploads",   # folder_name (display name)
                            ask_file_folder,       # folder_path (lookup key)
                            "",                    # parent_path
                            ""                     # full_path
                        )
                        
                        # Prepare a BytesIO for ingestion (upload_document reads .filename)
                        ingest_bytes = io.BytesIO(file_content)
                        ingest_bytes.filename = uploaded_filename
                        
                        ingestion_result = await run_in_threadpool(
                            doc_assistant.upload_document,
                            ingest_bytes,
                            ask_file_folder
                        )
                        if ingestion_result and ingestion_result.get("success"):
                            print(f"[ask-file] Ingestion OK: file_id={ingestion_result.get('file_id')}, chunks={ingestion_result.get('chunks_created')}")
                            # Retrieve saved file path from DB for response
                            file_info = await run_in_threadpool(
                                doc_assistant.db_service.get_file,
                                ingestion_result.get("file_id")
                            )
                            if file_info and len(file_info) > 4:
                                saved_file_path = file_info[4]  # file_path column
                        else:
                            msg = ingestion_result.get("message", "unknown") if ingestion_result else "no result"
                            print(f"[ask-file] Ingestion skipped/failed: {msg}")
                    except Exception as ingest_err:
                        print(f"[ask-file] Ingestion error (non-fatal): {ingest_err}")
                    
                    # -----------------------------------------------
                    # STEP C: Extract content & analyze (existing logic)
                    # -----------------------------------------------
                    # Reset BytesIO for content extraction
                    file_bytes.seek(0)
                    
                    # -----------------------------------------------
                    # STEP A: Ingest into ChromaDB + SQLite via the SAME
                    #         pipeline used by training.py upload-files
                    # -----------------------------------------------
                    ingestion_result = None
                    ask_file_folder = "ask_file_uploads"
                    saved_file_path = ""
                    try:
                        # Register the folder in the DB so get-all-files-folders can find it
                        # (same as scan_folder_path / create_folder does)
                        await run_in_threadpool(
                            doc_assistant.db_service.save_folder,
                            "Ask File Uploads",   # folder_name (display name)
                            ask_file_folder,       # folder_path (lookup key)
                            "",                    # parent_path
                            ""                     # full_path
                        )
                        
                        # Prepare a BytesIO for ingestion (upload_document reads .filename)
                        ingest_bytes = io.BytesIO(file_content)
                        ingest_bytes.filename = uploaded_filename
                        
                        ingestion_result = await run_in_threadpool(
                            doc_assistant.upload_document,
                            ingest_bytes,
                            ask_file_folder
                        )
                        if ingestion_result and ingestion_result.get("success"):
                            print(f"[ask-file] Ingestion OK: file_id={ingestion_result.get('file_id')}, chunks={ingestion_result.get('chunks_created')}")
                            # Retrieve saved file path from DB for response
                            file_info = await run_in_threadpool(
                                doc_assistant.db_service.get_file,
                                ingestion_result.get("file_id")
                            )
                            if file_info and len(file_info) > 4:
                                saved_file_path = file_info[4]  # file_path column
                        else:
                            msg = ingestion_result.get("message", "unknown") if ingestion_result else "no result"
                            print(f"[ask-file] Ingestion skipped/failed: {msg}")
                    except Exception as ingest_err:
                        print(f"[ask-file] Ingestion error (non-fatal): {ingest_err}")
                    
                    # -----------------------------------------------
                    # STEP C: Extract content & analyze (existing logic)
                    # -----------------------------------------------
                    # Reset BytesIO for content extraction
                    file_bytes.seek(0)
                    
                    # Extract content using ContentExtractor
                    content_extractor = ContentExtractor()
                    
                    # Extract content based on file type
                    extractors = {
                        'pdf': content_extractor.extract_from_pdf,
                        'xlsx': content_extractor.extract_from_excel,
                        'xls': content_extractor.extract_from_excel,
                        'csv': content_extractor.extract_from_csv,
                        'png': content_extractor.extract_from_image,
                        'jpg': content_extractor.extract_from_image,
                        'jpeg': content_extractor.extract_from_image,
                        'docx': content_extractor.extract_from_docx,
                        'doc': content_extractor.extract_from_doc,
                        'txt': content_extractor.extract_from_txt
                    }
                    
                    extractor = extractors.get(uploaded_extension)
                    if not extractor:
                        await run_in_threadpool(
                            create_user_question_log,
                            user_id=user_id,
                            agent_id=agent_id,
                            question=question or "",
                            sql="N",
                            chart="N",
                            web="N",
                            executed="N",
                            start_time=question_start_time,
                            api_endpoint="",
                            api_params="",
                            api_response_validator=""
                        )
                        return {
                            "success": False,
                            "message": f"Unsupported file type for analysis: {uploaded_extension}",
                            "details": {"filename": uploaded_filename}
                        }
                    
                    # Run extraction in threadpool as it is CPU bound
                    content = await run_in_threadpool(extractor, file_bytes)
                    
                    if not content or (isinstance(content, str) and (content.startswith("Error") or content.startswith("Unsupported"))):
                        await run_in_threadpool(
                            create_user_question_log,
                            user_id=user_id,
                            agent_id=agent_id,
                            question=question or "",
                            sql="N",
                            chart="N",
                            web="N",
                            executed="N",
                            start_time=question_start_time,
                            api_endpoint="",
                            api_params="",
                            api_response_validator=""
                        )
                        return {
                            "success": False,
                            "message": f"Failed to extract content from file: {content or 'Unknown error'}",
                            "details": {"filename": uploaded_filename}
                        }
                    
                    # Analyze using AIAnalyzer
                    ai_analyzer = AIAnalyzer(str(api_key))
                    
                    # If question is provided, use it for context; otherwise just analyze
                    analysis_query = question_to_use if question_to_use and not question_to_use.startswith("@analyze") else uploaded_filename
                    
                    # Run analysis in threadpool as it calls OpenAI API
                    analysis = await run_in_threadpool(
                        ai_analyzer.analyze_document,
                        content,
                        analysis_query
                    )
                    
                    # Log successful execution
                    await run_in_threadpool(
                        create_user_question_log,
                        user_id=user_id,
                        agent_id=agent_id,
                        question=question or "",
                        sql="N",
                        chart="N",
                        web="N",
                        executed="Y",
                        start_time=question_start_time,
                        api_endpoint="",
                        api_params="",
                        api_response_validator=""
                    )
                    
                    # Build ingestion metadata for response
                    ingestion_file_id = ingestion_result.get("file_id", "") if ingestion_result and ingestion_result.get("success") else ""
                    ingestion_chunks = ingestion_result.get("chunks_created", 0) if ingestion_result and ingestion_result.get("success") else 0
                    
                    # Build response details
                    response_details = {
                        "question": question,
                        "document_query": question_to_use,
                        "sql": "N",
                        "api_endpoint": "",
                        "api_params": "",
                        "api_response_validator": "",
                        "document_result_type": "document_analysis",
                        "document_analysis": {
                            "filename": uploaded_filename,
                            "file_type": uploaded_extension,
                            "folder_path": ask_file_folder,
                            "relative_path": uploaded_filename,
                            "folder_detected": None,
                            "analysis": analysis or "AI analysis completed",
                            "all_images": [],
                            "file_path": saved_file_path,
                            "file_size": len(file_content),
                            "upload_date": datetime.now().isoformat(),
                            "is_temporary_upload": False,
                            "ingested": bool(ingestion_file_id),
                            "file_id": ingestion_file_id,
                            "chunks_created": ingestion_chunks
                        }
                    }

                    try:
                        # Ensure context is string and not too long
                        ctx = str(analysis) if analysis else "File analyzed."
                        talkback = await run_in_threadpool(
                            service.generate_continuation_talkback_cached,
                            question=question_to_use,
                            answer_context=ctx[:1000],
                            columns=[],
                            n_items=3,
                            use_qwen=settings.default_use_qwen_for_summary_and_web(),
                        )
                        response_details["talkback"] = talkback
                    except Exception as e:
                        print(f"Error generating talkback for upload analysis: {e}")

                    return {
                        "success": True,
                        "message": "File analyzed successfully",
                        "details": response_details
                    }
                        
                except Exception as e:
                    import traceback
                    error_msg = f"Error analyzing uploaded file: {str(e)}"
                    print(f"Error in file analysis: {traceback.format_exc()}")
                    
                    await run_in_threadpool(
                        create_user_question_log,
                        user_id=user_id,
                        agent_id=agent_id,
                        question=question or "",
                        sql="N",
                        chart="N",
                        web="N",
                        executed="N",
                        start_time=question_start_time,
                        api_endpoint="",
                        api_params="",
                        api_response_validator=""
                    )
                    
                    return {
                        "success": False,
                        "message": error_msg,
                        "details": {"filename": uploaded_filename, "error": str(e)}
                    }
            
            # ==========================================
            # BRANCH 2: No file uploaded - Normal DocumentAssistant flow
            # ==========================================
            print("[FILE] /ask-file branch=query-only")
            
            # Build query string - add default @file prefix if no prefix present
            query_str = str(question_to_use) if question_to_use else ""
            
            # Check if query has a prefix (@file, @find, @analyze)
            if query_str and not any(query_str.startswith(prefix) for prefix in ["@file", "@find", "@analyze"]):
                # Default to @file prefix
                query_str = f"@file {query_str}"
            
            # Call handle_query with the query string in threadpool (heavy operation)
            doc_results = await run_in_threadpool(doc_assistant.handle_query, query_str)
            print(f"[FILE] /ask-file handle_query done: type={doc_results.get('type', 'unknown')}")
           
            # Determine result type
            result_type = doc_results.get('type', 'unknown')
           
            # Build response based on result type
            response_details = {
                "question": question,
                "document_query": query_str,
                "sql": "N",
                "api_endpoint": "",
                "api_params": "",
                "api_response_validator": "",
                "document_result_type": result_type
            }
           
            # Add result data based on type
            if result_type == 'document_search':
                api_base = get_base_url().rstrip("/")
                folder_groups = doc_results.get('folder_groups', {})
                for folder_key, items in folder_groups.items():
                    for item in items:
                        rel = item.get("page_download_url")
                        if rel and not str(rel).startswith("http"):
                            item["page_download_url"] = f"{api_base}{rel}"
                        _convert_document_hit_image_fields(item, agent_id, api_base)
                chunk_hits = doc_results.get('chunk_hits') or []
                for hit in chunk_hits:
                    if not isinstance(hit, dict):
                        continue
                    rel = hit.get("page_download_url")
                    if rel and not str(rel).startswith("http"):
                        hit["page_download_url"] = f"{api_base}{rel}"
                    _convert_document_hit_image_fields(hit, agent_id, api_base)
                search_results = doc_results.get('results', []) or []
                for hit in search_results:
                    if not isinstance(hit, dict):
                        continue
                    rel = hit.get("page_download_url")
                    if rel and not str(rel).startswith("http"):
                        hit["page_download_url"] = f"{api_base}{rel}"
                    _convert_document_hit_image_fields(hit, agent_id, api_base)

                response_details["document_search"] = {
                    "query": doc_results.get('query', ''),
                    "folder_detected": doc_results.get('folder_detected'),
                    "chunk_hits": chunk_hits,
                    "folder_groups": folder_groups,
                    "results": search_results,
                    "overall_summary": doc_results.get('overall_summary', ''),
                    "matched_agent_ids": doc_results.get('matched_agent_ids', []),
                }
            elif result_type == 'document_analysis':
                # Convert all_images paths to URLs
                all_images = doc_results.get('all_images', [])
                converted_images = []
                for img in all_images:
                    if isinstance(img, dict):
                        # If image is a dict with 'path' key
                        converted_img = img.copy()
                        if 'path' in converted_img:
                            converted_img['path'] = convert_image_path_to_url(converted_img['path'], agent_id)
                        converted_images.append(converted_img)
                    elif isinstance(img, str):
                        # If image is just a string path
                        converted_images.append(convert_image_path_to_url(img, agent_id))
                    else:
                        # Keep as is if unknown format
                        converted_images.append(img)
                
                response_details["document_analysis"] = {
                    "file_id": doc_results.get("file_id"),
                    "filename": doc_results.get('filename', ''),
                    "file_type": doc_results.get('file_type', ''),
                    "folder_path": doc_results.get('folder_path', ''),
                    "relative_path": doc_results.get('relative_path', ''),
                    "folder_detected": doc_results.get('folder_detected'),
                    "analysis": doc_results.get('analysis', ''),
                    "all_images": converted_images,  # Use converted images
                    "file_path": doc_results.get('file_path', ''),
                    "file_size": doc_results.get('file_size'),
                    "upload_date": doc_results.get('upload_date'),
                    "preview_url": doc_results.get("preview_url"),
                    "preview_content_type": doc_results.get("preview_content_type"),
                    "preview_mode": doc_results.get("preview_mode"),
                }
            elif result_type == 'document_find':
                api_base = get_base_url().rstrip("/")
                find_results = doc_results.get('results', []) or []
                for result in find_results:
                    if not isinstance(result, dict):
                        continue
                    for row in result.get("matching_page_urls") or []:
                        if isinstance(row, dict):
                            rel = row.get("page_download_url")
                            if rel and not str(rel).startswith("http"):
                                row["page_download_url"] = f"{api_base}{rel}"
                    if result.get("page_images"):
                        result["page_images"] = _convert_document_page_images(
                            result["page_images"], agent_id, api_base
                        )
                    result.pop('relevant_images', None)

                response_details["document_find"] = {
                    "query": doc_results.get('query', ''),
                    "folder_detected": doc_results.get('folder_detected'),
                    "results": find_results,
                    "total_files_searched": doc_results.get('total_files_searched', 0),
                    "message": doc_results.get('message'),
                    "matched_agent_ids": doc_results.get('matched_agent_ids', []),
                }
            elif result_type == 'error':
                err_msg = doc_results.get('error', 'Document operation failed')
                # If file not found, return a "no files found" description and skip suggested questions
                if "not found" in err_msg.lower() or "no files" in err_msg.lower():
                    response_details["talkback"] = ""
                    # Ensure the description is set for the frontend
                    if "document_search" in response_details:
                        response_details["document_search"]["overall_summary"] = "no files found"
                    elif "document_find" in response_details:
                        response_details["document_find"]["overall_summary"] = "no files found"
                    
                    await run_in_threadpool(
                        create_user_question_log,
                        user_id=user_id,
                        agent_id=agent_id,
                        question=question,
                        sql="N",
                        chart="N",
                        web="N",
                        executed="Y",
                        start_time=question_start_time,
                        api_endpoint="",
                        api_params="",
                        api_response_validator=""
                    )
                    return {
                        "success": True,
                        "message": "no files found",
                        "details": response_details
                    }

                # Handle other error cases
                await run_in_threadpool(
                    create_user_question_log,
                    user_id=user_id,
                    agent_id=agent_id,
                    question=question,
                    sql="N",
                    chart="N",
                    web="N",
                    executed="N",
                    start_time=question_start_time,
                    api_endpoint="",
                    api_params="",
                    api_response_validator=""
                )
                return {
                    "success": False,
                    "message": err_msg,
                    "details": response_details
                }
            else:
                # Unknown result type
                await run_in_threadpool(
                    create_user_question_log,
                    user_id=user_id,
                    agent_id=agent_id,
                    question=question,
                    sql="N",
                    chart="N",
                    web="N",
                    executed="N",
                    start_time=question_start_time,
                    api_endpoint="",
                    api_params="",
                    api_response_validator=""
                )
                return {
                    "success": False,
                    "message": "Unknown document operation result type",
                    "details": response_details
                }
           
            # Collect matched agent IDs from results to refine dispatch scope
            matched_agent_ids = set()
            
            # Use pre-calculated matched_agent_ids if available from service (handles multi-agent hits in @find)
            if doc_results.get('matched_agent_ids'):
                for aid in doc_results['matched_agent_ids']:
                    if aid:
                        matched_agent_ids.add(str(aid))
            
            results_list = doc_results.get('results', [])
            
            if not matched_agent_ids:
                if result_type == 'document_search':
                    for r in results_list:
                        aid = r.get('agent_id')
                        if aid:
                            matched_agent_ids.add(str(aid))
                elif result_type == 'document_analysis':
                    aid = doc_results.get('agent_id')
                    if aid:
                        matched_agent_ids.add(str(aid))
                elif result_type == 'document_find':
                    for r in results_list:
                        aid = r.get('agent_id')
                        if aid:
                            matched_agent_ids.add(str(aid))
            
            if matched_agent_ids:
                response_details["matched_agent_ids"] = list(matched_agent_ids)
                print(f"📁✅ [FILE] matched_agent_ids: {response_details['matched_agent_ids']}")

            _attach_talkback_documents(response_details, agent_id=agent_id)

            # Generate talkback before returning so frontend receives it in this response.
            try:
                summary_text = doc_results.get('overall_summary') or doc_results.get('analysis') or ""
                summary_str = str(summary_text)
                
                # If no results found, show "no files found" and skip suggested questions
                # Check results_list, summary, and analysis
                has_content = bool(results_list) or bool(summary_text.strip()) or bool(doc_results.get('analysis'))
                
                if not has_content:
                    summary_str = "no files found"
                    response_details["talkback"] = None # Use None to explicitly suppress
                    print("📁🔍 [FILE] No content found, suppressing talkback.")
                    # Ensure the description is set in the details as well
                    if "document_search" in response_details:
                        response_details["document_search"]["overall_summary"] = summary_str
                    elif "document_find" in response_details:
                        response_details["document_find"]["overall_summary"] = summary_str
                else:
                    if not summary_str:
                        summary_str = f"Document Search/Analysis completed. Found {len(results_list)} results."

                    talkback = await run_in_threadpool(
                        service.generate_continuation_talkback_cached,
                        question=question_to_use,
                        answer_context=summary_str[:1000],
                        columns=[],
                        n_items=3,
                        use_qwen=settings.default_use_qwen_for_summary_and_web(),
                    )
                    response_details["talkback"] = talkback
            except Exception as e:
                print(f"Error generating talkback for document: {e}")

            # Determine final message based on whether files were found
            final_message = "Document operation completed successfully"
            if not doc_results.get('results', []) and not doc_results.get('analysis'):
                if not doc_results.get('overall_summary') or doc_results.get('overall_summary') == "no files found":
                    final_message = "no files found"

            # Log successful execution
            await run_in_threadpool(
                create_user_question_log,
                user_id=user_id,
                agent_id=agent_id,
                question=question,
                sql="N",
                chart="N",
                web="N",
                executed="Y",
                start_time=question_start_time,
                api_endpoint="",
                api_params="",
                api_response_validator=""
            )
           
            return {
                "success": True,
                "message": final_message,
                "details": response_details
            }
           
        except Exception as e:
            import traceback
            error_msg = f"Error performing document operation: {str(e)}"
            print(f"Error in document operation: {traceback.format_exc()}")
           
            create_user_question_log(
                user_id=user_id,
                agent_id=agent_id,
                question=question,
                sql="N",
                chart="N",
                web="N",
                executed="N",
                start_time=question_start_time,
                api_endpoint="",
                api_params="",
                api_response_validator=""
            )
           
            return {
                "success": False,
                "message": error_msg,
                "details": {"question": question, "error": str(e)}
            }
   
    except Exception as e:
        import traceback
        error_msg = f"Error processing file question: {str(e)}"
        print(f"Error in ask_file: {traceback.format_exc()}")
       
        # Clean up temp file on error
        if temp_file_path and os.path.exists(temp_file_path):
            try:
                os.remove(temp_file_path)
            except:
                pass
       
        try:
            await run_in_threadpool(
                create_user_question_log,
                user_id=user_id,
                agent_id=agent_id,
                question=question,
                sql="N",
                chart="N",
                web="N",
                executed="N",
                start_time=question_start_time,
                api_endpoint="",
                api_params="",
                api_response_validator=""
            )
        except:
            pass
       
        return {
            "success": False,
            "message": error_msg,
            "details": {"error": str(e)}
        }

@router.post("/get-alternate-prompts")
async def generate_error_recovery_questions(
    request: ErrorRecoveryRequest,
    current_user: User = Depends(get_current_user)
):
    """
    Generate alternate/recovery questions when user's question fails.
    Returns validated questions with SQL that are similar to the failed question.
    For API data source: Returns empty recovery questions list with all fields.
    For Database data source: Returns validated questions with SQL and API fields.
    """
    try:
        # Normalize agent_id to list for consistent handling
        if isinstance(request.agent_id, list):
            agent_ids = request.agent_id
        elif isinstance(request.agent_id, str) and request.agent_id.startswith('[') and request.agent_id.endswith(']'):
            try:
                return get_askme_service(agent_id=request.agent_id)
            except Exception as e:
                return None
 
        elif isinstance(request.agent_id, str):
            try:
                agent_ids = json.loads(request.agent_id)
                if not isinstance(agent_ids, list):
                    agent_ids = [request.agent_id]
            except:
                agent_ids = [request.agent_id]
        else:
            agent_ids = request.agent_id if request.agent_id else []

        if not agent_ids or len(agent_ids) == 0:
             return {
                "success": False,
                "message": "No agent IDs provided",
                "details": {
                    "failed_question": request.failed_question,
                    "recovery_questions": []
                }
            }

        primary_agent_id = agent_ids[0] if agent_ids else None
        service = get_askme_service(agent_id=primary_agent_id) if primary_agent_id else None
       
        # Handle based on data source
        if request.data_source.upper() == "API":
            # For API: Return empty recovery questions list with all required fields
            # Recovery questions are not generated for API, so return empty list
            recovery_questions = []
           
            return {
                "success": True,
                "message": "Recovery questions for API data source (currently empty)",
                "details": {
                    "failed_question": request.failed_question,
                    "recovery_questions": recovery_questions  # Empty list for API
                }
            }
       
        elif request.data_source.upper() == "DATABASE":
            if not service:
                return {
                    "success": False,
                    "message": f"Agent configuration not found for agent '{primary_agent_id}'",
                    "details": {"failed_question": request.failed_question, "recovery_questions": []}
                }
            # Use dedicated executor to avoid blocking the default Starlette thread pool.
            # generate_error_recovery_questions runs 30-120+ seconds (LLM + SQL validation).
            # Running in the shared pool would starve other endpoints and make them stuck.
            loop = asyncio.get_event_loop()
            recovery_questions = await loop.run_in_executor(
                _LONG_RUNNING_EXECUTOR,
                functools.partial(
                    service.generate_error_recovery_questions,
                    failed_question=request.failed_question,
                    max_questions=request.max_questions
                )
            )
           
            # Enhance each recovery question with API fields (empty strings)
            enhanced_recovery_questions = []
            for question in recovery_questions:
                enhanced_question = {
                    "question": question.get("question", ""),
                    "sql": question.get("sql", ""),
                    "api_endpoint": "",
                    "api_params": "",
                    "api_response_validator": ""
                }
                enhanced_recovery_questions.append(enhanced_question)
           
            return {
                "success": True,
                "message": f"Generated {len(enhanced_recovery_questions)} alternate questions successfully",
                "details": {
                    "failed_question": request.failed_question,
                    "recovery_questions": enhanced_recovery_questions  # List of {"question": str, "sql": str, "api_endpoint": "", "api_params": "", "api_response_validator": ""}
                }
            }
        else:
            return {
                "success": False,
                "message": f"Invalid data_source: {request.data_source}. Must be 'API' or 'Database'",
                "details": {
                    "failed_question": request.failed_question,
                    "recovery_questions": []
                }
            }
           

        # Calculate number of questions per agent
        num_agents = len(agent_ids)
        total_needed = request.max_questions
        questions_per_agent = total_needed // num_agents
        remainder = total_needed % num_agents

        all_recovery_questions = []

        # Iterate through each agent to get their share of recovery questions
        for i, agent_id in enumerate(agent_ids):
            # Calculate how many questions this specific agent should provide
            current_agent_count = questions_per_agent + (1 if i < remainder else 0)
            if current_agent_count <= 0:
                continue

            def get_service_safe(aid):
                try:
                    return get_askme_service(agent_id=aid)
                except Exception as e:
                    print(f"Error getting service for agent {aid}: {e}")
                    return None

            service = await run_in_threadpool(get_service_safe, agent_id)
            if not service:
                print(f"Warning: Agent configuration not found for agent_id: {agent_id}")
                continue
            
            # Handle based on data source
            if request.data_source.upper() == "API":
                 return {
                    "success": True,
                    "message": "Recovery questions for API data source (currently empty)",
                    "details": {
                        "failed_question": request.failed_question,
                        "recovery_questions": all_recovery_questions  # Empty list for API
                    }
                }
            
            elif request.data_source.upper() == "DATABASE":
                # For Database: Generate recovery questions with validated SQL
                recovery_questions = await run_in_threadpool(
                    service.generate_error_recovery_questions,
                    failed_question=request.failed_question,
                    max_questions=current_agent_count
                )
                
                # Enhance each recovery question with API fields and agent_id
                for question in (recovery_questions or []):
                    all_recovery_questions.append({
                        "question": question.get("question", ""),
                        "sql": question.get("sql", ""),
                        "api_endpoint": "",
                        "api_params": "",
                        "api_response_validator": "",
                        "agent_id": agent_id # Track which agent provided the question
                    })
        
        return {
            "success": True,
            "message": f"Generated {len(all_recovery_questions)} alternate questions successfully",
            "details": {
                "failed_question": request.failed_question,
                "recovery_questions": all_recovery_questions
            }
        }
            
    except Exception as e:
        return {
            "success": False,
            "message": f"Error generating recovery questions: {str(e)}",
            "details": {
                "failed_question": request.failed_question,
                "recovery_questions": []
            }
        }



@router.post("/get-it-checked")
async def get_it_checked(
    request: GetItCheckedRequest,
    current_user: User = Depends(get_current_user),
    db: Session = Depends(get_db)
):
    """
    Get it checked endpoint (single-agent) - reports that user is not getting correct answer.

    Body:
        agent_id: Agent ID string (required, single agent)
        question: Question text (required)
        sql: SQL query (optional)
        chart: Chart code (optional)
        start_time: Start time in format "%Y-%m-%d %H:%M:%S" (required)
        api_endpoint: API endpoint (optional)
        api_params: API parameters (optional)
        api_response_validator: API response validator (optional)
        is_error: True if reporting an error, False if answer is just wrong (optional)

    Returns:
        Success response with log_id, email_sent, admin_emails in details
    """
    try:
        # Use agent_id as a single string (single-agent endpoint)
        agent_id = request.agent_id if isinstance(request.agent_id, str) else request.agent_id[0] if request.agent_id else ""

        if not agent_id:
            return {
                "success": False,
                "message": "No agent ID provided",
                "details": {}
            }

        status_type = "E" if request.is_error else "D"
        user_id = request.user_id
        user_info = f"{request.username} (User ID : {user_id})"

        # ---- 1. Fetch agent configuration ----------------------------------------
        def get_agent_config_safe():
            try:
                return get_agent_config(agent_id=agent_id)
            except Exception:
                return None

        agent_config = await run_in_threadpool(get_agent_config_safe)
        if not agent_config:
            return {
                "success": False,
                "message": f"Agent configuration not found for agent_id: {agent_id}",
                "details": {}
            }

        # ---- 2. Fetch admin e-mails -----------------------------------------------
        def get_admin_emails():
            return db.query(User.email).select_from(AgentAdmin).outerjoin(
                User, cast(AgentAdmin.admin_id, Integer) == User.id
            ).filter(
                AgentAdmin.agent_id == agent_id,
                User.email.isnot(None)
            ).all()

        admin_emails = await run_in_threadpool(get_admin_emails)
        admin_email_ids = [email for (email,) in admin_emails if email]

        if not admin_email_ids:
            return {
                "success": False,
                "message": f"No admin emails found for agent_id: {agent_id}",
                "details": {}
            }

        # ---- 3. Create question log entry ----------------------------------------
        log_id = await run_in_threadpool(
            create_user_question_log,
            user_id=user_id,
            agent_id=agent_id,
            question=request.question,
            sql=request.sql or "N",
            chart=request.chart or "N",
            web="N",
            executed=status_type,
            start_time=request.start_time,
            api_endpoint=request.api_endpoint or "",
            api_params=request.api_params or "",
            api_response_validator=request.api_response_validator or ""
        )

        # ---- 4. Build & send email -----------------------------------------------
        subject, body = get_mail_content(
            user_info=user_info,
            question=request.question,
            sql=request.sql or "N",
            agent=agent_config.get("agent_name", agent_id),
            timestamp=request.start_time,
            type=status_type
        )

        def get_service_safe():
            try:
                return get_askme_service(agent_id=agent_id)
            except Exception:
                return None

        service = await run_in_threadpool(get_service_safe)
        email_sent = False
        if service:
            email_sent = await run_in_threadpool(
                service.send_email,
                sender_email=agent_config.get("sender_email", ""),
                sender_name="askme360",
                recipient_email=admin_email_ids,
                subject=subject,
                body_text="Issue : ",
                body_html=body,
                agent_conf=agent_config
            )
        else:
            print(f"Warning: Could not get service for agent {agent_id} â€” email not sent")

        if not email_sent:
            print(f"Warning: Failed to send email for get-it-checked request (agent_id={agent_id})")

        return {
            "success": True,
            "message": "Issue reported successfully and email sent to admin",
            "details": {
                "log_id": log_id,
                "email_sent": email_sent,
                "admin_emails": admin_email_ids
            }
        }

    except Exception as e:
        import traceback
        traceback.print_exc()
        return {
            "success": False,
            "message": f"Error processing get-it-checked request: {str(e)}",
            "details": {}
        }


async def _reconcile_upload_core(
    *,
    prev_df: str,
    question: str,
    user_id: str,
    agent_id: str,
    file: UploadFile,
) -> Dict[str, Any]:
    """
    Shared implementation for POST /reconcile and talkback-ask-question multipart uploads.
    user_id matches the /reconcile form contract (reserved for future use).
    """
    _ = user_id

    temp_path = None
    try:
        # Validate file type: xlsx or csv only
        fn = file.filename or ""
        ext = fn.rsplit(".", 1)[-1].lower() if "." in fn else ""
        if ext not in ("xlsx", "xls", "csv"):
            return {
                "success": False,
                "message": f"Unsupported file type '{ext}'. Only xlsx, xls, or csv are allowed.",
                "details": {"filename": fn},
            }

        # Save upload to temp file for reading
        def save_upload():
            suffix = ".xlsx" if ext in ("xlsx", "xls") else ".csv"
            with tempfile.NamedTemporaryFile(delete=False, suffix=suffix) as tmp:
                shutil.copyfileobj(file.file, tmp)
                return tmp.name

        temp_path = await run_in_threadpool(save_upload)

        # Load dataframe from file
        def read_file():
            if ext == "csv":
                return pd.read_csv(temp_path)
            return pd.read_excel(temp_path)

        excel_df = await run_in_threadpool(read_file)

        # Fetch agent config and service
        def get_agent_config_safe():
            try:
                return get_agent_config(agent_id=agent_id)
            except Exception as e:
                print(f"Error fetching agent config: {e}")
                return None

        agent_config = await run_in_threadpool(get_agent_config_safe)
        if not agent_config:
            return {
                "success": False,
                "message": f"Agent '{agent_id}' not found",
                "details": {},
            }

        def get_service_safe():
            try:
                return get_askme_service(agent_id=agent_id)
            except Exception as e:
                print(f"Error getting askme service: {e}")
                return None

        service = await run_in_threadpool(get_service_safe)
        if not service:
            return {
                "success": False,
                "message": f"Could not load service for agent '{agent_id}'",
                "details": {},
            }

        # Parse prev_df (list of dicts or JSON string) to DataFrame
        parsed = service._parse_df_input(prev_df)
        prev_df_df = pd.DataFrame(parsed) if parsed else pd.DataFrame()

        # Validate both dataframes non-empty
        if prev_df_df.empty:
            return {
                "success": False,
                "message": "Previous dataframe (prev_df) is empty or invalid",
                "details": {},
            }
        if excel_df.empty:
            return {
                "success": False,
                "message": "Uploaded file produced an empty dataframe",
                "details": {},
            }
        # Run reconcile_and_merge (LLM + exec) in threadpool
        try:
            result_df, reconciliation_result = await run_in_threadpool(
                service.reconcile_and_merge, prev_df_df, excel_df, question, do_df=True
            )
        except Exception as e:
            return {
                "success": False,
                "message": "Merge code execution failed",
                "details": {"error": str(e)},
            }

        if not isinstance(result_df, pd.DataFrame):
            return {
                "success": False,
                "message": "Merge did not produce a valid dataframe",
                "details": {},
            }
        # _reconciled and _source are optional (e.g. when question is compare/aggregate, not reconcile)

        # Convert string-numeric columns back to proper types; keep key/categorical columns as-is so they don't become NaN.
        # _normalise_value in reconcile_and_merge converts everything to strings for comparison.
        for col in result_df.columns:
            if col in ("_source", "_reconciled"):
                continue
            converted = pd.to_numeric(result_df[col], errors="coerce")
            # If conversion produced all NaN and original had non-empty values, keep as string (e.g. month "2024-01")
            if converted.isna().all() and result_df[col].astype(str).str.strip().ne("").any():
                continue
            result_df[col] = converted

        # Serialize merged data
        df_records = result_df.to_dict("records")
        df_records = clean_nan_values(df_records)
        reconciliation_report = reconciliation_result if isinstance(reconciliation_result, dict) else {}
        reconciliation_report = clean_nan_values(reconciliation_report)

        # Fallback if report missing or malformed
        if not reconciliation_report:
            reconciliation_report = {
                "status": "UNKNOWN",
                "summary": "Report generation failed or not provided",
                "missing_rows": [],
                "extra_rows": [],
                "value_mismatches": [],
                "column_differences": {"in_excel_not_df": [], "in_df_not_excel": []},
                "recommendations": "",
            }

        _source_counts = None
        if "_source" in result_df.columns:
            _source_counts = result_df["_source"].value_counts().to_dict()

        # â”€â”€ Generate chart data from result_df â”€â”€â”€â”€â”€â”€â”€â”€â”€â”€â”€â”€â”€â”€â”€â”€â”€â”€â”€â”€â”€â”€â”€â”€â”€â”€â”€
        chart_fig = {}
        chart_code_str = None
        try:
            # Use only data columns for chart (exclude internal reconciliation columns)
            chart_cols = [c for c in result_df.columns if c not in ("_reconciled", "_source")]
            if len(result_df) > 0 and len(result_df) < 101 and len(chart_cols) >= 2:
                import json as json_module
                json_code = await run_in_threadpool(
                    service.generate_json_code_cached,
                    question=question,
                    columns=chart_cols,
                    model="gpt-4.1"
                )
                if json_code and json_code != "N":
                    chart_code_str = json_code
                    # Prepare a chart-friendly DataFrame without internal columns
                    print(f"[reconcile] chart_cols: {chart_cols}")
                    chart_df = result_df[chart_cols].copy()
                    compile(json_code, '<string>', 'exec')
                    exec_globals = {"pd": pd, "json": json_module, "json_module": json_module}
                    local_vars = {"df": chart_df, "pd": pd, "json": json_module, "json_module": json_module}
                    exec(json_code, exec_globals, local_vars)
                    chart_json = local_vars.get("chart_json")
                    if chart_json and isinstance(chart_json, dict):
                        if "data" in chart_json and "layout" in chart_json:
                            # Ensure layout has title and axes
                            layout = chart_json.setdefault("layout", {})
                            if "title" not in layout:
                                layout["title"] = question[:80] if len(question) > 80 else question
                            if "xaxis" not in layout and chart_cols:
                                layout["xaxis"] = {"title": chart_cols[0]}
                            if "yaxis" not in layout and len(chart_cols) > 1:
                                layout["yaxis"] = {"title": chart_cols[1]}
                            chart_fig = chart_json
        except Exception as chart_err:
            print(f"[reconcile] Chart generation failed: {chart_err}")
            chart_fig = {}

        _cols = list(result_df.columns)
        response = {
            "success": True,
            "message": "Reconcile completed",
            "details": {
                # Align with ask-question / multi-agent ``details.df`` envelope for clients.
                "df": {
                    "agent_id": str(agent_id),
                    "headers": _cols,
                    "data": df_records,
                    "chart_code": chart_code_str if chart_code_str else "N",
                    "merged": True,
                    "tabular_from_merge": True,
                },
                "_source_counts": _source_counts,
                "reconciliation_report": reconciliation_report,
                "chart": clean_nan_values(chart_fig),
                "chart_code": chart_code_str,
            },
        }
        return drop_null_keys(clean_nan_values(response))
    except Exception as e:
        return {
            "success": False,
            "message": f"Error processing reconcile request: {str(e)}",
            "details": {"error": str(e)},
        }
    finally:
        if temp_path and os.path.exists(temp_path):
            try:
                os.remove(temp_path)
            except Exception:
                pass




@router.post("/reconcile")
async def reconcile(
    prev_df: str = Form(...),
    question: str = Form(...),
    user_id: str = Form(...),
    agent_id: str = Form(...),
    file: UploadFile = File(...),
    current_user: User = Depends(get_current_user)
):
    """
    Reconcile endpoint: accepts previous dataframe/context, previous question,
    current question, user/question ids, agent_id, and an xlsx or csv file.

    Form Data (multipart/form-data):
        prev_df: Previous dataframe state (e.g. JSON string) (required)
        prev_question: Previous question text (required)
        question: Current question text (required)
        user_id: User ID (required)
        question_id: Question ID (required)
        agent_id: Agent ID for LLM config (required)
        file: Uploaded file - xlsx or csv only (required)

    Returns:
        Merged data plus reconciliation report (data, columns, _source_counts, reconciliation_report)
    """
    _ = current_user
    return await _reconcile_upload_core(
        prev_df=prev_df,
        question=question,
        user_id=user_id,
        agent_id=agent_id,
        file=file,
    )


@router.post("/multi-agent-get-it-checked")
async def multi_agent_get_it_checked(
    request: GetItCheckedRequest,
    current_user: User = Depends(get_current_user),
    db: Session = Depends(get_db)
):
    """
    Multi-agent get-it-checked endpoint - reports that user is not getting correct answer
    for multiple agents.

    Body:
        agent_id: List of Agent ID strings (required, multi-agent)
        question: Question text (required)
        sql: SQL query (optional)
        chart: Chart code (optional)
        start_time: Start time in format "%Y-%m-%d %H:%M:%S" (required)
        api_endpoint: API endpoint (optional)
        api_params: API parameters (optional)
        api_response_validator: API response validator (optional)
        is_error: True if reporting an error, False if answer is just wrong (optional)

    Returns:
        Success response with per-agent results in details.results
    """
    try:
        # Normalize agent_id to a list (supports single string or list)
        if isinstance(request.agent_id, list):
            agent_ids = request.agent_id
        elif (
            isinstance(request.agent_id, str)
            and request.agent_id.startswith("[")
            and request.agent_id.endswith("]")
        ):
            try:
                agent_ids = json.loads(request.agent_id)
                if not isinstance(agent_ids, list):
                    agent_ids = [request.agent_id]
            except Exception:
                agent_ids = [request.agent_id]
        else:
            agent_ids = [request.agent_id]

        if not agent_ids:
            return {
                "success": False,
                "message": "No agent IDs provided",
                "details": {}
            }

        status_type = "E" if request.is_error else "D"
        user_id = request.user_id
        user_info = f"{request.username} (User ID : {user_id})"

        per_agent_results = []
        any_success = False

        for agent_id in agent_ids:
            agent_result = {"agent_id": agent_id, "success": False}

            # ---- 1. Fetch agent configuration --------------------------------
            def get_agent_config_safe(aid=agent_id):
                try:
                    return get_agent_config(agent_id=aid)
                except Exception:
                    return None

            agent_config = await run_in_threadpool(get_agent_config_safe)
            if not agent_config:
                agent_result["message"] = f"Agent configuration not found for agent_id: {agent_id}"
                per_agent_results.append(agent_result)
                continue

            # ---- 2. Fetch admin e-mails for this agent -----------------------
            def get_admin_emails(aid=agent_id):
                return db.query(User.email).select_from(AgentAdmin).outerjoin(
                    User, cast(AgentAdmin.admin_id, Integer) == User.id
                ).filter(
                    AgentAdmin.agent_id == aid,
                    User.email.isnot(None)
                ).all()

            admin_emails = await run_in_threadpool(get_admin_emails)
            admin_email_ids = [email for (email,) in admin_emails if email]

            if not admin_email_ids:
                agent_result["message"] = f"No admin emails found for agent_id: {agent_id}"
                per_agent_results.append(agent_result)
                continue

            # ---- 3. Create question log entry --------------------------------
            log_id = await run_in_threadpool(
                create_user_question_log,
                user_id=user_id,
                agent_id=agent_id,
                question=request.question,
                sql=request.sql or "N",
                chart=request.chart or "N",
                web="N",
                executed=status_type,
                start_time=request.start_time,
                api_endpoint=request.api_endpoint or "",
                api_params=request.api_params or "",
                api_response_validator=request.api_response_validator or ""
            )

            # ---- 4. Build & send email ---------------------------------------
            subject, body = get_mail_content(
                user_info=user_info,
                question=request.question,
                sql=request.sql or "N",
                agent=agent_config.get("agent_name", agent_id),
                timestamp=request.start_time,
                type=status_type
            )

            def get_service_safe(aid=agent_id):
                try:
                    return get_askme_service(agent_id=aid)
                except Exception:
                    return None

            service = await run_in_threadpool(get_service_safe)
            email_sent = False
            if service:
                email_sent = await run_in_threadpool(
                    service.send_email,
                    sender_email=agent_config.get("sender_email", ""),
                    sender_name="askme360",
                    recipient_email=admin_email_ids,
                    subject=subject,
                    body_text="Issue : ",
                    body_html=body,
                    agent_conf=agent_config
                )
            else:
                print(f"Warning: Could not get service for agent {agent_id} â€” email not sent")

            if not email_sent:
                print(f"Warning: Failed to send email for multi-agent-get-it-checked request (agent_id={agent_id})")

            agent_result.update({
                "success": True,
                "message": "Issue reported successfully",
                "log_id": log_id,
                "email_sent": email_sent,
                "admin_emails": admin_email_ids
            })
            any_success = True
            per_agent_results.append(agent_result)

        if not any_success:
            return {
                "success": False,
                "message": "Failed to process multi-agent-get-it-checked for all agents",
                "details": {"results": per_agent_results}
            }

        return {
            "success": True,
            "message": "Issue reported successfully and emails sent to admins",
            "details": {"results": per_agent_results}
        }

    except Exception as e:
        import traceback
        traceback.print_exc()
        return {
            "success": False,
            "message": f"Error processing multi-agent-get-it-checked request: {str(e)}",
            "details": {}
        }


@router.post("/multi-agent-generate-followup-questions")
async def multi_agent_generate_followup_questions(
    request: GenerateFollowupRequest,
    current_user: User = Depends(get_current_user)
):
    """
    Generate a list of followup questions across multiple agents (multi-agent).

    Distributes n_questions equally among all provided agents.
    For Database data source: Returns validated followup questions with SQL
    For API data source: Returns simple followup questions without SQL validation

    Send columns as: df.columns.tolist()
    """
    try:
        # Normalize agent_id to list
        if isinstance(request.agent_id, list):
            agent_ids = request.agent_id
        elif isinstance(request.agent_id, str) and request.agent_id.startswith('[') and request.agent_id.endswith(']'):
            try:
                agent_ids = json.loads(request.agent_id)
                if not isinstance(agent_ids, list):
                    agent_ids = [request.agent_id]
            except Exception:
                agent_ids = [request.agent_id]
        elif isinstance(request.agent_id, str):
            agent_ids = [request.agent_id]
        else:
            agent_ids = request.agent_id if request.agent_id else []

        if not agent_ids:
            return {
                "success": False,
                "message": "No agent IDs provided",
                "details": {
                    "question": request.question,
                    "followup_questions": [],
                    "data_source": request.data_source
                }
            }

        # Normalize columns: handle List[str] or List[List[str]]
        columns_to_use = []
        if request.columns and any(isinstance(c, list) for c in request.columns):
            seen = set()
            for item in request.columns:
                if isinstance(item, list):
                    for col in item:
                        if col not in seen:
                            columns_to_use.append(col)
                            seen.add(col)
                else:
                    if item not in seen:
                        columns_to_use.append(item)
                        seen.add(item)
        else:
            columns_to_use = request.columns

        # Distribute n_questions equally among agents
        num_agents = len(agent_ids)
        total_needed = min(request.n_questions, 5 * num_agents)
        questions_per_agent = total_needed // num_agents
        remainder = total_needed % num_agents
        quotas = [questions_per_agent + (1 if i < remainder else 0) for i in range(num_agents)]

        all_followups = []

        def get_service_safe(aid):
            try:
                return get_askme_service(agent_id=aid)
            except Exception as e:
                print(f"Error getting service for agent {aid}: {e}")
                return None

        if request.data_source.upper() == "API":
            for i, agent_id in enumerate(agent_ids):
                current_agent_count = quotas[i]
                if current_agent_count <= 0:
                    continue
                service = await run_in_threadpool(get_service_safe, agent_id)
                if not service:
                    print(f"Warning: Agent configuration not found for agent_id: {agent_id}")
                    continue
                questions = await run_in_threadpool(
                    service.generate_followup_cached,
                    question=request.question,
                    sql="",
                    columns=columns_to_use
                )
                for question in (questions[:current_agent_count] if questions else []):
                    all_followups.append({
                        "question": question,
                        "sql": "",
                        "api_endpoint": "",
                        "api_params": "",
                        "api_response_validator": "",
                        "agent_id": agent_id
                    })

        elif request.data_source.lower() == "database":
            # Normalize sql to a list (one per agent): support list, dict keyed by agent_id, or single string
            if isinstance(request.sql, dict):
                sql_list = [request.sql.get(str(aid), request.sql.get(aid, "")) for aid in agent_ids]
            elif isinstance(request.sql, list):
                sql_list = list(request.sql)[:num_agents]
                if len(sql_list) < num_agents:
                    pad = sql_list[-1] if sql_list else ""
                    sql_list = sql_list + [pad] * (num_agents - len(sql_list))
            else:
                sql_list = [request.sql] * num_agents
 
            # Per-agent: run each agent's SQL on that agent's DB and generate validated followups with that agent's service
            for i, agent_id in enumerate(agent_ids):
                if quotas[i] <= 0:
                    continue
                service = await run_in_threadpool(get_service_safe, agent_id)
                if not service:
                    print(f"Warning: Agent configuration not found for agent_id: {agent_id}")
                    continue
                sql_for_agent = sql_list[i] if i < len(sql_list) else (sql_list[-1] if sql_list else "")
                if not (sql_for_agent and str(sql_for_agent).strip()):
                    continue
                # Run this agent's SQL on this agent's DB to get DataFrame (correct schema/columns for validation)
                try:
                    df_for_agent = await run_in_threadpool(service.run_sql_cached, sql_for_agent)
                except Exception as e:
                    print(f"Warning: Could not run SQL for agent {agent_id}: {e}")
                    df_for_agent = None
                if df_for_agent is None or not hasattr(df_for_agent, "columns"):
                    df_for_agent = pd.DataFrame(columns=columns_to_use if columns_to_use else [])
                validated = await run_in_threadpool(
                    service.generate_validated_followup_questions,
                    question=request.question,
                    sql=sql_for_agent,
                    df=df_for_agent,
                    max_questions=quotas[i]
                )
                for followup in (validated or []):
                    all_followups.append({
                        "question": followup.get("question", ""),
                        "sql": followup.get("sql", ""),
                        "api_endpoint": "",
                        "api_params": "",
                        "api_response_validator": "",
                        "agent_id": agent_id
                    })

        if not all_followups:
            return {
                "success": False,
                "message": "Failed to generate any followup questions from the provided agents",
                "details": {
                    "question": request.question,
                    "followup_questions": [],
                    "data_source": request.data_source
                }
            }

        return {
            "success": True,
            "message": "Follow-up questions generated and combined successfully",
            "details": {
                "question": request.question,
                "followup_questions": all_followups,
                "data_source": request.data_source
            }
        }

    except Exception as e:
        import traceback
        traceback.print_exc()
        return {
            "success": False,
            "message": f"Error generating followup questions: {str(e)}",
            "details": {
                "question": request.question,
                "followup_questions": [],
                "data_source": request.data_source if hasattr(request, 'data_source') else "Unknown"
            }
        }


@router.post("/multi-agent-get-alternate-prompts")
async def multi_agent_error_recovery(
    request: ErrorRecoveryRequest,
    current_user: User = Depends(get_current_user)
):
    """
    Generate alternate/recovery questions across multiple agents (multi-agent).

    Each agent independently generates max_questions questions (same as single-agent).
    Results from all agents are combined.
    For API data source: Returns empty recovery questions list.
    For Database data source: Returns validated questions with SQL.
    """
    try:
        # Normalize agent_id to list
        if isinstance(request.agent_id, list):
            agent_ids = request.agent_id
        elif isinstance(request.agent_id, str) and request.agent_id.startswith('[') and request.agent_id.endswith(']'):
            try:
                agent_ids = json.loads(request.agent_id)
                if not isinstance(agent_ids, list):
                    agent_ids = [request.agent_id]
            except Exception:
                agent_ids = [request.agent_id]
        elif isinstance(request.agent_id, str):
            agent_ids = [request.agent_id]
        else:
            agent_ids = request.agent_id if request.agent_id else []

        if not agent_ids:
            return {
                "success": False,
                "message": "No agent IDs provided",
                "details": {
                    "failed_question": request.failed_question,
                    "recovery_questions": []
                }
            }

        # Handle API data source (return early with empty list)
        if request.data_source.upper() == "API":
            return {
                "success": True,
                "message": "Recovery questions for API data source (currently empty)",
                "details": {
                    "failed_question": request.failed_question,
                    "recovery_questions": []
                }
            }

        all_recovery_questions = []

        # Each agent generates the full max_questions count (same as single-agent)
        for agent_id in agent_ids:
            def get_service_safe(aid):
                try:
                    return get_askme_service(agent_id=aid)
                except Exception as e:
                    print(f"Error getting service for agent {aid}: {e}")
                    return None

            service = await run_in_threadpool(get_service_safe, agent_id)
            if not service:
                print(f"Warning: Agent configuration not found for agent_id: {agent_id}")
                continue

            recovery_questions = await run_in_threadpool(
                service.generate_error_recovery_questions,
                failed_question=request.failed_question,
                max_questions=request.max_questions
            )

            for question in (recovery_questions or []):
                all_recovery_questions.append({
                    "question": question.get("question", ""),
                    "sql": question.get("sql", ""),
                    "api_endpoint": "",
                    "api_params": "",
                    "api_response_validator": "",
                    "agent_id": agent_id
                })

        return {
            "success": True,
            "message": f"Generated {len(all_recovery_questions)} alternate questions successfully",
            "details": {
                "failed_question": request.failed_question,
                "recovery_questions": all_recovery_questions
            }
        }

    except Exception as e:
        return {
            "success": False,
            "message": f"Error generating recovery questions: {str(e)}",
            "details": {
                "failed_question": request.failed_question,
                "recovery_questions": []
            }
        }


def load_video_model():
    return WhisperModel("medium", compute_type="int8")

@router.post("/voice-input")
async def voice_input(
    file: UploadFile = File(...),
    current_user: User = Depends(get_current_user)
):
    """
    Transcribe voice input to text using faster_whisper.
    """
    tmpfile_path = None
    try:
        # Save uploaded file to a temporary file
        def save_upload_to_temp():
            with tempfile.NamedTemporaryFile(delete=False, suffix=".wav") as tmpfile:
                shutil.copyfileobj(file.file, tmpfile)
                return tmpfile.name
        
        tmpfile_path = await run_in_threadpool(save_upload_to_temp)

        # Transcribe
        def transcribe_audio():
            model = load_video_model()
            segments, _ = model.transcribe(tmpfile_path, language=None, task="transcribe")
            return " ".join([segment.text for segment in segments])

        full_text = await run_in_threadpool(transcribe_audio)

        return {
            "success": True,
            "message": "Voice transcribed successfully",
            "details": {
                "text": full_text
            }
        }

    except Exception as e:
        return {
            "success": False,
            "message": f"Transcription error: {str(e)}",
            "details": {"text": ""}
        }
    finally:
        # Clean up temporary file
        if tmpfile_path and os.path.exists(tmpfile_path):
            os.remove(tmpfile_path)



@router.post("/contextual-question")
async def contextual_question(
    request: ContextualQuestionRequest,
    current_user: User = Depends(get_current_user)
):
    """
    Generate a standalone question based on conversation history.
    """
    try:
        def get_service_safe():
            try:
                return get_askme_service(agent_id=request.agent_id)
            except Exception as e:
                return None

        service = await run_in_threadpool(get_service_safe)
        if not service:
            return {
                "success": False,
                "message": f"Agent configuration not found for agent_id: {request.agent_id}",
                "question": request.current_question
            }
            
        question = await run_in_threadpool(
            service.generate_contextual_question,
            previous_questions=request.previous_questions,
            current_question=request.current_question
        )
        
        return {
            "success": True,
            "question": question
        }
    except Exception as e:
        return {
            "success": False,
            "message": f"Error generating contextual question: {str(e)}",
            "question": request.current_question
        }



def _attach_multi_agent_session_key(payload: Any, session_key: str) -> Any:
    if isinstance(payload, dict):
        out = dict(payload)
        out["session_key"] = session_key
        return out
    return payload


def _pick_question_for_error_talkback(*candidates: Any) -> str:
    """Prefer contextual/rewritten question, then original, for continuation talkback."""
    for p in candidates:
        s = str(p or "").strip()
        if s:
            return s
    return ""


def _truncate_sql_for_talkback_prompt(raw: Any, max_len: int = 6000) -> str:
    """Large JSON error envelopes can break or bloat the LLM; truncate for talkback only."""
    sql_s = raw
    if isinstance(sql_s, (dict, list)):
        try:
            sql_s = json.dumps(sql_s, default=str)
        except Exception:
            sql_s = str(sql_s)
    sql_s = (sql_s or "").strip()
    if len(sql_s) > max_len:
        return sql_s[: max_len - 24] + "\n...(truncated for talkback)"
    return sql_s


def _fallback_talkback_for_db_sql_error(no_rows: bool) -> str:
    """Structured suggestions when the LLM path fails or returns empty (always show something in details)."""
    if no_rows:
        return (
            "If you want, I can also:\n"
            "1. Try a different date range, status, or filter so the query returns rows.\n"
            "2. Remove or relax ID filters and list top records instead.\n"
            "3. Break the question into a smaller summary (counts or totals) first, then drill down."
        )
    return (
        "If you want, I can also:\n"
        "1. Rephrase using concrete table or column names and one clear filter (no bind placeholders like :id).\n"
        "2. Ask for a simpler SELECT (counts or one dimension) that matches your training schema.\n"
        "3. Use a sample literal value instead of a placeholder to validate filters and joins."
    )


async def _talkback_for_db_sql_error(
    service: Any,
    raw_sql: Any,
    *,
    no_rows: bool,
    question: str,
) -> str:
    """
    Same stack as successful DB answers: ``generate_continuation_talkback_cached`` (n_items=3).
    Used when SQL is invalid (not executed) or the query returns no rows.
    Always returns a non-empty string (LLM or fallback).
    """
    q = str(question or "").strip() or "the user's question"
    sql_s = _truncate_sql_for_talkback_prompt(raw_sql)
    if no_rows:
        answer_context = (
            "The query executed but returned no rows. There is no tabular data to show. "
            "Propose follow-up questions that could help the user obtain results "
            "(different filters, time range, join, or scope)."
        )
    else:
        answer_context = (
            "The generated SQL did not pass validation or failed generation; it was not executed. "
            "Propose follow-up questions aligned with the user's question and the SQL/error context, "
            "so the user can obtain valid, executable results."
        )
    if not service:
        return _fallback_talkback_for_db_sql_error(no_rows)
    try:
        out = await run_in_threadpool(
            service.generate_continuation_talkback_cached,
            question=q,
            answer_context=answer_context,
            columns=[],
            n_items=3,
            df=None,
            sql=sql_s if sql_s else None,
        )
        if out and str(out).strip():
            return str(out).strip()
    except Exception as e:
        import traceback
        print(f"[talkback] db sql error path: {e}\n{traceback.format_exc()}")
    return _fallback_talkback_for_db_sql_error(no_rows)


async def _persist_multi_agent_talkback_turn(
    service: Any,
    auth_uid: str,
    effective_session_key: str,
    agent_ids: List[str],
    question_to_use: str,
    success: bool,
    message: str,
    route: str,
    error_detail: Optional[str] = None,
) -> None:
    """Append one turn to talkback_session_state."""
    if not service or not agent_ids:
        return
    aids = [str(a).strip() for a in agent_ids if str(a).strip()]
    if not aids:
        return
    max_ans = 1800
    parts = [f"success={success}", f"route={route}", message or ""]
    if error_detail:
        parts.append(error_detail[:400])
    answer = " | ".join(p for p in parts if p).strip()
    if len(answer) > max_ans:
        answer = answer[: max_ans - 3] + "..."
    meta: Dict[str, Any] = {
        "source": "multi_agent",
        "route": route,
        "success": bool(success),
        "agent_ids": aids,
    }
    try:
        db_key = service.derive_talkback_session_key(
            auth_uid, aids, client_session_key=effective_session_key
        )
        await run_in_threadpool(
            service.append_turn_to_talkback_session,
            db_key,
            auth_uid,
            aids,
            str(question_to_use or ""),
            answer,
            10,
            None,
            meta,
            aids,
        )
    except Exception as exc:
        print(f"[multi-agent-ask] talkback persist failed ({route}): {exc}")


@router.post("/multi-agent-ask-question")
async def multi_agent_ask_question(
    request: AskQuestionRequest,
    current_user: User = Depends(get_current_user)
):
    """
    Ask a question to the agent.
    Returns SQL, DataFrame, and optionally chart data.

    Body:
        agent_id: Agent ID (required)
        question: Question to ask (required)
        in_context: Use context from previous questions (optional)
        is_chart: Generate chart data (optional)
        session_key: Optional. When non-empty after stripping, the same value is echoed in the
            response; otherwise a new session key is minted (same semantics as talkback).

    Response dicts include ``session_key`` (reused or minted) for clients to pass on later
    (e.g. to talkback endpoints).
    """
    # Track start time for logging
    question_start_time = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
    skip_user_question_log = bool(getattr(request, "is_dashboard", False))

    # Initialize variables for logging
    generated_sql = None
    api_endpoint = None
    api_params = None
    api_response_validator = None
    api_name = ""

    auth_uid = str(getattr(current_user, "id", None) or request.user_id)
    if talkback_client_session_key_is_empty(request):
        await _purge_talkback_sessions_older_than_24h()
    effective_session_key = _resolve_talkback_effective_session_key(request, auth_uid)

    def _sk(payload):
        return _attach_multi_agent_session_key(payload, effective_session_key)

    try:
        # 1. Normalize inputs
        service = None
        question_id = request.question_id
        user_id = str(request.user_id).strip()
        if user_id != auth_uid and auth_uid != "1":
            return await _return_sk_tb(
                "saved_question_unauthorized",
                {
                    "success": False,
                    "message": f"Not authorized to run saved questions for user_id {user_id}",
                    "details": {},
                },
            )
        
        # Normalize agent_id (omit None / empty; JSON list strings; multi-id lists).
        # Dashboard often sends only question_id — agent scope must come from the saved Question row.
        agent_ids = _normalize_agent_ids(request.agent_id)

        # Initialize early (used by talkback persist + validation); may be overridden by saved question.
        question_to_use = (
            (request.question if request.question is not None else getattr(request, "my_question", None)) or ""
        )

        primary_agent_id = agent_ids[0] if agent_ids else ""

        async def _return_sk_tb(route: str, d: Dict[str, Any]):
            if service:
                ok = bool(d.get("success"))
                msg = str(d.get("message") or "")
                err = None
                det = d.get("details")
                if not ok and isinstance(det, dict):
                    if det.get("error") is not None:
                        err = str(det.get("error"))
                    elif det.get("message") is not None:
                        err = str(det.get("message"))[:400]
                await _persist_multi_agent_talkback_turn(
                    service,
                    auth_uid,
                    effective_session_key,
                    agent_ids,
                    question_to_use,
                    ok,
                    msg,
                    route,
                    err,
                )
            return _sk(d)

        async def _return_json_tb(route: str, body: Dict[str, Any]):
            b = dict(body)
            b["session_key"] = effective_session_key
            if service:
                ok = bool(b.get("success"))
                msg = str(b.get("message") or "")
                err = None
                det = b.get("details")
                if not ok and isinstance(det, dict):
                    if det.get("error") is not None:
                        err = str(det.get("error"))
                    elif det.get("message") is not None:
                        err = str(det.get("message"))[:400]
                await _persist_multi_agent_talkback_turn(
                    service,
                    auth_uid,
                    effective_session_key,
                    agent_ids,
                    question_to_use,
                    ok,
                    msg,
                    route,
                    err,
                )
            return JSONResponse(content=_to_json_safe(b))

        if not agent_ids and not question_id:
            return await _return_sk_tb(
                "no_agent_ids",
                {"success": False, "message": "No agent IDs provided", "details": {}},
            )

        # Initialize variables that may be overridden by saved question
        sql_to_use = request.sql
        api_endpoint_to_use = request.api_endpoint
        api_params_to_use = request.api_params
        api_response_validator_to_use = request.api_response_validator
        chart_code_to_use = request.chart_json if request.chart_json is not None else getattr(request, "code", None)
        datasource_to_use = None
        request_datasource_override = normalize_request_datasource_override(
            getattr(request, "datasource", None)
        )
        if request_datasource_override == "API":
            datasource_to_use = "API"
        elif request_datasource_override == "FILE":
            datasource_to_use = "File"
        elif request_datasource_override == "DATABASE":
            datasource_to_use = "Database"

        # 2. Fetch saved question if question_id is provided
        if question_id:
            def fetch_saved_question():
                with get_db_session() as db:
                    if auth_uid == "1":
                        q = db.query(Question).filter(
                            Question.id == int(question_id),
                        ).first()
                    else:
                        q = db.query(Question).filter(
                            Question.id == int(question_id),
                            Question.user_id == user_id,
                        ).first()
                    if q:
                        return {
                            "question": q.question,
                            "agent_id": q.agent_id,
                            "sql": q.sql,
                            "api_endpoint": q.api_endpoint,
                            "api_params": q.api_params,
                            "api_response_validator": q.api_response_validator,
                            "chart": q.chart
                        }
                    return None
            
            saved_question = await run_in_threadpool(fetch_saved_question)
            if not saved_question:
                return await _return_sk_tb(
                    "saved_question_not_found",
                    {
                        "success": False,
                        "message": f"Saved question with id {question_id} not found for user_id {user_id}",
                        "details": {},
                    },
                )

            if not agent_ids:
                agent_ids = _normalize_agent_ids(saved_question.get("agent_id"))
                primary_agent_id = agent_ids[0] if agent_ids else ""
            
            # Populate variables from saved question if not already provided
            if not question_to_use:
                question_to_use = saved_question.get("question")
            if not sql_to_use:
                sql_to_use = saved_question.get("sql")
            if not api_endpoint_to_use:
                api_endpoint_to_use = saved_question.get("api_endpoint")
            if not api_params_to_use:
                api_params_to_use = saved_question.get("api_params")
            if not api_response_validator_to_use:
                api_response_validator_to_use = saved_question.get("api_response_validator")
            if not chart_code_to_use:
                chart_code_to_use = saved_question.get("chart")
            if not datasource_to_use:
                has_sql = _has_meaningful_saved_sql(saved_question.get("sql"))
                has_api = _has_meaningful_saved_api(saved_question)
                if has_sql and has_api:
                    datasource_to_use = None
                elif has_api:
                    datasource_to_use = "API"
                else:
                    datasource_to_use = "Database"

        if not agent_ids:
            return await _return_sk_tb(
                "no_agent_ids",
                {"success": False, "message": "No agent IDs provided", "details": {}},
            )
        primary_agent_id = agent_ids[0]

        request.question = question_to_use

        # Use dispatch-selected agent ids as the hard scope.
        # If datasource is explicitly selected, only keep dispatch agents that support it.
        if request_datasource_override:
            from app.services.agent_config import get_agent_by_id, ordered_available_datasources

            compatible_agent_ids: List[str] = []
            incompatible_agent_ids: Dict[str, List[str]] = {}
            for aid in agent_ids:
                row = get_agent_by_id(str(aid))
                avail = ordered_available_datasources(row) if row else []
                normalized_avail = [str(ds).strip().upper() for ds in avail]
                if request_datasource_override in normalized_avail:
                    compatible_agent_ids.append(str(aid))
                else:
                    incompatible_agent_ids[str(aid)] = avail

            if not compatible_agent_ids:
                return await _return_sk_tb(
                    "datasource_dispatch_mismatch",
                    {
                        "success": False,
                        "message": (
                            f"Datasource '{request.datasource}' is not enabled for the selected dispatch agent(s)."
                        ),
                        "details": {
                            "requested_datasource": request_datasource_override,
                            "selected_dispatch_agent_ids": agent_ids,
                            "agent_datasource_map": incompatible_agent_ids,
                        },
                    },
                )

            if len(compatible_agent_ids) < len(agent_ids):
                print(
                    "⚠️ [multi-agent-ask] filtered dispatch agents by datasource",
                    {
                        "requested_datasource": request_datasource_override,
                        "before": agent_ids,
                        "after": compatible_agent_ids,
                    },
                )

            agent_ids = compatible_agent_ids
            primary_agent_id = agent_ids[0]

        if len(agent_ids) == 1:
            datasource_to_use = _infer_ask_datasource(
                datasource_to_use,
                api_endpoint_to_use,
                api_params_to_use,
                multi_agent_unset_ok=False,
            )
        elif datasource_to_use is None:
            datasource_to_use = _infer_ask_datasource(
                None,
                api_endpoint_to_use,
                api_params_to_use,
                multi_agent_unset_ok=True,
            )

        if len(agent_ids) == 1:
            try:
                from app.services.agent_config import get_agent_by_id, ordered_available_datasources

                _row_ma = get_agent_by_id(str(primary_agent_id))
                _avail_ma = ordered_available_datasources(_row_ma) if _row_ma else []
                _aup_ma = [str(x).strip().upper() for x in _avail_ma]
                if "DATABASE" not in _aup_ma:
                    if "API" in _aup_ma:
                        datasource_to_use = "API"
                    elif "FILE" in _aup_ma and len(_aup_ma) == 1:
                        datasource_to_use = "File"
            except Exception:
                pass

        # 3. Setup Agent Service
        def get_service_safe(aid_val):
            try: return get_askme_service(agent_id=aid_val)
            except: return None
            
        service_agent_ids = agent_ids if len(agent_ids) > 1 else primary_agent_id
        service = await run_in_threadpool(get_service_safe, service_agent_ids)

        if not service:
            # Fallback for service
            for aid in agent_ids:
                s = await run_in_threadpool(get_service_safe, aid)
                if s:
                    service = s
                    primary_agent_id = aid
                    break
            if not service:
                return await _return_sk_tb(
                    "no_service",
                    {
                        "success": False,
                        "message": f"Agent configuration not found for {primary_agent_id}",
                        "details": {},
                    },
                )
        reset_global_usage_totals()

        # Contextual question generation
        
        question_to_process = question_to_use
        print(f"question_to_process: {question_to_process}")
        # if request.inContext and request.previous_questions:
        #     try:
        #         question_to_process = await run_in_threadpool(
        #             service.generate_contextual_question,
        #             previous_questions=request.previous_questions,
        #             current_question=question_to_use
        #         )
        #     except Exception as e:
        #         print(f"Error generating contextual question: {e}")

        print(f"question_to_process1: {question_to_process}")
        # Match single-agent ask: @askme / @web analyze prior df or web — do not run training/SQL routing.
        _q_low = (question_to_use or "").lower()
        # 3b. Training-driven pipeline when no preloaded SQL (1+ agents), then legacy multi-agent map
        multi_strategy = None
        datasource_map = None
        _sql_preloaded = _has_meaningful_saved_sql(sql_to_use)
        single_agent_dual_source = False
        if len(agent_ids) == 1 and not (request.datasource or "").strip():
            try:
                from app.services.agent_config import get_agent_by_id, ordered_available_datasources

                _row = get_agent_by_id(str(primary_agent_id))
                _avail = ordered_available_datasources(_row) if _row else []
                single_agent_dual_source = (
                    "DATABASE" in _avail and "API" in _avail
                )
            except Exception:
                single_agent_dual_source = False

        if (
            not _sql_preloaded
            and (len(agent_ids) > 1 or single_agent_dual_source)
            and "@askme" not in _q_low
            and "@web" not in _q_low
        ):
            try:
                _mr_pipe = int(
                    getattr(service, "_max_row_count", "5000") or 5000
                )
            except (TypeError, ValueError):
                _mr_pipe = 5000

            def _run_training_pipeline():
                return multi_agent_pipeline(
                    question_to_process,
                    agent_ids,
                    sql_to_use=None,
                    max_rows=_mr_pipe,
                    datasource_filter=request.datasource,
                )

            # multi_agent_pipeline only triggers create_user_question_log on full success below.
            # Log exceptions and non-success outcomes so failures are visible in user_question_logs.
            try:
                pipe = await run_in_threadpool(_run_training_pipeline)
            except Exception as pipe_exc:
                if not skip_user_question_log:
                    _exc_sql = json.dumps(
                        {
                            "training_pipeline_exception": True,
                            "error": str(pipe_exc),
                            "type": type(pipe_exc).__name__,
                        },
                        default=str,
                    )
                    if len(_exc_sql) > 50000:
                        _exc_sql = _exc_sql[:50000] + "...(truncated)"
                    await run_in_threadpool(
                        create_user_question_log,
                        user_id=user_id,
                        agent_id=(
                            agent_ids[0]
                            if len(agent_ids) == 1
                            else json.dumps(agent_ids)
                        ),
                        question=question_to_use,
                        sql=_exc_sql,
                        chart="N",
                        web="N",
                        executed="E",
                        start_time=question_start_time,
                        api_endpoint="",
                        api_params="",
                        api_response_validator="",
                    )
                raise

            if pipe.get("used_pipeline") and pipe.get("success"):
                from app.services.multi_agent_pipeline import (
                    merge_dataframes,
                    build_response as _build_pipeline_response,
                    infer_merge_strategy,
                )

                raw_frames = pipe.get("df")
                file_results = pipe.get("file_results") or []
                sub_tasks = pipe.get("sub_tasks") or []
                sql_traces = pipe.get("sql_traces") or []
                api_traces_pipe = pipe.get("api_traces") or []
                sub_tasks_for_resp = _enrich_sub_tasks_with_api_traces(
                    sub_tasks, api_traces_pipe
                )
                _comb_sql, _comb_ep, _comb_pr, _comb_val = (
                    _log_fields_from_training_pipeline(pipe)
                )

                # ── Re-merge with strategy awareness ──────────────────────────
                # pipe["df"] is already a merged DataFrame produced by execute_plan.
                # For multi-agent requests we re-run merge_dataframes with the
                # per-agent frames so the correct strategy (join / union / compare /
                # forecast / single) is applied.  For a single agent the existing
                # merged frame is used as-is (strategy="single").
                if len(agent_ids) > 1 and isinstance(raw_frames, pd.DataFrame) and not raw_frames.empty:
                    # Collect per-agent DataFrames stored in sql_traces / api_traces
                    _agent_frames: List[Any] = []
                    for _st in sql_traces:
                        _df_st = _st.get("df") if isinstance(_st, dict) else None
                        if _df_st is not None and isinstance(_df_st, pd.DataFrame) and not _df_st.empty:
                            _agent_frames.append(_df_st)
                    for _at in api_traces_pipe:
                        _res = (_at.get("result") or {}) if isinstance(_at, dict) else {}
                        _df_at = _res.get("df") if isinstance(_res, dict) else None
                        if _df_at is not None and isinstance(_df_at, pd.DataFrame) and not _df_at.empty:
                            _agent_frames.append(_df_at)
                    if len(_agent_frames) > 1:
                        _strategy = infer_merge_strategy(question_to_process, _agent_frames)
                        merged_df, merge_meta = merge_dataframes(
                            _agent_frames,
                            strategy=_strategy,
                            question=question_to_process,
                            max_rows=1000,
                        )
                    else:
                        # Per-agent frames not cached in traces → use the already-merged df
                        merged_df = raw_frames
                        merge_meta = pipe.get("merge_meta") or {"merge_strategy": "single", "strategies": []}
                else:
                    merged_df = raw_frames if isinstance(raw_frames, pd.DataFrame) else pd.DataFrame()
                    merge_meta = pipe.get("merge_meta") or {
                        "merge_strategy": "single" if len(agent_ids) == 1 else "unknown",
                        "strategies": [],
                    }

                # When agents have disjoint schemas (e.g. invoice columns ≠ sales columns
                # ≠ trip columns), merge_results returns an empty merged DataFrame but
                # stores per-agent frames in separate_dfs.  Pull those from the pipeline
                # so the response builder can emit a per-agent payload instead of nothing.
                separate_dfs = pipe.get("separate_dfs") or []

                # has_tabular mirrors the logic in _execute_plan_async:
                #   True if either the merged frame has rows OR there are separate per-agent frames.
                has_tabular = (
                    merged_df is not None and not getattr(merged_df, "empty", True)
                ) or bool(separate_dfs)

                # ── Build standard response payload ────────────────────────────
                _built = _build_pipeline_response(
                    merged_df,
                    merge_meta,
                    sub_tasks=clean_nan_values(sub_tasks_for_resp),
                    file_results=clean_nan_values(file_results),
                    question=question_to_process,
                    api_traces=clean_nan_values(api_traces_pipe),
                )

                _st_ma = [dict(x) for x in (sub_tasks or []) if isinstance(x, dict)]
                sql_for_log = json.dumps(
                    {
                        "training_driven_pipeline": True,
                        "mixed_datasource_execution": _infer_mixed_datasource_execution(
                            _st_ma
                        ),
                        "sub_tasks": clean_nan_values(sub_tasks),
                        "sql_traces": _flatten_sql_traces_list(sql_traces),
                        "api_traces": clean_nan_values(api_traces_pipe),
                    },
                    default=str,
                )
                if not _comb_sql:
                    _comb_sql = sql_for_log

                df_records = []
                formatted_columns = []
                if has_tabular:
                    df_records = clean_nan_values(
                        merged_df.to_dict("records")
                    )
                    columns = list(merged_df.columns)
                    formatted_columns = [
                        str(c).replace("_", " ").capitalize() for c in columns
                    ]

                # --- 1) Condition: disjoint tabular frames (merge not possible) ---
                # merge_results sets merge_strategy "separate_frames" and fills separate_dfs.
                agents_list: List[Dict[str, Any]] = []
                if separate_dfs:
                    for item in separate_dfs:
                        item_df = item.get("df")
                        if item_df is None or getattr(item_df, "empty", True):
                            continue
                        cols = list(item_df.columns)
                        agents_list.append(
                            {
                                "agent_id": str(item.get("agent_id") or ""),
                                "headers": [
                                    str(c).replace("_", " ").capitalize()
                                    for c in cols
                                ],
                                "data": clean_nan_values(item_df.to_dict("records")),
                                "chart_code": "N",
                            }
                        )

                _strategy = merge_meta.get("merge_strategy") or ""
                # --- 2) List-shaped df only when multiple non-mergeable frames (explicit flag) ---
                use_disjoint_list = (
                    _strategy == "separate_frames"
                    and len(agents_list) > 1
                )

                # Same agent scope as logging (single id string vs JSON array when multiple).
                _agent_id_for_df = (
                    str(agent_ids[0])
                    if len(agent_ids) == 1
                    else (json.dumps(agent_ids) if agent_ids else "")
                )

                # --- 3) Final df payload (unchanged shapes for merge success / single table) ---
                if use_disjoint_list:
                    df_payload = _df_disjoint_tabular_list(agents_list)
                elif separate_dfs and agents_list:
                    df_payload = _df_unmerged_multi_agent(
                        agents_list,
                        multi_agent_mixed=True,
                        training_driven_pipeline=True,
                        chart_code="N",
                    )
                    df_payload["agent_id"] = _agent_id_for_df
                else:
                    df_payload = {
                        "agent_id": _agent_id_for_df,
                        "headers": formatted_columns,
                        "data": df_records,
                        "chart_code": "N",
                        "multi_agent_mixed": True,
                        "tabular_from_merge": has_tabular,
                        "training_driven_pipeline": True,
                    }
                    if len(agent_ids) > 1:
                        df_payload["merged"] = True

                _first_tab_aid_tr = None
                for _ag in agents_list or []:
                    _r = _ag.get("data") or []
                    if _r and len(_r) < 101:
                        _first_tab_aid_tr = str(_ag.get("agent_id") or "")
                        break
                _populate_per_agent_and_merged_chart_codes(
                    df_payload,
                    service=service,
                    question_default=question_to_process,
                    chart_code_to_use=chart_code_to_use,
                    sub_tasks=sub_tasks_for_resp,
                    root_chart_code_for_aid=None,
                    first_tabular_agent_id=_first_tab_aid_tr,
                )

                # Non-training multi_agent flow sets details.chart (Plotly JSON) by exec'ing
                # chart_code; the training pipeline only stored code on df. UIs that bind to
                # details.chart never receive a figure unless we run the same step here.
                _training_chart_json: Optional[Dict[str, Any]] = None
                _training_chart_code_str: Optional[str] = None
                if (
                    isinstance(df_payload, dict)
                    and has_tabular
                    and df_records
                    and len(df_records) < 101
                ):
                    _cc_tr = df_payload.get("chart_code")
                    if _cc_tr and str(_cc_tr).strip() and _cc_tr != "N":
                        try:
                            _df_chart = pd.DataFrame(df_records)
                            _g = {
                                "pd": pd,
                                "json": json,
                                "json_module": json,
                                "go": go,
                                "px": px,
                            }
                            _loc: Dict[str, Any] = {
                                "df": _df_chart,
                                "pd": pd,
                                "json": json,
                                "json_module": json,
                                "go": go,
                                "px": px,
                            }
                            exec(_cc_tr, _g, _loc)
                            _cj = _loc.get("chart_json")
                            if _cj is not None and isinstance(_cj, dict):
                                _training_chart_json = clean_nan_values(_cj)
                                _training_chart_code_str = _cc_tr
                        except Exception as e:
                            print(f"training_driven details.chart: {e}")

                response_details = {
                    "sql": sql_for_log,
                    "df": df_payload,
                    "file_results": clean_nan_values(file_results),
                    "sub_tasks": clean_nan_values(sub_tasks_for_resp),
                    "multi_agent_datasource_map": clean_nan_values(
                        pipe.get("multi_agent_datasource_map") or {}
                    ),
                    "raw_decomposition": clean_nan_values(
                        pipe.get("raw_decomposition") or []
                    ),
                    "datasource_map_list": clean_nan_values(
                        pipe.get("datasource_map_list") or []
                    ),
                    "original_question": question_to_use if request.inContext else None,
                    "api_endpoint": _comb_ep or "",
                    "api_params": _comb_pr or "",
                    "api_response_validator": _comb_val or "",
                }
                if _training_chart_json is not None:
                    response_details["chart"] = _training_chart_json
                if _training_chart_code_str:
                    response_details["chart_code"] = _training_chart_code_str
                _attach_talkback_documents(
                    response_details,
                    agent_id=agent_ids[0] if agent_ids else None,
                )
                try:
                    if use_disjoint_list and agents_list:
                        talkback_columns = []
                        for _b in agents_list:
                            talkback_columns.extend(_b.get("headers") or [])
                        context_str = (
                            f"Disjoint results from {len(agents_list)} sources (could not merge). "
                        )
                        for _b in agents_list:
                            _aid = _b.get("agent_id")
                            _rows = _b.get("data") or []
                            context_str += f"\nAgent {_aid}: {len(_rows)} row(s). "

                    else:
                        talkback_columns = formatted_columns or []
                        context_str = (
                            f"Columns: {str(talkback_columns)}"
                            if talkback_columns
                            else "No data returned."
                        )
                        if df_records and len(df_records) > 0 and (settings.default_use_qwen_for_summary_and_web() and _qwen_configured()):
                            records_str = str(df_records)
                            context_str += f"\nSample data: {records_str[:500]}"

                    talkback_tb_df = None
                    if use_disjoint_list and agents_list:
                        _rows_tb = []
                        for _b in agents_list:
                            _rows_tb.extend(_b.get("data") or [])
                        if _rows_tb:
                            try:
                                talkback_tb_df = pd.DataFrame(_rows_tb)
                            except Exception:
                                talkback_tb_df = None
                    elif df_records:
                        try:
                            talkback_tb_df = pd.DataFrame(df_records)
                        except Exception:
                            talkback_tb_df = None
                    talkback = await run_in_threadpool(
                        service.generate_continuation_talkback_cached,
                        question=question_to_process,
                        answer_context=context_str,
                        columns=talkback_columns,
                        n_items=3,
                        df=talkback_tb_df,
                        sql=_comb_sql,
                        api_endpoint=_comb_ep,
                        api_params=_comb_pr,
                        api_response_validator=_comb_val,
                        use_qwen=settings.default_use_qwen_for_summary_and_web()
                    )
                    response_details["talkback"] = talkback
                except Exception as e:
                    print(f"Error generating talkback for training-driven multi-agent: {e}")
                if not skip_user_question_log:
                    # One row per request: combine all DB sql_traces + API api_traces
                    # into a single log entry using the existing helper.
                    # For mixed agents (e.g. agent 1 DB + agent 8 API):
                    #   sql        → SQL from DB agent(s)
                    #   api_*      → endpoint/params/validator from API agent(s)
                    #   agent_id   → JSON array when multiple agents
                    # (_comb_* computed above for continuation talkback + logging)
                    await run_in_threadpool(
                        create_user_question_log,
                        user_id=user_id,
                        agent_id=(
                            agent_ids[0]
                            if len(agent_ids) == 1
                            else json.dumps(agent_ids)
                        ),
                        question=question_to_use,
                        sql=_comb_sql,
                        chart=_training_chart_code_str or "N",
                        web="N",
                        executed="Y",
                        start_time=question_start_time,
                        api_endpoint=_comb_ep,
                        api_params=_comb_pr,
                        api_response_validator=_comb_val,
                    )
                try:
                    await run_in_threadpool(
                        service.merge_sql_query_summaries_into_details_cached,
                        response_details,
                        question_to_process,
                    )
                except Exception as _sq_sum_trma_exc:
                    print(f"sql_query_summaries (training multi-agent): {_sq_sum_trma_exc}")
                return await _return_json_tb(
                    "training_pipeline",
                    {
                        "success": True,
                        "message": "Question processed successfully (training-driven multi-agent)",
                        "details": response_details,
                        "question": question_to_process,
                    },
                )

            # Pipeline returned without success: one row via _log_fields_from_training_pipeline + create_user_question_log.
            if not skip_user_question_log:
                sql_a, api_ep_a, api_pr_a, api_val_a = _log_fields_from_training_pipeline(
                    pipe
                )
                if not sql_a:
                    sql_a = json.dumps(
                        {
                            "used_pipeline": pipe.get("used_pipeline"),
                            "success": pipe.get("success"),
                            "reason": pipe.get("reason"),
                        },
                        default=str,
                    )
                if len(sql_a) > 50000:
                    sql_a = sql_a[:50000] + "...(truncated)"
                await run_in_threadpool(
                    create_user_question_log,
                    user_id=user_id,
                    agent_id=(
                        agent_ids[0]
                        if len(agent_ids) == 1
                        else json.dumps(agent_ids)
                    ),
                    question=question_to_use,
                    sql=sql_a,
                    chart="N",
                    web="N",
                    executed="N",
                    start_time=question_start_time,
                    api_endpoint=api_ep_a,
                    api_params=api_pr_a,
                    api_response_validator=api_val_a,
                )

        if (
            len(agent_ids) > 1
            and "@askme" not in _q_low
            and "@web" not in _q_low
        ):
            def _build_map():
                return build_multi_agent_datasource_map(
                    agent_ids, question_to_process, getattr(request, "datasource", None)
                )

            datasource_map, map_err = await run_in_threadpool(_build_map)
            if map_err:
                return await _return_sk_tb(
                    "datasource_map_error",
                    {
                        "success": False,
                        "message": map_err,
                        "details": {},
                    },
                )
            multi_strategy = classify_multi_agent_strategy(datasource_map, agent_ids)

            if multi_strategy == "MIXED":
                try:
                    _mr = int(getattr(service, "_max_row_count", "5000") or 5000)
                except (TypeError, ValueError):
                    _mr = 5000

                _mixed_reuse_both_payload = (
                    _has_meaningful_saved_sql(sql_to_use)
                    and _has_meaningful_request_api(
                        api_endpoint_to_use, api_params_to_use
                    )
                )

                def _run_mixed():
                    return execute_mixed_multi_agent_sync(
                        agent_ids,
                        datasource_map,
                        question_to_process,
                        sql_to_use,
                        max_rows=_mr,
                        api_endpoint_to_use=api_endpoint_to_use,
                        api_params_to_use=api_params_to_use,
                        api_response_validator_to_use=api_response_validator_to_use,
                        mixed_reuse_both_payload=_mixed_reuse_both_payload,
                    )

                mix_out = await run_in_threadpool(_run_mixed)
                merged_df = mix_out.get("df")
                file_results = mix_out.get("file_results") or []
                gen_sql_mixed = mix_out.get("generated_sql")
                merge_meta = mix_out.get("merge_meta") or {}
                mix_success = bool(mix_out.get("success"))
                has_tabular = bool(mix_out.get("has_tabular"))

                if not mix_success:
                    if not skip_user_question_log:
                        await run_in_threadpool(
                            create_user_question_log,
                            user_id=user_id,
                            agent_id=json.dumps(agent_ids),
                            question=question_to_use,
                            sql=json.dumps(datasource_map),
                            chart="N",
                            web="N",
                            executed="N",
                            start_time=question_start_time,
                            api_endpoint="",
                            api_params="",
                            api_response_validator="",
                        )
                    return await _return_sk_tb(
                        "mixed_datasource_fail",
                        {
                            "success": False,
                            "message": "No results from mixed multi-agent execution",
                            "details": {
                                "datasource_map": datasource_map,
                                "merge_meta": merge_meta,
                                "file_results": clean_nan_values(file_results),
                            },
                        },
                    )

                df_records = []
                formatted_columns = []
                if (
                    merged_df is not None
                    and hasattr(merged_df, "empty")
                    and not merged_df.empty
                ):
                    df_records = clean_nan_values(merged_df.to_dict("records"))
                    columns = list(merged_df.columns)
                    formatted_columns = [
                        str(c).replace("_", " ").capitalize() for c in columns
                    ]

                sql_for_log = _details_sql_envelope_mixed(
                    gen_sql_mixed,
                    question_to_process,
                    primary_agent_id,
                    agent_ids,
                    datasource_map,
                    mix_out.get("api_details") or {},
                    training_driven_pipeline=False,
                )
                _agent_id_for_df_mixed = (
                    str(agent_ids[0])
                    if len(agent_ids) == 1
                    else (json.dumps(agent_ids) if agent_ids else "")
                )
                _mix_ep, _mix_pr, _mix_val = _format_api_fields_from_merge_result(
                    mix_out.get("api_details") or {}
                )
                df_payload_mixed: Dict[str, Any] = {
                    "agent_id": _agent_id_for_df_mixed,
                    "headers": formatted_columns,
                    "data": df_records,
                    "chart_code": "N",
                    "multi_agent_mixed": True,
                    "tabular_from_merge": has_tabular,
                }
                if len(agent_ids) > 1:
                    df_payload_mixed["merged"] = True
                _populate_per_agent_and_merged_chart_codes(
                    df_payload_mixed,
                    service=service,
                    question_default=question_to_process,
                    chart_code_to_use=chart_code_to_use,
                    sub_tasks=None,
                    root_chart_code_for_aid=None,
                    first_tabular_agent_id=None,
                )
                _mixed_ds_chart_json: Optional[Dict[str, Any]] = None
                _mixed_ds_chart_code_str: Optional[str] = None
                if has_tabular and df_records and len(df_records) < 101:
                    _cc_mixed = df_payload_mixed.get("chart_code")
                    if _cc_mixed and str(_cc_mixed).strip() and _cc_mixed != "N":
                        try:
                            _df_mixed_chart = pd.DataFrame(df_records)
                            _g_m = {
                                "pd": pd,
                                "json": json,
                                "json_module": json,
                                "go": go,
                                "px": px,
                            }
                            _loc_m: Dict[str, Any] = {
                                "df": _df_mixed_chart,
                                "pd": pd,
                                "json": json,
                                "json_module": json,
                                "go": go,
                                "px": px,
                            }
                            exec(_cc_mixed, _g_m, _loc_m)
                            _cj_m = _loc_m.get("chart_json")
                            if _cj_m is not None and isinstance(_cj_m, dict):
                                _mixed_ds_chart_json = clean_nan_values(_cj_m)
                                _mixed_ds_chart_code_str = _cc_mixed
                        except Exception as e:
                            print(f"mixed_datasource details.chart: {e}")
                response_details: Dict[str, Any] = {
                    "sql": sql_for_log,
                    "df": df_payload_mixed,
                    "file_results": clean_nan_values(file_results),
                    "question": question_to_process,
                    "original_question": question_to_use if request.inContext else None,
                    "api_endpoint": _mix_ep or "",
                    "api_params": _mix_pr or "",
                    "api_response_validator": _mix_val or "",
                    "datasource_map": datasource_map,
                    "merge_meta": merge_meta,
                }
                if _mixed_ds_chart_json is not None:
                    response_details["chart"] = _mixed_ds_chart_json
                if _mixed_ds_chart_code_str:
                    response_details["chart_code"] = _mixed_ds_chart_code_str
                _attach_talkback_documents(
                    response_details,
                    agent_id=agent_ids[0] if agent_ids else None,
                )
                if not skip_user_question_log:
                    await run_in_threadpool(
                        create_user_question_log,
                        user_id=user_id,
                        agent_id=json.dumps(agent_ids),
                        question=question_to_use,
                        sql=sql_for_log,
                        chart=_mixed_ds_chart_code_str or "N",
                        web="N",
                        executed="Y",
                        start_time=question_start_time,
                        api_endpoint=_mix_ep or "",
                        api_params=_mix_pr or "",
                        api_response_validator=_mix_val or "",
                    )
                try:
                    await run_in_threadpool(
                        service.merge_sql_query_summaries_into_details_cached,
                        response_details,
                        question_to_process,
                    )
                except Exception as _sq_sum_mix_exc:
                    print(f"sql_query_summaries (mixed): {_sq_sum_mix_exc}")
                return await _return_json_tb(
                    "mixed_datasource",
                    {
                        "success": True,
                        "message": "Question processed successfully (mixed datasources)",
                        "details": response_details,
                        "question": question_to_process,
                    },
                )

            if multi_strategy == "DB_MULTI":
                datasource_to_use = "Database"
            elif multi_strategy == "API_MULTI":
                datasource_to_use = "API"
            elif not datasource_to_use and datasource_map:
                t0 = datasource_map.get(primary_agent_id, {}).get("type", "DATABASE")
                datasource_to_use = (
                    "API"
                    if t0 == "API"
                    else ("File" if t0 == "FILE" else "Database")
                )

        if not datasource_to_use:
            datasource_to_use = "Database"

        # 4. Handle API Logic
        if datasource_to_use.upper() == "API":
            try:
                _max_rows = getattr(service, "_max_row_count", "1000")
                api_max_rows = int(_max_rows) if _max_rows is not None else 1000
            except (TypeError, ValueError):
                api_max_rows = 1000
            
            # Reuse logic
            if api_endpoint_to_use and api_params_to_use and api_endpoint_to_use != "N" and api_endpoint_to_use != "[]":
                print("API parameters present - Reusing saved API calls")
                try:
                    def parse_list_field(field_val):
                        if not field_val or field_val == "N": return []
                        if isinstance(field_val, list): return field_val
                        if isinstance(field_val, str):
                            field_val = field_val.strip()
                            if field_val.startswith('['):
                                try: return json.loads(field_val)
                                except: pass
                        return [field_val]

                    api_endpoints_list = parse_list_field(api_endpoint_to_use)
                    api_params_list = parse_list_field(api_params_to_use)
                    api_validators_list = parse_list_field(api_response_validator_to_use)
                    
                    apis_list = await run_in_threadpool(service.get_all_apis, primary_agent_id)
                    
                    agents = []
                    api_details = []
                    import ast
                    
                    for i, endpoint in enumerate(api_endpoints_list):
                        params = api_params_list[i] if i < len(api_params_list) else {}
                        validator = api_validators_list[i] if i < len(api_validators_list) else ""
                        if not validator: validator = ""
                        
                        # Fix up params
                        if isinstance(params, str) and params.strip():
                            try:
                                if params.strip().startswith('{'): params = ast.literal_eval(params)
                                elif params.strip().startswith('['):
                                    temp_list = json.loads(params)
                                    params = temp_list[0] if temp_list else {}
                                else: params = json.loads(params)
                            except:
                                try: params = json.loads(params)
                                except: pass
                        if not isinstance(params, dict): params = {}

                        api_info = next((a for a in apis_list if a.get("endpoint") == endpoint), {})
                        is_sap = api_info.get("extra1", "N")
                        
                        sub_df = None
                        if is_sap == "Y":
                            sap_result = await run_in_threadpool(
                                service.process_sap_query,
                                service_root=endpoint,
                                entity_set=api_info.get("base_params"),
                                user_query=question_to_use,
                                response_schema=api_info.get("response_schema"),
                                transformation_code=validator,
                                username="S4h_aa",
                                password="Welcome1",
                                verify_ssl=False,
                                api_query_params=params
                            )
                            sub_df = sap_result.get("df")
                        else:
                            _, _, sub_df, _ = await run_in_threadpool(
                                service.call_api_and_transform,
                                params, endpoint, api_info.get("response_schema", ""), validator, question_to_use
                            )
                        
                        if sub_df is not None:
                            if hasattr(sub_df, "head"):
                                sub_df = sub_df.head(api_max_rows)
                            sub_cols = list(sub_df.columns)
                            formatted_sub_cols = [str(c).replace("_", " ").capitalize() for c in sub_cols]
                            sub_records = clean_nan_values(sub_df.to_dict("records"))
                            agent_id_label = agent_ids[i] if i < len(agent_ids) else f"agent_{i+1}"
                            agents.append({
                                "agent_id": agent_id_label,
                                "headers": formatted_sub_cols,
                                "data": sub_records,
                                "api_name": api_info.get("name") if api_info else "",
                                "chart_code": "N",
                            })
                            api_details.append({"endpoint": endpoint, "agent_id": agent_id_label, "name": api_info.get("name") if api_info else ""})
                    
                    if agents:
                        _api_traces_reuse: List[Dict[str, Any]] = []
                        _sub_tasks_reuse: List[Dict[str, Any]] = []
                        for i, endpoint in enumerate(api_endpoints_list):
                            pr = api_params_list[i] if i < len(api_params_list) else ""
                            val = api_validators_list[i] if i < len(api_validators_list) else ""
                            aid_l = agent_ids[i] if i < len(agent_ids) else primary_agent_id
                            _api_traces_reuse.append(
                                {
                                    "agent_id": str(aid_l),
                                    "sub_question": question_to_process,
                                    "result": _api_result_dict_for_envelope(endpoint, pr, val),
                                }
                            )
                            _sub_tasks_reuse.append(
                                {
                                    "sub_question": question_to_process,
                                    "agent_id": str(aid_l),
                                    "datasource": "API",
                                }
                            )
                        _sql_reuse = _details_sql_envelope_api(
                            sub_tasks=_sub_tasks_reuse,
                            api_traces=_api_traces_reuse,
                            training_driven_pipeline=False,
                        )
                        _aid_top_reuse = (
                            str(agent_ids[0])
                            if len(agent_ids) == 1
                            else (json.dumps(agent_ids) if agent_ids else "")
                        )
                        df_response = _df_api_multi_frame_payload(
                            agents,
                            fallback_agent_id=_aid_top_reuse,
                            multi_agent_mixed=len(agent_ids) > 1,
                        )
                        response_details = {
                            "sql": _sql_reuse,
                            "df": df_response,
                            "question": question_to_use,
                            "original_question": None,
                            "api_endpoint": api_endpoints_list,
                            "api_params": api_params_list,
                            "api_response_validator": api_validators_list,
                        }
                        return await _return_json_tb(
                            "api_reuse",
                            {
                                "success": True,
                                "message": "Question processed successfully",
                                "details": response_details,
                                "question": question_to_process,
                            },
                        )
                except Exception as e:
                    print(f"Error in multi-agent API reuse: {e}")

            # Generation logic
            all_apis = []
            for aid in agent_ids:
                def _get_api(aid_val):
                    try:
                        s_val = get_askme_service(agent_id=aid_val)
                        return s_val.get_all_apis(aid_val)
                    except: return []
                apis = await run_in_threadpool(_get_api, aid)
                if apis: all_apis.extend(apis)
                
            if not all_apis:
                return await _return_sk_tb(
                    "api_no_candidates",
                    {"success": False, "message": "No APIs configured across selected agents", "details": {}},
                )
                
            def select_and_call():
                svc = service._ensure_api_chat_service()
                selected_apis = svc._select_api_from_candidates(request.question, all_apis)
                if not selected_apis: return None
                print(f"DEBUG: Selected APIs: {[a.get('name') for a in selected_apis]}")
                return svc.call_multiple_apis_and_merge(selected_apis=selected_apis, user_query=request.question)
                
            merge_result = await run_in_threadpool(select_and_call)
            if not merge_result or not merge_result.get("success"):
                return await _return_sk_tb(
                    "api_merge_fail",
                    {
                        "success": False,
                        "message": merge_result.get("message", "No results returned from APIs")
                        if merge_result
                        else "No matching APIs found",
                        "details": {},
                    },
                )
            
            df = merge_result.get("df")
            api_details = merge_result.get("api_details", [])
            api_endpoints_list = merge_result.get("api_endpoints", [])
            api_params_list = merge_result.get("api_params", [])
            api_validators_list = merge_result.get("api_validators", [])
            
            # Cap rows by agent max_row_count (from agent config)
            if isinstance(df, list) and df and isinstance(df[0], pd.DataFrame):
                df = [sub.head(api_max_rows) if hasattr(sub, "head") else sub for sub in df]
            elif df is not None and hasattr(df, "head"):
                df = df.head(api_max_rows)
            
            _api_traces_gen: List[Dict[str, Any]] = []
            _sub_tasks_gen: List[Dict[str, Any]] = []
            for i, ep in enumerate(api_endpoints_list):
                pr = api_params_list[i] if i < len(api_params_list) else ""
                val = api_validators_list[i] if i < len(api_validators_list) else ""
                det = api_details[i] if i < len(api_details) else {}
                aid_g = str(det.get("agent_id", "") or (agent_ids[0] if agent_ids else ""))
                _api_traces_gen.append(
                    {
                        "agent_id": aid_g,
                        "sub_question": question_to_process,
                        "result": _api_result_dict_for_envelope(ep, pr, val),
                    }
                )
                _sub_tasks_gen.append(
                    {
                        "sub_question": question_to_process,
                        "agent_id": aid_g,
                        "datasource": "API",
                    }
                )
            _sql_gen = _details_sql_envelope_api(
                sub_tasks=_sub_tasks_gen,
                api_traces=_api_traces_gen,
                training_driven_pipeline=False,
            )
            _aid_top_gen = (
                str(agent_ids[0])
                if len(agent_ids) == 1
                else (json.dumps(agent_ids) if agent_ids else "")
            )

            if isinstance(df, list) and df and isinstance(df[0], pd.DataFrame):
                agents = []
                for i, sub_df in enumerate(df):
                    sub_cols = list(sub_df.columns)
                    sub_records = clean_nan_values(sub_df.to_dict("records"))
                    agent_id_val = str(api_details[i].get("agent_id", "")) if i < len(api_details) else ""
                    agents.append({
                        "agent_id": agent_id_val,
                        "headers": [str(c).capitalize() for c in sub_cols],
                        "data": sub_records,
                        "api_name": api_details[i].get("name", "") if i < len(api_details) else "",
                        "chart_code": "N",
                    })
                df_response = _df_api_multi_frame_payload(
                    agents,
                    fallback_agent_id=_aid_top_gen,
                    multi_agent_mixed=len(agent_ids) > 1,
                )
            else:
                df_response = (
                    {
                        "agent_id": _aid_top_gen,
                        "headers": [str(c).capitalize() for c in df.columns],
                        "data": clean_nan_values(df.to_dict("records")),
                        "api_name": api_details[0].get("name") if api_details else "",
                        "chart_code": "N",
                        "merged": True,
                        "multi_agent_mixed": len(agent_ids) > 1,
                        "tabular_from_merge": df is not None and not getattr(df, "empty", True),
                    }
                    if df is not None
                    else {
                        "agent_id": _aid_top_gen,
                        "headers": [],
                        "data": [],
                        "chart_code": "N",
                        "merged": False,
                        "multi_agent_mixed": len(agent_ids) > 1,
                        "tabular_from_merge": False,
                    }
                )

            response_details = {
                "sql": _sql_gen,
                "df": df_response,
                "question": request.question,
                "original_question": None,
                "api_endpoint": api_endpoints_list,
                "api_params": api_params_list,
                "api_response_validator": api_validators_list,
            }
            # This branch returns before the main DB/SQL chart block; build small-table Plotly here.
            _api_merge_chart = "N"
            if isinstance(df_response, dict):
                _apix = (
                    df[0]
                    if isinstance(df, list) and len(df) == 1 and isinstance(df[0], pd.DataFrame)
                    else (
                        df
                        if not isinstance(df, list)
                        else None
                    )
                )
                if _apix is not None and isinstance(_apix, pd.DataFrame) and not getattr(
                    _apix, "empty", True
                ):
                    _arec = clean_nan_values(_apix.to_dict("records"))
                    _acols = list(_apix.columns) if _arec else []
                    if _arec and _acols and 0 < len(_arec) < 101:
                        _cj: Any = None
                        _jc: str = "N"
                        try:
                            if (
                                chart_code_to_use
                                and str(chart_code_to_use).strip()
                                and chart_code_to_use != "N"
                            ):
                                try:
                                    _jc = chart_code_to_use
                                    _g, _lv = _chart_code_exec_context(pd.DataFrame(_apix))
                                    exec(_jc, _g, _lv)
                                    _cj = _chart_json_from_exec_locals(_lv)
                                except Exception as _rc:
                                    print(f"[api_merge] chart reuse: {_rc}")
                                    _cj = None
                                    _jc = "N"
                            if _cj is None:
                                _jc = await run_in_threadpool(
                                    service.generate_json_code_cached,
                                    question=question_to_process,
                                    columns=_acols,
                                    model="gpt-4.1",
                                )
                                if _jc and _jc != "N":
                                    _g, _lv = _chart_code_exec_context(pd.DataFrame(_apix))
                                    exec(_jc, _g, _lv)
                                    _cj = _chart_json_from_exec_locals(_lv)
                            if _cj is not None:
                                response_details["chart"] = clean_nan_values(_cj)
                                response_details["chart_code"] = _jc
                                _api_merge_chart = _jc
                        except Exception as _cge:
                            print(f"[api_merge] chart: {_cge}")
                        df_response["chart_code"] = _api_merge_chart
                    else:
                        df_response["chart_code"] = "N"
                else:
                    df_response["chart_code"] = "N"
            # Log question
            await run_in_threadpool(
                create_user_question_log, user_id=user_id, agent_id=json.dumps(agent_ids), question=request.question,
                sql=_sql_gen, chart=_api_merge_chart, web="N", executed="Y", start_time=question_start_time,
                api_endpoint=json.dumps(api_endpoints_list), api_params=json.dumps(api_params_list), api_response_validator=json.dumps(api_validators_list)
            )
            try:
                # Use a summary of the agents' data for talkback
                if isinstance(df_response, list) and df_response:
                    summary_text = f"API results from {len(df_response)} agents."
                elif isinstance(df_response, dict) and "agents" in df_response:
                    summary_text = f"API results from {len(df_response['agents'])} agents."
                else:
                    summary_text = "API results found."
                tb_cols_ma = []
                if df is not None:
                    if isinstance(df, list) and df and isinstance(df[0], pd.DataFrame):
                        try:
                            tb_cols_ma = list(pd.concat(df, ignore_index=True).columns)
                        except Exception:
                            tb_cols_ma = list(df[0].columns) if df[0] is not None else []
                    elif hasattr(df, "columns"):
                        tb_cols_ma = list(df.columns)
                talkback = await run_in_threadpool(
                    service.generate_continuation_talkback_cached,
                    question=request.question,
                    answer_context=summary_text,
                    columns=tb_cols_ma,
                    n_items=3,
                    df=df,
                    sql=None,
                    api_endpoint=api_endpoints_list,
                    api_params=api_params_list,
                    api_response_validator=api_validators_list,
                    use_qwen=settings.default_use_qwen_for_summary_and_web(),
                )
                response_details["talkback"] = talkback
            except Exception as e:
                print(f"Error generating talkback for API: {e}")

            return await _return_json_tb(
                "api_merge",
                {
                    "success": True,
                    "message": "Question processed successfully",
                    "details": response_details,
                    "question": question_to_process,
                },
            )

        
        # Handle @askme analysis requests
        if question_to_use and "@askme" in question_to_use and settings.IS_ASKME_ACTIVE:
            query = question_to_use.replace("@askme", "").strip()

            if not query:
                if not skip_user_question_log:
                    await run_in_threadpool(
                        create_user_question_log,
                        user_id=user_id,
                        agent_id=agent_ids if len(agent_ids) > 1 else primary_agent_id,
                        question=question_to_use,
                        sql="N",
                        chart="N",
                        web="N",
                        executed="N",
                        start_time=question_start_time,
                        api_endpoint="",
                        api_params="",
                        api_response_validator=""
                    )
                return await _return_sk_tb(
                    "askme_analysis",
                    {
                        "success": False,
                        "message": "Please provide your analysis request after @askme (e.g., 'build a forecast', 'do trend analysis', 'detailed summary')",
                        "details": {"question": request.question},
                    },
                )
            
            # Get dataframe - either from request or need to run a query first
            analysis_df = None
            
            # Option 1: DataFrame provided in request
            if request.df:
                # Parse DF input
                request.df = service._parse_df_input(request.df)
                
            if request.df and len(request.df) > 0:
                analysis_df = pd.DataFrame(request.df)
            else:
                # Option 2: Same as download-excel @askme — hydrate from request/saved SQL, else API
                _ctx_askme_ma = SimpleNamespace(
                    my_question=question_to_use,
                    sql=sql_to_use,
                    code=chart_code_to_use,
                    api_endpoint=api_endpoint_to_use,
                    api_params=api_params_to_use,
                    api_response_validator=api_response_validator_to_use,
                    question_id=str(question_id) if question_id else None,
                    df=None,
                )
                prior_input, prior_err = await _askme_resolve_prior_df_for_analysis(
                    service, _ctx_askme_ma, str(primary_agent_id)
                )
                if prior_err:
                    if not skip_user_question_log:
                        await run_in_threadpool(
                            create_user_question_log,
                            user_id=user_id,
                            agent_id=agent_ids if len(agent_ids) > 1 else primary_agent_id,
                            question=question_to_use,
                            sql="N",
                            chart="N",
                            web="N",
                            executed="N",
                            start_time=question_start_time,
                            api_endpoint="",
                            api_params="",
                            api_response_validator="",
                        )
                    return await _return_sk_tb(
                        "askme_analysis",
                        {
                            "success": False,
                            "message": prior_err,
                            "details": {"question": request.question},
                        },
                    )
                if prior_input:
                    _parsed_askme = service._parse_df_input(prior_input)
                    if _parsed_askme and len(_parsed_askme) > 0:
                        analysis_df = pd.DataFrame(_parsed_askme)
                if analysis_df is None or (
                    hasattr(analysis_df, "empty") and analysis_df.empty
                ):
                    if not skip_user_question_log:
                        await run_in_threadpool(
                            create_user_question_log,
                            user_id=user_id,
                            agent_id=agent_ids if len(agent_ids) > 1 else primary_agent_id,
                            question=question_to_use,
                            sql="N",
                            chart="N",
                            web="N",
                            executed="N",
                            start_time=question_start_time,
                            api_endpoint="",
                            api_params="",
                            api_response_validator="",
                        )
                    return await _return_sk_tb(
                        "askme_analysis",
                        {
                            "success": False,
                            "message": "No previous data found. Please provide a dataframe in the 'df' field or run a query first to get data for analysis.",
                            "details": {"question": request.question},
                        },
                    )
            
            # Generate analysis based on user request
            try:
                analysis_result = await run_in_threadpool(service.ask_data_analysis_cached, question=query, df=analysis_df, use_qwen=settings.default_use_qwen_for_summary_and_web())
                
                is_chart_available = "N"
                result_df = None
                chart_fig = None
                analysis_text = None
                # Extract text analysis if available
                if analysis_result.get('has_text') and analysis_result.get('text'):
                    analysis_text = analysis_result['text']
                
                # Get result dataframe if available
                if analysis_result.get('dataframe') is not None:
                    result_df = analysis_result['dataframe']
                    # Convert to records format
                    chart_df = analysis_result.get('dataframe')
                    chart_columns = list(chart_df.columns) if hasattr(chart_df, 'columns') else []
                    if chart_df is not None and len(chart_df) > 0 and len(chart_df) < 101:
                        json_code = await run_in_threadpool(
                            service.generate_json_code_cached,
                            question=question_to_process,
                            columns=chart_columns,
                            model="gpt-4.1",
                            use_qwen=settings.default_use_qwen_for_summary_and_web(),
                        )
                        
                        if json_code and json_code != "N":
                            try:
                                # Validate syntax before execution
                                compile(json_code, '<string>', 'exec')
                                
                                df_for_code = pd.DataFrame(chart_df)
                                exec_globals, local_vars = _chart_code_exec_context(
                                    df_for_code
                                )
                                exec(json_code, exec_globals, local_vars)
                                
                                # chart_json dict, or chart_fig from go.Figure() (common from Phi/small LLMs)
                                chart_json = _chart_json_from_exec_locals(
                                    local_vars
                                ) or {}
                                
                                # Ensure it has Plotly structure (data and layout)
                                if "data" not in chart_json or "layout" not in chart_json:
                                    chart_json = {}
                                else:
                                    # Ensure layout has title and axes
                                    if "title" not in chart_json.get("layout", {}):
                                        chart_json.setdefault("layout", {})["title"] = query[:50] if len(query) > 50 else query
                                    if "xaxis" not in chart_json.get("layout", {}):
                                        chart_json.setdefault("layout", {})["xaxis"] = {"title": chart_columns[0] if len(chart_columns) > 0 else ""}
                                    if "yaxis" not in chart_json.get("layout", {}) and len(chart_columns) > 1:
                                        chart_json.setdefault("layout", {})["yaxis"] = {"title": chart_columns[1] if len(chart_columns) > 1 else ""}
                                
                                if chart_json is not None:
                                    chart_fig = chart_json
                                    is_chart_available = json_code
                            except SyntaxError as syntax_err:
                                print(f"Chart code has syntax error: {syntax_err}")
                                chart_json = {}
                                chart_fig = chart_json
                                is_chart_available = "N"
                            except Exception as exec_err:
                                print(f"Error executing chart code: {exec_err}")
                                chart_json = {}
                                chart_fig = chart_json
                                is_chart_available = "N"
                        else:
                            # json_code is None or "N", create default chart
                            chart_json = {}
                            chart_fig = chart_json
                            is_chart_available = "N"
                
                    if hasattr(result_df, 'to_dict'):
                        df_records = result_df.to_dict('records')
                        # Clean NaN values to make JSON serializable
                        df_records = clean_nan_values(df_records)
                        columns = list(result_df.columns)
                    else:
                        df_records = []
                        columns = []
                else:
                    df_records = []
                    columns = []
                
                # Build response
                response_details = {
                    "question": request.question,
                    "analysis_query": query,
                    "sql": "N",
                    "api_endpoint": "",
                    "api_params": "",
                    "api_response_validator": ""
                }
                
                # Add text analysis (raw and HTML for frontend)
                if analysis_text:
                    response_details["text_analysis"] = analysis_text
                    response_details["text_analysis_html"] = format_llm_response_as_html(
                        normalize_analysis_text_for_html(analysis_text)
                    )
                
                # Add chart if available
                if chart_fig is not None:
                    # chart_fig is already chart_json (dict) from the execution above
                    try:
                        if isinstance(chart_fig, dict):
                            # Clean NaN values to make JSON serializable
                            chart_fig = clean_nan_values(chart_fig)
                            response_details["chart"] = chart_fig
                            response_details["chart_code"] = is_chart_available
                        else:
                            # Fallback: if it's still a plotly figure, convert to dict
                            chart_json = chart_fig.to_dict()
                            # Clean NaN values to make JSON serializable
                            chart_json = clean_nan_values(chart_json)
                            response_details["chart"] = chart_json
                            response_details["chart_code"] = is_chart_available
                    except Exception as e:
                        print(f"Error processing chart: {e}")
                
                # Add dataframe if available
                if result_df is not None and len(df_records) > 0:
                    formatted_columns = [col.replace('_', ' ').capitalize() for col in columns]
                    response_details["df"] = {
                        "agent_id": str(primary_agent_id),
                        "headers": formatted_columns,
                        "data": df_records,
                        "chart_code": is_chart_available,
                    }
                
                # Check if any analysis was generated (flags and/or actual text / dataframe)
                _txt_ma = (analysis_text or analysis_result.get("text") or "").strip()
                _has_any_ma = (
                    analysis_result.get("has_text")
                    or analysis_result.get("has_dataframe")
                    or analysis_result.get("has_chart")
                    or bool(_txt_ma)
                    or (analysis_result.get("dataframe") is not None)
                )
                if not _has_any_ma:
                    if not skip_user_question_log:
                        await run_in_threadpool(
                            create_user_question_log,
                            user_id=user_id,
                            agent_id=agent_ids if len(agent_ids) > 1 else primary_agent_id,
                            question=question_to_use,
                            sql="N",
                            chart="N",
                            web="N",
                            executed="N",
                            start_time=question_start_time,
                            api_endpoint="",
                            api_params="",
                            api_response_validator=""
                        )
                    return await _return_sk_tb(
                        "askme_analysis",
                        {
                            "success": False,
                            "message": "Could not generate analysis. Please try rephrasing your request.",
                            "details": response_details,
                        },
                    )
                
                # Log successful execution
                if not skip_user_question_log:
                    await run_in_threadpool(
                        create_user_question_log,
                        user_id=user_id,
                        agent_id=agent_ids if len(agent_ids) > 1 else primary_agent_id,
                        question=question_to_use,
                        sql="N",
                        chart=is_chart_available,
                        web="N",
                        executed="Y",
                        start_time=question_start_time,
                        api_endpoint="",
                        api_params="",
                        api_response_validator=""
                    )
                
                try:
                    # Force to string for safe slicing
                    analysis_str = str(analysis_text) if analysis_text else ""
                    summary_text = analysis_str[:1000] if analysis_str else "Data analysis completed."
                    talkback_df = None
                    if result_df is not None and hasattr(result_df, "empty") and not result_df.empty:
                        talkback_df = result_df
                    elif analysis_df is not None and hasattr(analysis_df, "empty") and not analysis_df.empty:
                        talkback_df = analysis_df
                    talkback = await run_in_threadpool(
                        service.generate_continuation_talkback_cached,
                        question=question_to_process,
                        answer_context=summary_text,
                        columns=columns if 'columns' in locals() else [],
                        n_items=3,
                        df=talkback_df,
                        use_qwen=settings.default_use_qwen_for_summary_and_web(),
                    )
                    response_details["talkback"] = talkback
                except Exception as e:
                    print(f"Error generating talkback for @askme: {e}")

                return await _return_sk_tb(
                    "askme_analysis",
                    {
                        "success": True,
                        "message": "Analysis completed successfully",
                        "details": response_details,
                        "question": question_to_process,
                    },
                )
                
            except Exception as e:
                import traceback
                error_msg = f"Error performing analysis: {str(e)}"
                print(f"Error in @askme analysis: {traceback.format_exc()}")
                
                if not skip_user_question_log:
                    await run_in_threadpool(
                        create_user_question_log,
                        user_id=user_id,
                        agent_id=agent_ids if len(agent_ids) > 1 else primary_agent_id,
                        question=question_to_use,
                        sql="N",
                        chart="N",
                        web="N",
                        executed="N",
                        start_time=question_start_time,
                        api_endpoint="",
                        api_params="",
                        api_response_validator=""
                    )
                
                return await _return_sk_tb(
                    "askme_analysis",
                    {
                        "success": False,
                        "message": error_msg,
                        "details": {"question": request.question, "error": str(e)},
                    },
                )
        elif question_to_use and "@web" in question_to_use and settings.IS_WEB_ACTIVE:
            query = question_to_use.replace("@web", "").strip()

            if not query:
                if not skip_user_question_log:
                    await run_in_threadpool(
                        create_user_question_log,
                        user_id=user_id,
                        agent_id=agent_ids if len(agent_ids) > 1 else primary_agent_id,
                        question=question_to_use,
                        sql="N",
                        chart="N",
                        web="N",
                        executed="N",
                        start_time=question_start_time,
                        api_endpoint="",
                        api_params="",
                        api_response_validator=""
                    )
                return await _return_sk_tb(
                    "web_search",
                    {
                        "success": False,
                        "message": "Please provide your search request after @web",
                        "details": {"question": request.question},
                    },
                )
            
            # Get dataframe - either from request or need to run a query first
            analysis_df = None
            
            # Option 1: DataFrame provided in request
            if request.df:
                # Parse DF input
                request.df = service._parse_df_input(request.df)
                
            if request.df and len(request.df) > 0:
                analysis_df = pd.DataFrame(request.df)
            else:
                # Standalone @web: Tavily + optional compare — no prior table required
                analysis_df = pd.DataFrame()

            # Generate analysis based on user request
            try:
                analysis_result = await run_in_threadpool(
                    service.search_and_format_response,
                    user_query=query,
                    last_response=analysis_df, use_qwen=settings.default_use_qwen_for_summary_and_web()
                )
                
                # Handle both string and dict responses
                if isinstance(analysis_result, str):
                    # Wrap string response in expected format
                    analysis_text = analysis_result
                    has_text = True
                elif isinstance(analysis_result, dict):
                    # Handle dict response (legacy format)
                    analysis_text = analysis_result.get('text', analysis_result.get('formatted_response', ''))
                    has_text = analysis_result.get('has_text', bool(analysis_text))
                else:
                    analysis_text = str(analysis_result)
                    has_text = True
                
                is_chart_available = "N"
                result_df = None
                chart_fig = None
                
                # Build response
                response_details = {
                    "question": request.question,
                    "analysis_query": query,
                    "sql": "N",
                    "api_endpoint": "",
                    "api_params": "",
                    "api_response_validator": ""
                }
                
                # Add text analysis (raw and HTML for frontend)
                if has_text and analysis_text:
                    response_details["text_analysis"] = analysis_text
                    response_details["text_analysis_html"] = format_llm_response_as_html(
                        normalize_analysis_text_for_html(analysis_text)
                    )
                
                # Check if any analysis was generated
                if not has_text or not analysis_text:
                    if not skip_user_question_log:
                        await run_in_threadpool(
                            create_user_question_log,
                            user_id=user_id,
                            agent_id=request.agent_id,
                            question=question_to_use,
                            sql="N",
                            chart="N",
                            web="Y",
                            executed="N",
                            start_time=question_start_time,
                            api_endpoint="",
                            api_params="",
                            api_response_validator=""
                        )
                    return await _return_sk_tb(
                        "web_search",
                        {
                            "success": False,
                            "message": "Could not generate web search results. Please try rephrasing your request.",
                            "details": response_details,
                        },
                    )
                
                # Log successful execution
                if not skip_user_question_log:
                    await run_in_threadpool(
                        create_user_question_log,
                        user_id=user_id,
                        agent_id=agent_ids if len(agent_ids) > 1 else primary_agent_id,
                        question=question_to_use,
                        sql="N",
                        chart=is_chart_available,
                        web="Y",
                        executed="Y",
                        start_time=question_start_time,
                        api_endpoint="",
                        api_params="",
                        api_response_validator=""
                    )
                
                try:
                    # Force to string for safe slicing
                    analysis_str = str(analysis_text) if analysis_text else ""
                    summary_text = analysis_str[:1000] if analysis_str else "Web search results found."
                    tb_cols_web_ma = (
                        list(analysis_df.columns)
                        if analysis_df is not None and hasattr(analysis_df, "columns")
                        else []
                    )
                    talkback = await run_in_threadpool(
                        service.generate_continuation_talkback_cached,
                        question=question_to_use,
                        answer_context=summary_text,
                        columns=tb_cols_web_ma,
                        n_items=3,
                        df=analysis_df if analysis_df is not None and len(analysis_df) > 0 else None,
                        use_qwen=settings.default_use_qwen_for_summary_and_web(),
                    )
                    response_details["talkback"] = talkback
                except Exception as e:
                    print(f"Error generating talkback for @web: {e}")

                if analysis_df is not None and len(analysis_df) > 0:
                    df_records_web = clean_nan_values(analysis_df.to_dict("records"))
                    cols_web = list(analysis_df.columns)
                    formatted_cols_web = [
                        str(c).replace("_", " ").capitalize() for c in cols_web
                    ]
                    response_details["df"] = {
                        "agent_id": str(primary_agent_id),
                        "headers": formatted_cols_web,
                        "data": df_records_web,
                        "chart_code": is_chart_available,
                    }

                return await _return_sk_tb(
                    "web_search",
                    {
                        "success": True,
                        "message": "Web search completed successfully",
                        "details": response_details,
                        "question": question_to_process,
                    },
                )
                
            except Exception as e:
                import traceback
                error_msg = f"Error performing analysis: {str(e)}"
                print(f"Error in @web analysis: {traceback.format_exc()}")
                
                if not skip_user_question_log:
                    await run_in_threadpool(
                        create_user_question_log,
                        user_id=user_id,
                        agent_id=agent_ids if len(agent_ids) > 1 else primary_agent_id,
                        question=question_to_use,
                        sql="N",
                        chart="N",
                        web="N",
                        executed="N",
                        start_time=question_start_time,
                        api_endpoint="",
                        api_params="",
                        api_response_validator=""
                    )
                
                return await _return_sk_tb(
                    "web_search",
                    {
                        "success": False,
                        "message": error_msg,
                        "details": {"question": request.question, "error": str(e)},
                    },
                )

        
        df_records = []
        columns = []
        chart_code = "N"
        
        # Handle API datasource
        if datasource_to_use and datasource_to_use.upper() == "API":
            # Check if API parameters are already provided (reuse saved API parameters)
            if api_endpoint_to_use and api_params_to_use and api_response_validator_to_use:
                print("API parameters present - Reusing saved API call")
                
                # Reuse the provided API parameters directly
                api_endpoint = api_endpoint_to_use
                api_params = api_params_to_use
                api_response_validator = api_response_validator_to_use
                
                # Parse the saved params (they're stored as strings)
                import ast
                if isinstance(api_params, str) and api_params.strip():
                    try:
                        extracted_params = ast.literal_eval(api_params)
                    except:
                        extracted_params = {}
                else:
                    extracted_params = {}
                
                # Get api_info to determine if SAP (only when reusing parameters)
                apis_list = await run_in_threadpool(service.get_all_apis, request.agent_id)
                api_info = {}
                for data in apis_list:
                    if data.get("endpoint") == api_endpoint:
                        api_info = data
                        break
                
                is_sap = api_info.get("extra1", "N") if api_info else "N"
                api_name = api_info.get("name", "") if api_info else ""
                print(f"Is sap: {is_sap}")
            else:
                # Generate new API call (only when parameters not provided)
                print("Generating new API call")
                apis_list = await run_in_threadpool(service.get_all_apis, request.agent_id)
                
                if not apis_list:
                    if not skip_user_question_log:
                        await run_in_threadpool(
                            create_user_question_log,
                            user_id=user_id,
                            agent_id=request.agent_id,
                            question=question_to_use,
                            sql="N",
                            chart="N",
                            web="N",
                            executed="N",
                            start_time=question_start_time,
                            api_endpoint="",
                            api_params="",
                            api_response_validator=""
                        )
                    return await _return_sk_tb(
                        "api_flow",
                        {
                            "success": False,
                            "message": "No APIs configured for this agent",
                            "details": {"question": request.question},
                        },
                    )
                
                # Select appropriate API for the query
                selected_apis = await run_in_threadpool(
                    service.select_appropriate_api,
                    user_query=question_to_process,
                    available_apis=apis_list,
                    agent_id=request.agent_id
                )
                
                print(f"DEBUG: Selected APIs: {[a.get('name') for a in selected_apis]}")
                
                if not selected_apis:
                    if not skip_user_question_log:
                        await run_in_threadpool(
                            create_user_question_log,
                            user_id=user_id,
                            agent_id=agent_ids if len(agent_ids) > 1 else primary_agent_id,
                            question=question_to_use,
                            sql="N",
                            chart="N",
                            web="N",
                            executed="N",
                            start_time=question_start_time,
                            api_endpoint="",
                            api_params="",
                            api_response_validator=""
                        )
                    return await _return_sk_tb(
                        "api_flow",
                        {
                            "success": False,
                            "message": "No matching API found for this query",
                            "details": {"question": request.question},
                        },
                    )
                
                # Use first API from the list (for now, can be extended later)
                api_info = selected_apis[0]
                print(f"Selected API: {api_info.get('name')}")
                
                # If multiple APIs selected, log them
                if len(selected_apis) > 1:
                    api_names = ", ".join([api.get('name', '') for api in selected_apis])
                    print(f"Multiple APIs selected: {api_names}. Using first one: {api_info.get('name')}")
                
                is_sap = api_info.get("extra1", "N")
                api_name = api_info.get("name", "") if api_info else ""
                print(f"Is sap: {is_sap}")
            
            # Unified API call handling (for both reused and generated parameters)
            try:
                if is_sap == "Y":
                    # SAP Flow
                    if api_endpoint_to_use and api_params_to_use and api_response_validator_to_use:
                        # Reusing saved parameters
                        sap_result = await run_in_threadpool(
                            service.process_sap_query,
                            service_root=api_endpoint,
                            entity_set=api_info.get("base_params"),
                            user_query=question_to_process,
                            response_schema=api_info.get("response_schema"),
                            transformation_code=api_response_validator,
                            username="S4h_aa",
                            password="Welcome1",
                            verify_ssl=False,
                            api_query_params=extracted_params
                        )
                        df = sap_result.get("df")
                    else:
                        # Non-SAP Flow - Reusing parameters
                        data, data2, df, data3 = service.call_api_and_transform(
                            extracted_params,
                            api_endpoint,
                            "",  # Use original schema
                            api_response_validator,
                            question_to_process
                        )
                else:
                    # Generating new API call (non-SAP / new params)
                    extracted_params = await run_in_threadpool(service.extract_api_params, question_to_process, api_info)
                    data, data2, df, data3 = await run_in_threadpool(
                        service.call_api_and_transform,
                        extracted_params,
                        api_info.get("endpoint"),
                        api_info.get("response_schema"),
                        "",
                        question_to_process
                    )
                    api_endpoint = api_info.get("endpoint", "")
                    api_params = str(extracted_params) if extracted_params else ""
                    api_response_validator = data3 if data3 else ""
                
                generated_sql = ""
                
                # Convert DataFrame to records for API datasource
                df_records = []
                columns = []
                if df is not None:
                    if hasattr(df, 'to_dict') and hasattr(df, 'columns'):
                        if not df.empty:
                            df_records = df.to_dict('records')
                            # Clean NaN values to make JSON serializable
                            df_records = clean_nan_values(df_records)
                            columns = list(df.columns)
                    elif isinstance(df, list):
                        df_records = df
                        # Clean NaN values to make JSON serializable
                        df_records = clean_nan_values(df_records)
                        if df and isinstance(df[0], dict):
                            columns = list(df[0].keys())
                
                # Check if API call returned no data
                if df is None:
                    # Log error for no results from API
                    if not skip_user_question_log:
                        await run_in_threadpool(
                            create_user_question_log,
                            user_id=user_id,
                            agent_id=agent_ids if len(agent_ids) > 1 else primary_agent_id,
                            question=question_to_use,
                            sql="N",
                            chart="N",
                            web="N",
                            executed="N",
                            start_time=question_start_time,
                            api_endpoint=api_endpoint if api_endpoint else "",
                            api_params=api_params if api_params else "",
                            api_response_validator=api_response_validator if api_response_validator else ""
                        )
                    return await _return_sk_tb(
                        "api_flow",
                        {
                            "success": False,
                            "message": "No results returned from API",
                            "details": {"question": request.question},
                        },
                    )
            except Exception as e:
                # Log error for API call failure
                if not skip_user_question_log:
                    await run_in_threadpool(
                        create_user_question_log,
                        user_id=user_id,
                        agent_id=agent_ids if len(agent_ids) > 1 else primary_agent_id,
                        question=question_to_use,
                        sql="N",
                        chart="N",
                        web="N",
                        executed="N",
                        start_time=question_start_time,
                        api_endpoint=api_endpoint if api_endpoint else (api_info.get("endpoint", "") if api_info else ""),
                        api_params=api_params if api_params else "",
                        api_response_validator=api_response_validator if api_response_validator else ""
                    )
                return await _return_sk_tb(
                    "api_flow",
                    {
                        "success": False,
                        "message": f"Error calling API: {str(e)}",
                        "details": {"question": request.question, "error": str(e)},
                    },
                )
        
        # Handle Database datasource (original flow)
        else:
            multi_agent_expected_agents = None
            # Ensure Oracle connection for this agent
            service._ensure_sql()

            # Check if SQL is already provided (reuse saved SQL)
            if sql_to_use:
                print("SQL Present - Reusing saved SQL")
                print("sql_to_use", sql_to_use)
                generated_sql = sql_to_use
            else:
                # Generate SQL
                print("Creating SQL.....")
                generated_sql = await run_in_threadpool(service.generate_sql_cached, question=question_to_process)
            
            if not generated_sql:
                if not skip_user_question_log:
                    await run_in_threadpool(
                        create_user_question_log,
                        user_id=user_id,
                        agent_id=agent_ids if len(agent_ids) > 1 else primary_agent_id,
                        question=question_to_use,
                        sql="N",
                        chart="N",
                        web="N",
                        executed="N",
                        start_time=question_start_time,
                        api_endpoint="",
                        api_params="",
                        api_response_validator=""
                    )
                return await _return_sk_tb(
                    "sql_generation",
                    {
                        "success": False,
                        "message": "Failed to generate SQL for the question",
                        "details": {"question": request.question},
                    },
                )
            
            # Check if this is a multi-agent response: queries/agent_results/new list format from saved questions.
            is_multi_agent_json = False
            parsed = None
            sql_for_execution = generated_sql
            parsed = _merge_info_from_multi_agent_sql_blob(generated_sql)
            if parsed and parsed.get("queries"):
                is_multi_agent_json = True
                sql_for_execution = json.dumps(parsed)

            # For DB logging, serialize dict (e.g. agent_results) to string
            sql_for_log = json.dumps(generated_sql) if isinstance(generated_sql, dict) else (generated_sql or "")
            print("sql_for_log", sql_for_log)
            # Validate SQL (skip for multi-agent)
            if not is_multi_agent_json and not service.is_sql_valid_cached(sql=generated_sql):
                # Log error for invalid SQL
                if not skip_user_question_log:
                    await run_in_threadpool(
                        create_user_question_log,
                        user_id=user_id,
                        agent_id=agent_ids if len(agent_ids) > 1 else primary_agent_id,
                        question=question_to_use,
                        sql=sql_for_log,
                        chart="N",
                        web="N",
                        executed="N",
                        start_time=question_start_time,
                        api_endpoint="",
                        api_params="",
                        api_response_validator=""
                    )
                _det_ma_inv: Dict[str, Any] = {
                    "question": request.question,
                    "sql": _details_sql_envelope_from_db_generated(
                        generated_sql,
                        question_to_process,
                        str(primary_agent_id),
                        training_driven_pipeline=False,
                    ),
                }
                _det_ma_inv["talkback"] = await _talkback_for_db_sql_error(
                    service,
                    sql_for_log,
                    no_rows=False,
                    question=_pick_question_for_error_talkback(
                        question_to_process, question_to_use, getattr(request, "question", None)
                    ),
                )
                return await _return_sk_tb(
                    "sql_validation",
                    {
                        "success": False,
                        "message": (
                            "We could not use that as a valid query. Rephrase your question, or use one of "
                            "the suggested questions."
                        ),
                        "details": _det_ma_inv,
                    },
                )

            # When re-running saved multi-agent SQL (e.g. by question_id), use multi-agent service
            if is_multi_agent_json and parsed and parsed.get("queries"):
                query_keys = list(parsed["queries"].keys())
                extracted_agent_ids = sorted({
                    (k[5:] if str(k).startswith("agent") else str(k)) for k in query_keys
                })
                multi_agent_expected_agents = extracted_agent_ids
                if len(extracted_agent_ids) > 1:
                    multi_agent_service = get_askme_service(agent_id=extracted_agent_ids)
                    if multi_agent_service:
                        service = multi_agent_service

            # Run SQL (use sql_for_execution for multi-agent; pass merge_info per-request to avoid race)
            try:
                merge_info_arg = parsed if is_multi_agent_json else None
                df = await run_in_threadpool(
                    service.run_sql_cached,
                    sql_for_execution,
                    merge_info=merge_info_arg,
                    question=question_to_process,
                )
                
                # Check if SQL returned no data
                if df is None:
                    if not skip_user_question_log:
                        await run_in_threadpool(
                            create_user_question_log,
                            user_id=user_id,
                            agent_id=agent_ids if len(agent_ids) > 1 else primary_agent_id,
                            question=question_to_use,
                            sql=sql_for_log,
                            chart="N",
                            web="N",
                            executed="N",
                            start_time=question_start_time,
                            api_endpoint="",
                            api_params="",
                            api_response_validator=""
                        )
                    _det_ma_norow: Dict[str, Any] = {
                        "question": request.question,
                        "sql": _details_sql_envelope_from_db_generated(
                            generated_sql,
                            question_to_process,
                            str(primary_agent_id),
                            training_driven_pipeline=False,
                        ),
                    }
                    _det_ma_norow["talkback"] = await _talkback_for_db_sql_error(
                        service,
                        sql_for_log,
                        no_rows=True,
                        question=_pick_question_for_error_talkback(
                            question_to_process, question_to_use, getattr(request, "question", None)
                        ),
                    )
                    return await _return_sk_tb(
                        "sql_run",
                        {
                            "success": False,
                            "message": "No results returned for that query. Try rephrasing your question.",
                            "details": _det_ma_norow,
                        },
                    )
            except Exception as e:
                if isinstance(e, QueryTimeoutError):
                    if not skip_user_question_log:
                        await run_in_threadpool(
                            create_user_question_log,
                            user_id=user_id,
                            agent_id=agent_ids if len(agent_ids) > 1 else primary_agent_id,
                            question=question_to_use,
                            sql=sql_for_log,
                            chart="N",
                            web="N",
                            executed="N",
                            start_time=question_start_time,
                            api_endpoint="",
                            api_params="",
                            api_response_validator=""
                        )
                    return await _return_sk_tb(
                        "sql_run",
                        {
                            "success": False,
                            "message": "The query took too long and was stopped. Try a more specific question.",
                            "details": {
                                "question": request.question,
                                "error": str(e),
                            },
                        },
                    )
                if not skip_user_question_log:
                    await run_in_threadpool(
                        create_user_question_log,
                        user_id=user_id,
                        agent_id=agent_ids if len(agent_ids) > 1 else primary_agent_id,
                        question=question_to_use,
                        sql=sql_for_log,
                        chart="N",
                        web="N",
                        executed="N",
                        start_time=question_start_time,
                        api_endpoint="",
                        api_params="",
                        api_response_validator=""
                    )
                return await _return_sk_tb(
                    "sql_run",
                    {
                        "success": False,
                        "message": f"Error running SQL: {str(e)}",
                        "details": {
                            "sql": _details_sql_envelope_from_db_generated(
                                generated_sql,
                                question_to_process,
                                str(primary_agent_id),
                                training_driven_pipeline=False,
                            ),
                            "error": str(e),
                        },
                    },
                )

            # Saved / explicit multi-agent SQL: run_sql_cached may return a dict
            # (independent per-agent frames). Merge into one table (same as training
            # when merge_results uses the heterogeneous path) so saved chart_code
            # applies to a single df with all metric columns.
            if (
                is_multi_agent_json
                and parsed
                and isinstance(df, dict)
                and len(df) > 1
            ):
                try:
                    _qmap = parsed.get("queries") or {}
                    _order = list(
                        parsed.get("execution_order") or list(_qmap.keys())
                    )
                    _dfs: List[pd.DataFrame] = []
                    _aids: List[str] = []
                    for _k in _order:
                        _aid = (
                            str(_k)[5:]
                            if str(_k).startswith("agent")
                            else str(_k)
                        )
                        _sub = df.get(_aid)
                        if (
                            _sub is not None
                            and isinstance(_sub, pd.DataFrame)
                            and not _sub.empty
                        ):
                            _dfs.append(_sub)
                            _aids.append(_aid)
                    if len(_dfs) > 1:
                        try:
                            _mr = int(
                                getattr(
                                    service, "_max_row_count", 5000
                                )
                                or 5000
                            )
                        except (TypeError, ValueError):
                            _mr = 5000
                        _merged, _mmeta = merge_results(
                            _dfs, max_rows=_mr, frame_agent_ids=_aids
                        )
                        if (
                            _mmeta.get("merge_strategy") != "separate_frames"
                            and _merged is not None
                            and not getattr(_merged, "empty", True)
                        ):
                            df = _merged
                except Exception as _coalesce_exc:
                    print(
                        f"[multi-agent-ask] saved multi-agent df coalesce: {_coalesce_exc}"
                    )
        
        
        # Convert DataFrame or List to list of dicts (only for database datasource)
        # For API datasource, this was already done immediately after the API call
        df_multi_agent_agents = None  # When set, response has multiple independent DataFrames
        first_nonempty_agent_id: Optional[str] = None
        if datasource_to_use and datasource_to_use.upper() != "API":
            df_records = []
            columns = []
            
            # Handle multi-agent independent: dict of agent_id -> DataFrame
            if isinstance(df, dict):
                agents_list = []
                first_records, first_columns = [], []
                for aid, agent_df in df.items():
                    if agent_df is None or not hasattr(agent_df, 'to_dict'):
                        continue
                    rec = agent_df.to_dict('records') if not (hasattr(agent_df, 'empty') and agent_df.empty) else []
                    rec = clean_nan_values(rec)
                    cols = list(agent_df.columns) if hasattr(agent_df, 'columns') else []
                    agents_list.append({
                        "agent_id": str(aid),
                        "headers": [c.replace('_', ' ').capitalize() for c in cols],
                        "data": rec
                    })
                    if not first_records and rec:
                        first_records, first_columns = rec, cols
                        first_nonempty_agent_id = str(aid)
                df_multi_agent_agents = agents_list
                df_records = first_records
                columns = first_columns
            # Handle DataFrame (standard case)
            elif hasattr(df, 'to_dict') and hasattr(df, 'columns'):
                if not df.empty:
                    df_records = df.to_dict('records')
                    # Clean NaN values to make JSON serializable
                    df_records = clean_nan_values(df_records)
                    columns = list(df.columns)
            
            # Handle List (fallback or alternative return type)
            elif isinstance(df, list):
                df_records = df
                # Clean NaN values to make JSON serializable
                df_records = clean_nan_values(df_records)
                if df and isinstance(df[0], dict):
                    columns = list(df[0].keys())



        formatted_columns = [col.replace('_', ' ').capitalize() for col in columns]

        _agent_id_for_df_ma = (
            str(agent_ids[0])
            if len(agent_ids) == 1
            else (json.dumps(agent_ids) if agent_ids else "")
        )

        if df_multi_agent_agents is not None:
            response_details_df = _df_unmerged_multi_agent(df_multi_agent_agents)
            response_details_df["agent_id"] = _agent_id_for_df_ma
        else:
            response_details_df = {
                "agent_id": _agent_id_for_df_ma,
                "headers": formatted_columns,
                "data": df_records,
                "api_name": api_name,
            }
            if len(agent_ids) > 1:
                response_details_df["merged"] = True

        if datasource_to_use and datasource_to_use.upper() == "API":
            _api_aid = str(primary_agent_id)
            details_sql_ma = _details_sql_envelope_api(
                sub_tasks=[
                    {
                        "sub_question": question_to_process,
                        "agent_id": _api_aid,
                        "datasource": "API",
                    }
                ],
                api_traces=[
                    {
                        "agent_id": _api_aid,
                        "sub_question": question_to_process,
                        "result": _api_result_dict_for_envelope(
                            api_endpoint, api_params, api_response_validator
                        ),
                    }
                ],
                training_driven_pipeline=False,
            )
        else:
            details_sql_ma = _details_sql_envelope_from_db_generated(
                generated_sql,
                question_to_process,
                str(primary_agent_id),
                training_driven_pipeline=False,
            )

        response_details = {
            "sql": details_sql_ma,
            "df": response_details_df,
            "question": question_to_process,
            "original_question": question_to_use if request.inContext else None,
            "api_endpoint": api_endpoint if api_endpoint else "",
            "api_params": api_params if api_params else "",
            "api_response_validator": api_response_validator if api_response_validator else "",
        }
        if multi_agent_expected_agents is not None:
            response_details["multi_agent_expected_agents"] = multi_agent_expected_agents

        # Generate chart when the primary tabular slice has <100 rows, or any per-agent
        # table is small (so we still build top-level chart / per-agent codes correctly).
        _any_agent_under_101 = False
        if df_multi_agent_agents:
            for _a in df_multi_agent_agents:
                _d = _a.get("data") or []
                if 0 < len(_d) < 101:
                    _any_agent_under_101 = True
                    break
        should_generate_chart = bool(
            df_records and len(columns) > 0 and len(df_records) < 101
        ) or _any_agent_under_101
        
        if should_generate_chart:
            # Generate JSON chart structure
            try:
                import json as json_module
                # Check if chart_json is provided as a string (Python code or JSON)
                if chart_code_to_use and chart_code_to_use.strip() and chart_code_to_use != "N":
                    print("Reusing provided chart JSON code")
                    df_for_code = pd.DataFrame(df_records)
                    try:
                        json_code = chart_code_to_use
                        # Execute chart code in a safe namespace.
                        exec_globals, local_vars = _chart_code_exec_context(
                            df_for_code
                        )
                        exec(json_code, exec_globals, local_vars)
                        chart_json = _chart_json_from_exec_locals(local_vars)
                        print("chart_json", chart_json)
                    except Exception as reuse_err:
                        # Reused code may reference stale columns (e.g. KeyError on renamed fields).
                        # Fall back to generating fresh chart code for the current dataframe schema.
                        print(f"Reused chart code failed, regenerating chart code: {reuse_err}")
                        chart_json = None
                        json_code = "N"
                else:
                    chart_json = None
                    json_code = "N"

                if (
                    chart_json is None
                    and df_records
                    and len(columns) > 0
                    and len(df_records) < 101
                ):
                    # Generate new Python code if chart_json is not provided
                    # Only generate if we have dataframe data
                    json_code = service.generate_json_code_cached(
                        question=question_to_process,
                        columns=columns,
                        model="gpt-4.1"
                    )

                    df_for_code = pd.DataFrame(df_records)
                    # Execute the generated code in a safe namespace
                    exec_globals, local_vars = _chart_code_exec_context(
                        df_for_code
                    )
                    exec(json_code, exec_globals, local_vars)

                    # Extract the chart_json variable created by the code
                    chart_json = _chart_json_from_exec_locals(local_vars)

                # If primary table is not chartable (empty, >=101 rows) but a per-agent
                # table is <101 rows, chart that slice so the top-level chart is not
                # stuck at chart_code "N".
                if (
                    chart_json is None
                    and df_multi_agent_agents
                    and _any_agent_under_101
                ):
                    for _ag in df_multi_agent_agents:
                        _d = _ag.get("data") or []
                        if not (0 < len(_d) < 101) or not isinstance(
                            _d[0] if _d else None, dict
                        ):
                            continue
                        _cols = list(_d[0].keys())
                        if not _cols:
                            continue
                        try:
                            _json_code = service.generate_json_code_cached(
                                question=question_to_process,
                                columns=_cols,
                                model="gpt-4.1",
                            )
                            if not _json_code or _json_code == "N":
                                continue
                            _df = pd.DataFrame(_d)
                            _g, _lv = _chart_code_exec_context(_df)
                            exec(_json_code, _g, _lv)
                            _cj = _chart_json_from_exec_locals(_lv)
                            if _cj is not None:
                                chart_json = _cj
                                json_code = _json_code
                                break
                        except Exception as _slice_exc:
                            print(
                                f"[multi-agent-ask] chart on small per-agent slice: {_slice_exc}"
                            )
                            continue

                # # Validate and ensure the structure matches Plotly format
                # if chart_json is None or not isinstance(chart_json, dict):
                #     # Fallback: create default Plotly structure if we have data
                #     if df_records and len(columns) > 0 and len(df_records) < 101:
                #         df_for_code = pd.DataFrame(df_records)
                #         chart_json = service._create_default_plotly_json(df_for_code, columns, question_to_process)
                #         json_code = "N"  # Default chart, no code
                #     else:
                #         # No data and no valid chart_json - skip chart
                #         chart_json = None
                #         json_code = "N"
                # else:
                #     # Ensure it has Plotly structure (data and layout)
                #     if "data" not in chart_json or "layout" not in chart_json:
                #         if df_records and len(columns) > 0 and len(df_records) < 101:
                #             df_for_code = pd.DataFrame(df_records)
                #             chart_json = service._create_default_plotly_json(df_for_code, columns, question_to_process)
                #             json_code = "N"
                #         else:
                #             chart_json = None
                #             json_code = "N"
                #     else:
                #         # Ensure layout has title and axes
                #         if "title" not in chart_json.get("layout", {}):
                #             chart_json.setdefault("layout", {})["title"] = question_to_process[:50] if len(question_to_process) > 50 else question_to_process
                #         if "xaxis" not in chart_json.get("layout", {}):
                #             chart_json.setdefault("layout", {})["xaxis"] = {"title": columns[0] if len(columns) > 0 else ""}
                #         if "yaxis" not in chart_json.get("layout", {}) and len(columns) > 1:
                #             chart_json.setdefault("layout", {})["yaxis"] = {"title": columns[1] if len(columns) > 1 else ""}
                
                
                # Add chart to response if we have valid chart_json
                if chart_json is not None:
                    # Clean NaN values to make JSON serializable
                    chart_json = clean_nan_values(chart_json)
                    response_details["chart"] = chart_json
                    response_details["chart_code"] = json_code 
                    chart_code = json_code
                else:
                    chart_code = "N"
                
            except Exception as e:
                # If JSON code generation/execution fails, fall back to default Plotly structure
                print(f"Error generating chart: {e}")
                import traceback
                print(traceback.format_exc())
                if df_records and len(columns) > 0 and len(df_records) < 101:
                    df_for_code = pd.DataFrame(df_records)
                    chart_json = {}
                    # Clean NaN values to make JSON serializable
                    chart_json = clean_nan_values(chart_json)
                    response_details["chart"] = chart_json
                    response_details["chart_error"] = str(e)
                    chart_code = "N"  # Chart generation failed
                else:
                    chart_code = "N"
        
        if isinstance(response_details.get("df"), dict):
            response_details["df"]["chart_code"] = chart_code
        _df_for_agents = response_details.get("df")
        if isinstance(_df_for_agents, dict) and _df_for_agents.get("agents"):
            _populate_per_agent_and_merged_chart_codes(
                _df_for_agents,
                service=service,
                question_default=question_to_process,
                chart_code_to_use=chart_code_to_use,
                sub_tasks=None,
                root_chart_code_for_aid=chart_code
                if should_generate_chart and chart_code != "N"
                else None,
                first_tabular_agent_id=first_nonempty_agent_id,
            )
        elif isinstance(_df_for_agents, list):
            _populate_per_agent_and_merged_chart_codes(
                _df_for_agents,
                service=service,
                question_default=question_to_process,
                chart_code_to_use=chart_code_to_use,
                sub_tasks=None,
                root_chart_code_for_aid=chart_code
                if should_generate_chart and chart_code != "N"
                else None,
                first_tabular_agent_id=first_nonempty_agent_id,
            )
        
        # Generate summary
        # if df_records:
        #     summary = service.generate_summary(
        #         question=question_to_use,
        #         df=df_records,
        #         model="gpt-4.1"
        #     )
        #     response_details["summary"] = summary
        
        # Log successful execution
        dataframe_str = str(columns) if columns else "N"
        _sql_for_log_ma = details_sql_ma if details_sql_ma else (generated_sql if generated_sql else "N")
        log_id = None
        if not skip_user_question_log:
            log_id = await run_in_threadpool(
                create_user_question_log,
                user_id=request.user_id,
                agent_id=agent_ids if len(agent_ids) > 1 else primary_agent_id,
                question=question_to_use,
                sql=_sql_for_log_ma,
                chart=chart_code,
                web="N",
                executed="Y",
                start_time=question_start_time,
                api_endpoint=api_endpoint if api_endpoint else "",
                api_params=api_params if api_params else "",
                api_response_validator=api_response_validator if api_response_validator else "",
                dataframe=dataframe_str
            )
        
        try:
            # Force columns to string for context
            context_str = f"Columns: {str(columns)}" if columns else "No data returned."
            if df_records and len(df_records) > 0 and (settings.default_use_qwen_for_summary_and_web() and _qwen_configured()):
                # Convert records to string for safe slicing
                records_str = str(df_records)
                context_str += f"\nSample data: {records_str[:500]}"

            talkback_tb_df = None
            if hasattr(df, "to_dict") and hasattr(df, "columns") and not getattr(df, "empty", True):
                talkback_tb_df = df
            elif isinstance(df, list) and df and isinstance(df[0], dict):
                try:
                    talkback_tb_df = pd.DataFrame(df)
                except Exception:
                    talkback_tb_df = None
            _sql_talkback_ma = (
                generated_sql
                if generated_sql and str(generated_sql).strip()
                else details_sql_ma
            )
            talkback = await run_in_threadpool(
                service.generate_continuation_talkback_cached,
                question=question_to_use if 'question_to_use' in locals() else question_to_process,
                answer_context=context_str,
                columns=columns,
                n_items=3,
                df=talkback_tb_df,
                sql=_sql_talkback_ma,
                api_endpoint=api_endpoint,
                api_params=api_params,
                api_response_validator=api_response_validator,
                use_qwen=settings.default_use_qwen_for_summary_and_web(),
            )
            response_details["talkback"] = talkback
        except Exception as e:
            print(f"Error generating talkback: {e}")

        _cached_sq_multi = getattr(request, "sql_query_summaries_cached", None)
        if _cached_sq_multi:
            response_details["sql_query_summaries"] = _cached_sq_multi
        else:
            try:
                await run_in_threadpool(
                    service.merge_sql_query_summaries_into_details_cached,
                    response_details,
                    question_to_process,
                )
            except Exception as _sq_sum_ma_exc:
                print(f"sql_query_summaries (multi-agent): {_sq_sum_ma_exc}")

        if not skip_user_question_log and log_id:
            asyncio.create_task(
                run_in_threadpool(
                    update_user_question_log_sql_query_summaries,
                    log_id,
                    response_details.get("sql_query_summaries"),
                )
            )

        if not skip_user_question_log and log_id:
            _totals = get_global_usage_totals()
            _input_tokens = int(_totals.get("input_tokens", 0) or 0)
            _output_tokens = int(_totals.get("output_tokens", 0) or 0)
            _total_tokens = int(_totals.get("total_tokens", (_input_tokens + _output_tokens)) or 0)
            asyncio.create_task(
                run_in_threadpool(
                    update_user_question_log_tokens,
                    log_id,
                    _input_tokens,
                    _output_tokens,
                    _total_tokens,
                )
            )
            
        return await _return_sk_tb(
            "main_success",
            {
                "success": True,
                "message": "Question processed successfully",
                "details": response_details,
                "question": question_to_process,
            },
        )

    except Exception as e:
        # Log exception error
        try:
            if not skip_user_question_log:
                await run_in_threadpool(
                    create_user_question_log,
                    user_id=user_id,
                    agent_id=agent_ids if len(agent_ids) > 1 else primary_agent_id,
                    question=question_to_use,
                    sql=generated_sql if generated_sql else "N",
                    chart="N",
                    web="N",
                    executed="N",
                    start_time=question_start_time,
                    api_endpoint=api_endpoint if api_endpoint else "",
                    api_params=api_params if api_params else "",
                    api_response_validator=api_response_validator if api_response_validator else ""
                )
        except:
            pass  # Don't fail if logging fails
        await _persist_multi_agent_talkback_turn(
            locals().get("service"),
            auth_uid,
            effective_session_key,
            locals().get("agent_ids") or [],
            str(locals().get("question_to_use") or ""),
            False,
            f"Error processing question: {str(e)}",
            "unhandled_exception",
            str(e),
        )
        return _sk({
            "success": False,
            "message": f"Error processing question: {str(e)}",
            "details": {"error": str(e)}
        })
