Comparison_Project/app/routes/dashboard.py

import logging
from flask import Blueprint, render_template, session, redirect, url_for, jsonify, request
from sqlalchemy import func, union_all
from app import db
from app.models.trench_excavation_model import TrenchExcavation
from app.models.tr_ex_client_model import TrenchExcavationClient
from app.models.manhole_excavation_model import ManholeExcavation
from app.models.mh_ex_client_model import ManholeExcavationClient
from app.models.laying_model import Laying
from app.models.laying_client_model import LayingClient
from app.models.subcontractor_model import Subcontractor

dashboard_bp = Blueprint("dashboard", __name__, url_prefix="/dashboard")

# Configure logging for debugging
logger = logging.getLogger(__name__)

# API to get dynamic filters from database
@dashboard_bp.route("/api/filters")
def get_filters():
    try:
        logger.info("=" * 60)
        logger.info("FETCHING RA BILLS - START")
        logger.info("=" * 60)
        
        # 1. Fetch Subcontractors (Linked to Trench entries)
        logger.info("Step 1: Fetching Subcontractors...")
        subcontractors = db.session.query(Subcontractor.subcontractor_name)\
            .join(TrenchExcavation, Subcontractor.id == TrenchExcavation.subcontractor_id)\
            .distinct().all()
        logger.info(f"✓ Subcontractors found: {len(subcontractors)}")
        logger.debug(f"  Subcontractor list: {[s[0] for s in subcontractors if s[0]]}")
        
        # 2. Check total records in TrenchExcavation table
        logger.info("Step 2: Checking TrenchExcavation table...")
        total_records = db.session.query(TrenchExcavation).count()
        logger.info(f"✓ Total TrenchExcavation records: {total_records}")
        
        # 3. Check records with RA_Bill_No
        logger.info("Step 3: Checking records with RA_Bill_No...")
        records_with_ra = db.session.query(TrenchExcavation).filter(TrenchExcavation.RA_Bill_No != None).count()
        logger.info(f"✓ Records with RA_Bill_No (not null): {records_with_ra}")
        
        # 4. Check for empty strings
        records_with_ra_and_value = db.session.query(TrenchExcavation).filter(
            TrenchExcavation.RA_Bill_No != None,
            TrenchExcavation.RA_Bill_No != ""
        ).count()
        logger.info(f"✓ Records with RA_Bill_No (not null & not empty): {records_with_ra_and_value}")
        
        # 5. Raw sample of RA_Bill_No values
        logger.info("Step 4: Sampling RA_Bill_No values from database...")
        sample_bills = db.session.query(TrenchExcavation.RA_Bill_No).limit(10).all()
        logger.debug(f"  Sample RA_Bill_No values (Subcontractor): {[str(r[0]) for r in sample_bills]}")
        
        sample_bills_client = db.session.query(TrenchExcavationClient.RA_Bill_No).limit(10).all()
        logger.debug(f"  Sample RA_Bill_No values (Client): {[str(r[0]) for r in sample_bills_client]}")
        
        # 6. Fetch RA Bills from BOTH Subcontractor and Client tables
        logger.info("Step 5: Fetching distinct RA Bills from both Subcontractor and Client data...")
        
        # Get RA bills from Subcontractor data
        subcon_ra_bills = db.session.query(TrenchExcavation.RA_Bill_No)\
            .filter(TrenchExcavation.RA_Bill_No != None)\
            .filter(TrenchExcavation.RA_Bill_No != "")\
            .distinct()
        
        logger.debug(f"  Subcontractor RA Bills (before union): {len(subcon_ra_bills.all())}")
        
        # Get RA bills from Client data
        client_ra_bills = db.session.query(TrenchExcavationClient.RA_Bill_No)\
            .filter(TrenchExcavationClient.RA_Bill_No != None)\
            .filter(TrenchExcavationClient.RA_Bill_No != "")\
            .distinct()
        
        logger.debug(f"  Client RA Bills (before union): {len(client_ra_bills.all())}")
        
        # Union both queries to get all unique RA bills
        ra_bills_union = db.session.query(TrenchExcavation.RA_Bill_No)\
            .filter(TrenchExcavation.RA_Bill_No != None)\
            .filter(TrenchExcavation.RA_Bill_No != "")\
            .union(
                db.session.query(TrenchExcavationClient.RA_Bill_No)\
                .filter(TrenchExcavationClient.RA_Bill_No != None)\
                .filter(TrenchExcavationClient.RA_Bill_No != "")
            ).order_by(TrenchExcavation.RA_Bill_No).all()
        
        logger.info(f"✓ Distinct RA Bills found (Combined): {len(ra_bills_union)}")
        ra_bills_list = [r[0] for r in ra_bills_union if r[0]]
        logger.info(f"  RA Bills list: {ra_bills_list}")
        
        # 7. Debug: Check data types
        if ra_bills_union:
            logger.debug(f"  First RA Bill value: {ra_bills_union[0][0]}")
            logger.debug(f"  First RA Bill type: {type(ra_bills_union[0][0])}")

        response = {
            "subcontractors": [s[0] for s in subcontractors if s[0]],
            "ra_bills": ra_bills_list
        }
        
        logger.info(f"✓ Response prepared successfully")
        logger.info("=" * 60)
        
        return jsonify(response)
        
    except Exception as e:
        logger.error("=" * 60)
        logger.error(f"ERROR in get_filters(): {str(e)}")
        logger.error(f"Error type: {type(e).__name__}")
        logger.exception("Full traceback:")
        logger.error("=" * 60)
        return jsonify({"error": str(e)}), 500

# API for the live abstract data - handles multiple table types
@dashboard_bp.route("/api/excavation-abstract")
def excavation_abstract():
    try:
        logger.info("=" * 60)
        logger.info("EXCAVATION ABSTRACT FETCH - START")
        logger.info("=" * 60)
        
        table_type = request.args.get('table_type', 'trench')
        subcon_name = request.args.get('subcontractor', 'All')
        ra_bill = request.args.get('ra_bill', 'Cumulative')
        
        logger.info(f"Request Parameters:")
        logger.info(f"  Table Type: {table_type}")
        logger.info(f"  Subcontractor: {subcon_name}")
        logger.info(f"  RA Bill: {ra_bill}")
        
        # Select models and match keys based on table type
        if table_type == 'trench':
            SubconModel = TrenchExcavation
            ClientModel = TrenchExcavationClient
            table_label = "Trench Excavation"
            location_key = 'Location'
            mh_key = 'MH_NO'
            excavation_columns = [
                ("Soft Murum", "0-1.5m", "Soft_Murum_0_to_1_5"),
                ("Soft Murum", "1.5-3.0m", "Soft_Murum_1_5_to_3_0"),
                ("Soft Murum", "3.0-4.5m", "Soft_Murum_3_0_to_4_5"),
                ("Hard Murum", "0-1.5m", "Hard_Murum_0_to_1_5"),
                ("Hard Murum", "1.5-3.0m", "Hard_Murum_1_5_to_3_0"),
                ("Soft Rock", "0-1.5m", "Soft_Rock_0_to_1_5"),
                ("Soft Rock", "1.5-3.0m", "Soft_Rock_1_5_to_3_0"),
                ("Hard Rock", "0-1.5m", "Hard_Rock_0_to_1_5"),
                ("Hard Rock", "1.5-3.0m", "Hard_Rock_1_5_to_3_0"),
                ("Hard Rock", "3.0-4.5m", "Hard_Rock_3_0_to_4_5"),
                ("Hard Rock", "4.5-6.0m", "Hard_Rock_4_5_to_6_0"),
                ("Hard Rock", "6.0-7.5m", "Hard_Rock_6_0_to_7_5"),
            ]
        elif table_type == 'manhole':
            SubconModel = ManholeExcavation
            ClientModel = ManholeExcavationClient
            table_label = "Manhole Excavation"
            location_key = 'Location'
            mh_key = 'MH_NO'
            excavation_columns = [
                ("Soft Murum", "0-1.5m", "Soft_Murum_0_to_1_5"),
                ("Soft Murum", "1.5-3.0m", "Soft_Murum_1_5_to_3_0"),
                ("Hard Murum", "0-1.5m", "Hard_Murum_0_to_1_5"),
                ("Hard Murum", "1.5-3.0m", "Hard_Murum_1_5_to_3_0"),
                ("Soft Rock", "0-1.5m", "Soft_Rock_0_to_1_5"),
                ("Soft Rock", "1.5-3.0m", "Soft_Rock_1_5_to_3_0"),
                ("Hard Rock", "0-1.5m", "Hard_Rock_0_to_1_5"),
                ("Hard Rock", "1.5-3.0m", "Hard_Rock_1_5_to_3_0"),
            ]
        elif table_type == 'laying':
            SubconModel = Laying
            ClientModel = LayingClient
            table_label = "Laying"
            location_key = 'Location'
            mh_key = 'MH_NO'
            excavation_columns = [
                ("Soft Murum", "0-1.5m", "Soft_Murum_0_to_1_5"),
                ("Soft Murum", "1.5-3.0m", "Soft_Murum_1_5_to_3_0"),
                ("Hard Murum", "0-1.5m", "Hard_Murum_0_to_1_5"),
                ("Hard Murum", "1.5-3.0m", "Hard_Murum_1_5_to_3_0"),
                ("Soft Rock", "0-1.5m", "Soft_Rock_0_to_1_5"),
                ("Soft Rock", "1.5-3.0m", "Soft_Rock_1_5_to_3_0"),
                ("Hard Rock", "0-1.5m", "Hard_Rock_0_to_1_5"),
                ("Hard Rock", "1.5-3.0m", "Hard_Rock_1_5_to_3_0"),
            ]
        else:
            return jsonify({"error": f"Invalid table_type: {table_type}"}), 400
        
        logger.info(f"Using table: {table_label}")
        
        # ===== FETCH SUBCONTRACTOR DATA =====
        logger.info(f"Fetching Subcontractor data ({SubconModel.__tablename__})...")
        subcon_query = db.session.query(SubconModel)
        
        # Check if SubconModel has subcontractor relationship
        if hasattr(SubconModel, 'subcontractor_id'):
            subcon_query = subcon_query.join(
                Subcontractor, Subcontractor.id == SubconModel.subcontractor_id
            )
            if subcon_name != 'All':
                subcon_query = subcon_query.filter(Subcontractor.subcontractor_name == subcon_name)
        
        subcon_results = subcon_query.all()
        logger.info(f"  Found {len(subcon_results)} subcontractor records")
        
        # ===== FETCH CLIENT DATA =====
        logger.info(f"Fetching Client data ({ClientModel.__tablename__})...")
        client_query = db.session.query(ClientModel)
        
        if ra_bill != 'Cumulative' and hasattr(ClientModel, 'RA_Bill_No'):
            client_query = client_query.filter(ClientModel.RA_Bill_No == ra_bill)
        
        client_results = client_query.all()
        logger.info(f"  Found {len(client_results)} client records")
        
        # ===== MATCH RECORDS BY MH_NO AND LOCATION =====
        logger.info("Matching records by MH_NO and Location...")
        matched_data = {}
        
        # Build a map of client records by MH_NO + Location
        client_map = {}
        for client_record in client_results:
            mh_no = getattr(client_record, mh_key)
            location = getattr(client_record, location_key)
            key = f"{location}|{mh_no}"
            client_map[key] = client_record
        
        logger.info(f"  Client map has {len(client_map)} unique MH_NO+Location combinations")
        
        # Match subcontractor records with client records
        match_count = 0
        for subcon_record in subcon_results:
            mh_no = getattr(subcon_record, mh_key)
            location = getattr(subcon_record, location_key)
            key = f"{location}|{mh_no}"
            
            # Only process if matching client record exists
            if key in client_map:
                match_count += 1
                client_record = client_map[key]
                
                # Aggregate excavation data for this matched pair
                for soil, depth, col_name in excavation_columns:
                    record_key = f"{soil}|{depth}|{location}|{mh_no}"
                    
                    # Get values
                    subcon_val = 0
                    client_val = 0
                    
                    if hasattr(subcon_record, col_name):
                        subcon_val = getattr(subcon_record, col_name) or 0
                    
                    if hasattr(client_record, col_name):
                        client_val = getattr(client_record, col_name) or 0
                    
                    # Only add if at least one has data
                    if subcon_val > 0 or client_val > 0:
                        if record_key not in matched_data:
                            matched_data[record_key] = {
                                "soil_type": soil,
                                "depth": depth,
                                "location": location,
                                "mh_no": mh_no,
                                "client_qty": 0,
                                "subcon_qty": 0
                            }
                        matched_data[record_key]["client_qty"] += client_val
                        matched_data[record_key]["subcon_qty"] += subcon_val
        
        logger.info(f"  Matched {match_count} subcontractor records with client records")
        logger.info(f"  Found {len(matched_data)} excavation items with data")
        
        # Calculate differences and format response
        data = []
        for key, item in matched_data.items():
            difference = item["subcon_qty"] - item["client_qty"]
            # Format label as: "Soft Murum 0-1.5m (Location - MH_NO)"
            label = f"{item['soil_type']} {item['depth']}"
            data.append({
                "label": label,
                "soil_type": item["soil_type"],
                "depth": item["depth"],
                "location": item["location"],
                "mh_no": item["mh_no"],
                "client_qty": round(item["client_qty"], 2),
                "subcon_qty": round(item["subcon_qty"], 2),
                "difference": round(difference, 2)
            })
        
        # Sort by location and mh_no for consistency
        data.sort(key=lambda x: (x["location"], x["mh_no"], x["soil_type"], x["depth"]))
        
        logger.info(f"Response prepared with {len(data)} matched records")
        logger.info("=" * 60)
        
        return jsonify(data)
    except Exception as e:
        logger.error("=" * 60)
        logger.error(f"ERROR in excavation_abstract(): {str(e)}")
        logger.error(f"Error type: {type(e).__name__}")
        logger.exception("Full traceback:")
        logger.error("=" * 60)
        return jsonify({"error": str(e)}), 500

@dashboard_bp.route("/")
def dashboard():
    if not session.get("user_id"):
        return redirect(url_for("auth.login"))
    return render_template("dashboard.html", title="Live Excavation Dashboard")