new dashboard 1

app/routes/generate_comparison_report.py

@@ -1,6 +1,8 @@
 from flask import Blueprint, render_template, request, send_file, flash
 import pandas as pd
 import io
+import re
+from collections import defaultdict
 
 from app.models.subcontractor_model import Subcontractor
 from app.models.trench_excavation_model import TrenchExcavation
@@ -14,26 +16,20 @@ from app.models.mh_dc_client_model import ManholeDomesticChamberClient
 from app.models.laying_client_model import LayingClient
 
 from app.utils.helpers import login_required
-import re
-
 
 generate_report_bp = Blueprint("generate_report", __name__, url_prefix="/report")
 
-
+# --- REGEX PATTERNS FOR TOTALING ---
-# sum field  of pipe laying (pipe_150_mm)
 PIPE_MM_PATTERN = re.compile(r"^pipe_\d+_mm$")
-# sum fields of MH dc (d_0_to_0_75)
+D_RANGE_PATTERN = re.compile(r"^d_\d+(?:_\d+)?_to_\d+(?:_\d+)?$")
-D_RANGE_PATTERN = re.compile( r"^d_\d+(?:_\d+)?_to_\d+(?:_\d+)?$")
 
+# --- UTILITIES ---
 
-# NORMALIZER
 def normalize_key(value):
     if value is None:
-        return None
+        return ""
     return str(value).strip().upper()
 
-
-# HEADER FORMATTER
 def format_header(header):
     if "-" in header:
         prefix, rest = header.split("-", 1)
@@ -44,7 +40,6 @@ def format_header(header):
     parts = rest.split("_")
     result = []
     i = 0
-
     while i < len(parts):
         if i + 1 < len(parts) and parts[i].isdigit() and parts[i + 1].isdigit():
             result.append(f"{parts[i]}.{parts[i + 1]}")
@@ -56,122 +51,125 @@ def format_header(header):
     final_text = " ".join(result)
     return f"{prefix}-{final_text}" if prefix else final_text
 
-
-# LOOKUP CREATOR 
 def make_lookup(rows, key_field):
-    lookup = {}
+    """Creates a mapping of (Location, Key) to a list of records."""
+    lookup = defaultdict(list)
     for r in rows:
-        location = normalize_key(r.get("Location"))
+        # Check both capitalized and lowercase keys for robustness
-        key_val = normalize_key(r.get(key_field))
+        loc = normalize_key(r.get("Location") or r.get("location"))
-
+        key = normalize_key(r.get(key_field) or r.get(key_field.lower()))
-        if location and key_val:
+        if loc and key:
-            lookup[(location, key_val)] = r
+            lookup[(loc, key)].append(r)
-
     return lookup
 
+def calculate_row_total(row_dict):
+    """Calculates total based on _total suffix or regex patterns."""
+    return sum(
+        float(v or 0) for k, v in row_dict.items()
+        if k.endswith("_total") or D_RANGE_PATTERN.match(k) or PIPE_MM_PATTERN.match(k)
+    )
+
+# --- CORE COMPARISON LOGIC ---
+
 
-# COMPARISON BUILDER
 def build_comparison(client_rows, contractor_rows, key_field):
-    contractor_lookup = make_lookup(contractor_rows, key_field)
+    # 1. Create Lookup for Subcontractors
+    contractor_lookup = {}
+    for r in contractor_rows:
+        loc = normalize_key(r.get("Location") or r.get("location"))
+        key = normalize_key(r.get(key_field) or r.get(key_field.lower()))
+        if loc and key:
+            contractor_lookup[(loc, key)] = r
+
     output = []
 
+    # 2. Iterate through Client rows
     for c in client_rows:
-        client_location = normalize_key(c.get("Location"))
+        loc_raw = c.get("Location") or c.get("location")
-        client_key = normalize_key(c.get(key_field))
+        key_raw = c.get(key_field) or c.get(key_field.lower())
+        
+        loc_norm = normalize_key(loc_raw)
+        key_norm = normalize_key(key_raw)
 
-        if not client_location or not client_key:
+        # Match check
-            continue
+        s = contractor_lookup.get((loc_norm, key_norm))
+        
+        # We only include the row if there is a match (Inner Join)
+        if s:
+            client_total = calculate_row_total(c)
+            sub_total = calculate_row_total(s)
 
-        s = contractor_lookup.get((client_location, client_key))
+            row = {
-        if not s:
+                "Location": loc_raw,
-            continue
+                key_field.replace("_", " "): key_raw
+            }
 
-        client_total = sum(
+            # Add Client Data
-            float(v or 0)
+            for k, v in c.items():
-            for k, v in c.items()
+                if k in ["id", "created_at"]: continue
-            if k.endswith("_total") or D_RANGE_PATTERN.match(k) or PIPE_MM_PATTERN.match(k)
+                row[f"Client-{k}"] = v
-        )
+            row["Client-Total"] = round(client_total, 2)
 
-        sub_total = sum(
+            row[" "] = "" # Spacer
-            float(v or 0)
-            for k, v in s.items()
-            if k.endswith("_total") or D_RANGE_PATTERN.match(k) or PIPE_MM_PATTERN.match(k)
-        )
 
-        diff = client_total - sub_total
+            # Add Subcontractor Data (Aligned on same row)
+            for k, v in s.items():
+                if k in ["id", "created_at", "subcontractor_id"]: continue
+                row[f"Subcontractor-{k}"] = v
+            
+            row["Subcontractor-Total"] = round(sub_total, 2)
+            row["Diff"] = round(client_total - sub_total, 2)
+            
+            output.append(row)
 
-        row = {
+    # 3. Handle the "Empty/Blank" scenario using pd.concat
-            "Location": client_location,
+    if not output:
-            key_field.replace("_", " "): client_key
+        # Create a basic dataframe with a message so the Excel file isn't empty/corrupt
-        }
+        return pd.DataFrame([{"Location": "N/A", "Message": "No matching data found"}])
-
-        # CLIENT DATA
-        for k, v in c.items():
-            if k in ["id", "created_at"]:
-                continue
-            row[f"Client-{k}"] = v
-
-        row["Client-Total"] = round(client_total, 2)
-        row[" "] = ""
-
-        # SUBCONTRACTOR DATA
-        for k, v in s.items():
-            if k in ["id", "created_at", "subcontractor_id"]:
-                continue
-            row[f"Subcontractor-{k}"] = v
-
-        row["Subcontractor-Total"] = round(sub_total, 2)
-        row["Diff"] = round(diff, 2)
-
-        output.append(row)
 
     df = pd.DataFrame(output)
     df.columns = [format_header(col) for col in df.columns]
     return df
 
+# --- EXCEL WRITER ---
 
-# EXCEL SHEET WRITER
 def write_sheet(writer, df, sheet_name, subcontractor_name):
+    if df.empty:
+        return
+        
     workbook = writer.book
     df.to_excel(writer, sheet_name=sheet_name, index=False, startrow=3)
     ws = writer.sheets[sheet_name]
 
+    # Formats
     title_fmt = workbook.add_format({"bold": True, "font_size": 14})
-    client_fmt = workbook.add_format({"bold": True, "border": 1, "bg_color": "#B6DAED"})
+    client_header_fmt = workbook.add_format({"bold": True, "border": 1, "bg_color": "#B6DAED", "align": "center"})
-    sub_fmt = workbook.add_format({"bold": True, "border": 1, "bg_color": "#F3A081"})
+    sub_header_fmt = workbook.add_format({"bold": True, "border": 1, "bg_color": "#F3A081", "align": "center"})
-    total_fmt = workbook.add_format({"bold": True, "border": 1, "bg_color": "#F7D261"})
+    total_fmt = workbook.add_format({"bold": True, "border": 1, "bg_color": "#F7D261", "align": "center"})
-    diff_fmt = workbook.add_format({"bold": True, "border": 1, "bg_color": "#82DD49"})
+    diff_fmt = workbook.add_format({"bold": True, "border": 1, "bg_color": "#82DD49", "align": "center"})
-    default_header_fmt = workbook.add_format({"bold": True,"border": 1,"bg_color": "#E7E6E6","align": "center","valign": "vcenter"})
+    default_header_fmt = workbook.add_format({"bold": True, "border": 1, "bg_color": "#E7E6E6", "align": "center"})
-
-
-    ws.merge_range(
-        0, 0, 0, len(df.columns) - 1,
-        "CLIENT vs SUBCONTRACTOR",
-        title_fmt
-    )
-    ws.merge_range(
-        1, 0, 1, len(df.columns) - 1,
-        f"Subcontractor Name - {subcontractor_name}",
-        title_fmt
-    )
 
+    # Header Titles
+    ws.merge_range(0, 0, 0, len(df.columns) - 1, "CLIENT vs SUBCONTRACTOR COMPARISON", title_fmt)
+    ws.merge_range(1, 0, 1, len(df.columns) - 1, f"Subcontractor: {subcontractor_name}", title_fmt)
 
     for col_num, col_name in enumerate(df.columns):
         if col_name.startswith("Client-"):
-            ws.write(3, col_num, col_name, client_fmt)
+            fmt = client_header_fmt
         elif col_name.startswith("Subcontractor-"):
-            ws.write(3, col_num, col_name, sub_fmt)
+            fmt = sub_header_fmt
-        elif col_name.endswith("_total") or col_name.endswith("_total") :
+        elif "Total" in col_name:
-            ws.write(3, col_num, col_name, total_fmt)
+            fmt = total_fmt
         elif col_name == "Diff":
-            ws.write(3, col_num, col_name, diff_fmt)
+            fmt = diff_fmt
         else:
-            ws.write(3, col_num, col_name, default_header_fmt)
+            fmt = default_header_fmt
+            
+        ws.write(3, col_num, col_name, fmt)
+        ws.set_column(col_num, col_num, 18)
 
-        ws.set_column(col_num, col_num, 20)
+# --- ROUTES ---
 
-
-# REPORT ROUTE
 @generate_report_bp.route("/comparison_report", methods=["GET", "POST"])
 @login_required
 def comparison_report():
@@ -180,48 +178,29 @@ def comparison_report():
     if request.method == "POST":
         subcontractor_id = request.form.get("subcontractor_id")
         if not subcontractor_id:
-            flash("Please select subcontractor", "danger")
+            flash("Please select a subcontractor", "danger")
-            return render_template("generate_comparison_report.html",subcontractors=subcontractors)
+            return render_template("generate_comparison_report.html", subcontractors=subcontractors)
 
         subcontractor = Subcontractor.query.get_or_404(subcontractor_id)
 
-        # -------- DATA --------
+        # Build Dataframes for each section
-        tr_client = [r.serialize() for r in TrenchExcavationClient.query.all()]
+        sections = [
-        tr_sub = [r.serialize() for r in TrenchExcavation.query.filter_by(
+            (TrenchExcavationClient, TrenchExcavation, "Tr.Ex"),
-            subcontractor_id=subcontractor_id
+            (ManholeExcavationClient, ManholeExcavation, "Mh.Ex"),
-        ).all()]
+            (ManholeDomesticChamberClient, ManholeDomesticChamber, "MH & DC"),
-        df_tr = build_comparison(tr_client, tr_sub, "MH_NO")
+            (LayingClient, Laying, "Laying")
+        ]
 
-        mh_client = [r.serialize() for r in ManholeExcavationClient.query.all()]
-        mh_sub = [r.serialize() for r in ManholeExcavation.query.filter_by(
-            subcontractor_id=subcontractor_id
-        ).all()]
-        df_mh = build_comparison(mh_client, mh_sub, "MH_NO")
-
-        dc_client = [r.serialize() for r in ManholeDomesticChamberClient.query.all()]
-        dc_sub = [r.serialize() for r in ManholeDomesticChamber.query.filter_by(
-            subcontractor_id=subcontractor_id
-        ).all()]
-        df_dc = build_comparison(dc_client, dc_sub, "MH_NO")
-        # df_dc = build_comparison_mh_dc(dc_client, dc_sub, "MH_NO")
-
-        lay_client = [r.serialize() for r in LayingClient.query.all()]
-        lay_sub = [r.serialize() for r in Laying.query.filter_by(
-            subcontractor_id=subcontractor_id
-        ).all()]
-        df_lay = build_comparison(lay_client, lay_sub, "MH_NO")
-        # df_lay = build_comparison_laying(lay_client, lay_sub, "MH_NO")
-
-
-        # -------- EXCEL --------
         output = io.BytesIO()
         filename = f"{subcontractor.subcontractor_name}_Comparison_Report.xlsx"
 
         with pd.ExcelWriter(output, engine="xlsxwriter") as writer:
-            write_sheet(writer, df_tr, "Tr.Ex", subcontractor.subcontractor_name)
+            for client_model, sub_model, sheet_name in sections:
-            write_sheet(writer, df_mh, "Mh.Ex", subcontractor.subcontractor_name)
+                c_data = [r.serialize() for r in client_model.query.all()]
-            write_sheet(writer, df_dc, "MH & DC", subcontractor.subcontractor_name)
+                s_data = [r.serialize() for r in sub_model.query.filter_by(subcontractor_id=subcontractor_id).all()]
-            write_sheet(writer, df_lay, "Laying", subcontractor.subcontractor_name)
+                
+                df = build_comparison(c_data, s_data, "MH_NO")
+                write_sheet(writer, df, sheet_name, subcontractor.subcontractor_name)
 
         output.seek(0)
         return send_file(
@@ -231,107 +210,4 @@ def comparison_report():
             mimetype="application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"
         )
 
-    return render_template("generate_comparison_report.html",subcontractors=subcontractors)
+    return render_template("generate_comparison_report.html", subcontractors=subcontractors)
-
-
-# def build_comparison_mh_dc(client_rows, contractor_rows, key_field):
-#     contractor_lookup = make_lookup(contractor_rows, key_field)
-#     mh_dc_fields = ManholeDomesticChamberClient.sum_mh_dc_fields()
-
-#     output = []
-
-#     for c in client_rows:
-#         loc = normalize_key(c.get("Location"))
-#         key = normalize_key(c.get(key_field))
-#         if not loc or not key:
-#             continue
-
-#         s = contractor_lookup.get((loc, key))
-#         if not s:
-#             continue
-
-#         client_total = sum(float(c.get(f, 0) or 0) for f in mh_dc_fields)
-#         sub_total = sum(float(s.get(f, 0) or 0) for f in mh_dc_fields)
-
-#         row = {
-#             "Location": loc,
-#             key_field.replace("_", " "): key
-#         }
-
-#         # CLIENT – ALL FIELDS
-#         for k, v in c.items():
-#             if k in ["id", "created_at"]:
-#                 continue
-#             row[f"Client-{k}"] = v
-
-#         row["Client-Total"] = round(client_total, 2)
-#         row[" "] = ""
-
-#         # SUBCONTRACTOR – ALL FIELDS
-#         for k, v in s.items():
-#             if k in ["id", "created_at", "subcontractor_id"]:
-#                 continue
-#             row[f"Subcontractor-{k}"] = v
-
-#         row["Subcontractor-Total"] = round(sub_total, 2)
-#         row["Diff"] = round(client_total - sub_total, 2)
-
-#         output.append(row)
-
-#     df = pd.DataFrame(output)
-#     df.columns = [format_header(col) for col in df.columns]
-#     return df
-
-
-# def build_comparison_laying(client_rows, contractor_rows, key_field):
-#     contractor_lookup = make_lookup(contractor_rows, key_field)
-#     laying_fields = Laying.sum_laying_fields()
-
-#     output = []
-
-#     for c in client_rows:
-#         loc = normalize_key(c.get("Location"))
-#         key = normalize_key(c.get(key_field))
-#         if not loc or not key:
-#             continue
-
-#         s = contractor_lookup.get((loc, key))
-#         if not s:
-#             continue
-
-#         client_total = sum(float(c.get(f, 0) or 0) for f in laying_fields)
-#         sub_total = sum(float(s.get(f, 0) or 0) for f in laying_fields)
-
-#         print("--------------",key,"----------")
-#         print("sum -client_total ",client_total)
-#         print("sum -sub_total ",sub_total)
-#         print("Diff ---- ",client_total - sub_total)
-#         print("------------------------")
-#         row = {
-#             "Location": loc,
-#             key_field.replace("_", " "): key
-#         }
-
-#         # CLIENT – ALL FIELDS
-#         for k, v in c.items():
-#             if k in ["id", "created_at"]:
-#                 continue
-#             row[f"Client-{k}"] = v
-
-#         row["Client-Total"] = round(client_total, 2)
-#         row[" "] = ""
-
-#         # SUBCONTRACTOR – ALL FIELDS
-#         for k, v in s.items():
-#             if k in ["id", "created_at", "subcontractor_id"]:
-#                 continue
-#             row[f"Subcontractor-{k}"] = v
-
-#         row["Subcontractor-Total"] = round(sub_total, 2)
-#         row["Diff"] = round(client_total - sub_total, 2)
-
-#         output.append(row)
-
-#     df = pd.DataFrame(output)
-#     df.columns = [format_header(col) for col in df.columns]
-#     return df