webhookasdsdfsdf

Reviewed-on: #10

.env

@@ -0,0 +1,26 @@
+# -----------------------------
+# Flask App Configuration
+# -----------------------------
+FLASK_ENV=development
+FLASK_DEBUG=True
+FLASK_HOST=0.0.0.0
+FLASK_PORT=5001
+
+# -----------------------------
+# Security
+# -----------------------------
+SECRET_KEY=change-this-to-strong-secret-key
+
+# -----------------------------
+# Database Configuration
+# -----------------------------
+DB_DIALECT=mysql
+DB_DRIVER=pymysql
+DB_HOST=127.0.0.1
+DB_PORT=3306
+DB_NAME=comparisondb
+DB_USER=root
+DB_PASSWORD=admin
+
+# DATABASE_URL=mysql+pymysql://root:root@localhost/comparisondb
+

.gitignore

@@ -1,9 +1,20 @@
-# Ignore folders
+# Python
+app/__pycache__/
+*.pyc
+*.pyo
+*.pyd
+*.__pycache__
+
+# Ingnor upload files
+app/static/uploads/
+
+# Ignore env files
+venv
+
+# Ignore Log files ss
+logs/
+
+# Ignore db folders
 instance/
-static/uploads/
 
-# Ignore files
-.env
 
-# Ignore by type
-*.log

Dockerfile

@@ -0,0 +1,24 @@
+FROM python:3.11-slim
+
+WORKDIR /app
+
+# Install system dependencies
+RUN apt-get update && apt-get install -y \
+    gcc \
+    && rm -rf /var/lib/apt/lists/*
+
+# Copy requirements and install Python dependencies
+COPY requirements.txt .
+RUN pip install --no-cache-dir -r requirements.txt
+
+# Copy application code
+COPY . .
+
+# Create necessary directories
+RUN mkdir -p app/logs app/static/uploads app/static/downloads
+
+# Expose port
+EXPOSE 5000
+
+# Run the application
+CMD ["python", "run.py"]

README.md

@@ -1,3 +1,583 @@
-# Comparison_Project
+# Comparison Project - Backend Documentation
 
-Comparison Project 
+A Flask-based web application for comparing and managing construction project data (excavation and manhole work) between subcontractors and clients. The system imports data from Excel files, stores it in a database, and generates comparison reports.
+
+## Table of Contents
+
+- [Project Overview](#project-overview)
+- [Tech Stack](#tech-stack)
+- [Project Structure](#project-structure)
+- [Database Schema](#database-schema)
+- [Application Flow](#application-flow)
+- [API Routes & Endpoints](#api-routes--endpoints)
+- [Setup & Installation](#setup--installation)
+
+---
+
+## Project Overview
+
+The Comparison Project is designed to:
+- **Manage subcontractors** and their excavation data (Manhole Excavation, Trench Excavation, Domestic Chambers)
+- **Import client-side project data** for comparison
+- **Compare subcontractor work** against client specifications
+- **Generate detailed reports** highlighting differences between client and subcontractor data
+- **User authentication** with login/registration system
+
+### Key Features
+
+✅ User Authentication (Register/Login/Logout)  
+✅ Subcontractor Management (Add/Edit/List/Delete)  
+✅ File Import (Excel/CSV) for both Client and Subcontractor data  
+✅ Data validation and duplicate detection  
+✅ Comparison Reports with Excel export  
+✅ Dashboard with file management  
+
+---
+
+## Tech Stack
+
+**Backend Framework**: Flask  
+**Database**: SQL Database (MySQL/PostgreSQL/SQLite configured via environment variables)  
+**ORM**: SQLAlchemy  
+**File Processing**: Pandas, OpenPyXL, XlsxWriter  
+**Authentication**: Werkzeug (Password hashing)  
+**Configuration**: Python Dotenv  
+
+### Dependencies
+
+```
+Flask
+pandas
+openpyxl
+xlrd
+Werkzeug
+python-dotenv
+cryptography
+xlsxwriter
+matplotlib 
+
+```
+
+---
+
+## Project Structure
+
+```
+Comparison_Project/
+├── run.py                          # Application entry point
+├── requirements.txt                # Python dependencies
+├── .env                            # Environment variables (DB, secrets)
+├── README.md                       # This file
+│
+├── app/
+│   ├── __init__.py                # Flask app initialization
+│   ├── config.py                  # Configuration settings
+│   │
+│   ├── models/                    # Database models (SQLAlchemy ORM)
+│   │   ├── user_model.py          # User table schema
+│   │   ├── subcontractor_model.py # Subcontractor table schema
+│   │   ├── manhole_excavation_model.py        # Subcontractor MH excavation
+│   │   ├── trench_excavation_model.py         # Subcontractor Trench excavation
+│   │   ├── manhole_domestic_chamber_model.py  # Subcontractor Domestic chamber
+│   │   ├── mh_ex_client_model.py   # Client MH excavation
+│   │   ├── tr_ex_client_model.py   # Client Trench excavation
+│   │   └── mh_dc_client_model.py   # Client Domestic chamber
+│   │
+│   ├── routes/                    # Flask Blueprints (API endpoints)
+│   │   ├── auth.py                # Login, Register, Logout
+│   │   ├── dashboard.py           # Dashboard page
+│   │   ├── file_import.py         # File upload endpoints
+│   │   ├── file_report.py         # Fetch and format data for reports
+│   │   ├── generate_comparison_report.py  # Comparison logic & Excel export
+│   │   ├── subcontractor_routes.py        # CRUD for subcontractors
+│   │   ├── user.py                # User management routes
+│   │   └── file_format.py         # File format validation
+│   │
+│   ├── services/                  # Business logic layer
+│   │   ├── db_service.py          # Database connection & SQLAlchemy init
+│   │   ├── file_service.py        # File parsing & data insertion
+│   │   └── user_service.py        # User authentication logic
+│   │
+│   ├── utils/                     # Helper functions
+│   │   ├── file_utils.py          # File path utilities
+│   │   ├── helpers.py             # Decorators, common helpers
+│   │   └── __pycache__/
+│   │
+│   ├── static/                    # Static assets
+│   │   ├── css/                   # Stylesheets
+│   │   ├── images/                # Images
+│   │   ├── uploads/               # Uploaded files directory
+│   │   │   ├── Client_Bill_1/
+│   │   │   └── sub_1/
+│   │   └── downloads/             # Generated reports
+│   │
+│   ├── templates/                 # Jinja2 HTML templates
+│   │   ├── base.html              # Base template
+│   │   ├── login.html
+│   │   ├── register.html
+│   │   ├── dashboard.html
+│   │   ├── file_import.html
+│   │   ├── file_import_client.html
+│   │   ├── file_format.html
+│   │   ├── file_report.html
+│   │   ├── generate_comparison_report.html
+│   │   ├── generate_comparison_client_vs_subcont.html
+│   │   ├── list_user.html
+│   │   ├── report.html
+│   │   ├── users.htm
+│   │   └── subcontractor/
+│   │       ├── add.html
+│   │       ├── edit.html
+│   │       └── list.html
+│   │
+│   └── logs/                      # Application logs
+│
+├── instance/                      # Instance folder (DB, temp files)
+└── logs/                          # Root level logs
+```
+
+---
+
+## Database Schema
+
+### 1. **Users Table** (`users`)
+Stores user authentication data.
+
+| Column | Type | Constraints | Purpose |
+|--------|------|-------------|---------|
+| `id` | Integer | Primary Key | User identifier |
+| `name` | String(120) | NOT NULL | User's full name |
+| `email` | String(120) | UNIQUE, NOT NULL | User's email |
+| `password_hash` | String(255) | NOT NULL | Hashed password (Werkzeug) |
+
+**Key Methods:**
+- `set_password(password)` - Hash and store password
+- `check_password(password)` - Verify password
+
+---
+
+### 2. **Subcontractors Table** (`subcontractors`)
+Stores subcontractor company information.
+
+| Column | Type | Constraints | Purpose |
+|--------|------|-------------|---------|
+| `id` | Integer | Primary Key | Subcontractor ID |
+| `subcontractor_name` | String(255) | NOT NULL | Company name |
+| `address` | String(500) | - | Company address |
+| `gst_no` | String(50) | - | GST number |
+| `pan_no` | String(50) | - | PAN number |
+| `mobile_no` | String(20) | - | Contact phone |
+| `email_id` | String(150) | - | Contact email |
+| `contact_person` | String(150) | - | Contact person name |
+| `status` | String(20) | Default: "Active" | Active/Inactive status |
+| `created_at` | DateTime | Default: today | Record creation date |
+
+**Relationships:**
+- One-to-Many with `manhole_excavation`
+- One-to-Many with `trench_excavation`
+- One-to-Many with `manhole_domestic_chamber`
+
+---
+
+### 3. **Manhole Excavation (Subcontractor)** (`manhole_excavation`)
+Stores manhole excavation work data from subcontractors.
+
+| Column Type | Purpose |
+|-----|---------|
+| `id` | Primary Key |
+| `subcontractor_id` | Foreign Key → Subcontractor |
+| `Location` | Worksite location |
+| `MH_NO` | Manhole number |
+| `RA_Bill_No` | RA Bill reference (for grouping) |
+| `Upto_IL_Depth`, `Cutting_Depth`, `ID_of_MH_m`, `Ex_Dia_of_Manhole`, `Area_of_Manhole` | Dimension fields |
+| **Excavation Categories** (by depth ranges): | |
+| Soft_Murum_0_to_1_5, Soft_Murum_1_5_to_3_0, etc. | Material type + depth range |
+| Hard_Murum_0_to_1_5, Hard_Murum_1_5_to_3_0, etc. | Hard mudstone layers |
+| Soft_Rock_0_to_1_5, Soft_Rock_1_5_to_3_0, etc. | Soft rock layers |
+| Hard_Rock_0_to_1_5 through Hard_Rock_6_0_to_7_5 | Hard rock layers |
+| **Totals** | Sum per category |
+| `Total` | Grand total excavation |
+| `Remarks`, `created_at` | Notes and timestamp |
+
+---
+
+### 4. **Trench Excavation (Subcontractor)** (`trench_excavation`)
+Stores trench/sewer line excavation work data from subcontractors.
+
+**Similar structure to Manhole Excavation with additional fields:**
+- Width categories: `Width_0_to_2_5`, `Width_2_5_to_3_0`, etc.
+- `Avg_Depth`, `Actual_Trench_Length`, `Pipe_Dia_mm`
+
+---
+
+### 5. **Manhole Domestic Chamber (Subcontractor)** (`manhole_domestic_chamber`)
+Stores domestic chamber construction data.
+
+| Key Fields | Purpose |
+|-----------|---------|
+| `id`, `subcontractor_id` | Primary & Foreign Key |
+| `Location`, `MH_NO`, `RA_Bill_No` | Identification |
+| `Depth_of_MH`, `MH_TOP_LEVEL`, `MH_IL_LEVEL` | Dimensions |
+| `d_0_to_1_5` through `d_6_0_to_6_5` | Depth-based excavation categories |
+| `Domestic_Chambers` | Count of chambers |
+| `DWC_Pipe_Length`, `UPVC_Pipe_Length` | Pipe lengths |
+
+---
+
+### 6-8. **Client Data Tables** (Similar structure to subcontractor models)
+
+- **`mh_ex_client`** - Manhole Excavation (Client specifications)
+- **`tr_ex_client`** - Trench Excavation (Client specifications)
+- **`mh_dc_client`** - Manhole Domestic Chamber (Client specifications)
+
+**Note:** Client tables do NOT have `subcontractor_id` as they represent client baseline data.
+
+---
+
+## Application Flow
+
+### 1. **User Authentication Flow**
+```
+User Access (/) 
+  ↓
+Redirect to /login
+  ↓
+[Login Page]
+  ├─ POST /auth/login
+  │   ↓
+  │   UserService.validate_login(email, password)
+  │   ↓
+  │   Query User table, verify password
+  │   ↓
+  │   Set session["user_id"], session["user_name"]
+  ↓
+Redirect to /dashboard
+```
+
+### 2. **Subcontractor File Import Flow**
+```
+User → /file/import [GET]
+  ↓
+Display form with subcontractor dropdown
+  ↓
+User selects subcontractor & uploads Excel file
+  ↓
+POST /file/import
+  ↓
+FileService.handle_file_upload()
+  ├─ Validate file type (xlsx, xls, csv)
+  ├─ Create upload folder: static/uploads/sub_{id}/
+  ├─ Save file
+  │
+  ├─ Read Excel sheets:
+  │  ├─ "Tr.Ex." (Trench Excavation)
+  │  ├─ "MH Ex." (Manhole Excavation)
+  │  └─ "MH & DC" (Manhole & Domestic Chamber)
+  │
+  ├─ Process each sheet:
+  │  ├─ Normalize column names
+  │  ├─ Clean data (handle NaN, empty values)
+  │  ├─ Check for duplicates by (Location, MH_NO, RA_Bill_No, subcontractor_id)
+  │  ├─ Insert records into respective tables
+  │  └─ Commit to database
+  │
+  └─ Return success/error message
+```
+
+### 3. **Client File Import Flow**
+```
+User → /file/import_client [GET]
+  ↓
+Display form for client file upload
+  ↓
+User uploads Excel file with client specifications
+  ↓
+POST /file/import_client
+  ↓
+FileService.handle_client_file_upload()
+  ├─ Validate & save file
+  ├─ Read sheets: "Tr.Ex.", "MH Ex.", "MH & DC"
+  ├─ Process and insert into:
+  │  ├─ mh_ex_client
+  │  ├─ tr_ex_client
+  │  └─ mh_dc_client
+  └─ Return status
+```
+
+### 4. **Comparison Report Generation Flow**
+```
+User → /report/generate_comparison [GET]
+  ↓
+Display form to select RA Bill No.
+  ↓
+User selects bill number
+  ↓
+POST /report/generate_comparison
+  ↓
+generate_report_bp routes request
+  ├─ Fetch client data (mh_ex_client, tr_ex_client, mh_dc_client)
+  ├─ Fetch subcontractor data (by RA_Bill_No)
+  ├─ For each record type:
+  │  ├─ Create lookup table by (Location, MH_NO)
+  │  ├─ Match client records with subcontractor records
+  │  ├─ Calculate totals for each category
+  │  ├─ Compute differences/variances
+  │  └─ Format for Excel output
+  │
+  ├─ Generate Excel file with comparison results:
+  │  ├─ Summary sheet
+  │  ├─ Detailed Trench Excavation comparison
+  │  ├─ Detailed Manhole Excavation comparison
+  │  └─ Detailed Domestic Chamber comparison
+  │
+  └─ Send file to client as download
+```
+
+### 5. **Dashboard & Data Retrieval Flow**
+```
+User → /dashboard [GET]
+  ↓
+Check session["user_id"] (authentication)
+  ↓
+Render dashboard.html
+  ├─ Display available reports
+  ├─ List recent RA Bills
+  ├─ Show subcontractor list
+  └─ Provide navigation to:
+     ├─ File import
+     ├─ Reports
+     ├─ Subcontractor management
+     └─ Logout
+```
+
+---
+
+## API Routes & Endpoints
+
+### Authentication Routes (`/auth`)
+| Method | Endpoint | Purpose | Auth |
+|--------|----------|---------|------|
+| GET/POST | `/login` | Login form & validation | ❌ |
+| GET/POST | `/register` | User registration | ❌ |
+| GET | `/logout` | Clear session & logout | ✅ |
+
+### Dashboard Route (`/dashboard`)
+| Method | Endpoint | Purpose | Auth |
+|--------|----------|---------|------|
+| GET | `/dashboard/` | Main dashboard | ✅ |
+
+### Subcontractor Routes (`/subcontractor`)
+| Method | Endpoint | Purpose | Auth |
+|--------|----------|---------|------|
+| GET | `/subcontractor/add` | Add form | ✅ |
+| POST | `/subcontractor/save` | Save new subcontractor | ✅ |
+| GET | `/subcontractor/list` | List all subcontractors | ✅ |
+| GET | `/subcontractor/edit/<id>` | Edit form | ✅ |
+| POST | `/subcontractor/update/<id>` | Update subcontractor | ✅ |
+| GET | `/subcontractor/delete/<id>` | Delete subcontractor | ✅ |
+
+### File Import Routes (`/file`)
+| Method | Endpoint | Purpose | Auth |
+|--------|----------|---------|------|
+| GET/POST | `/file/import` | Subcontractor file upload | ✅ |
+| GET/POST | `/file/import_client` | Client file upload | ✅ |
+| GET/POST | `/file/report` | View imported data report | ✅ |
+
+### Report Routes (`/report`)
+| Method | Endpoint | Purpose | Auth |
+|--------|----------|---------|------|
+| GET/POST | `/report/generate_comparison` | Generate comparison report | ✅ |
+| GET/POST | `/report/generate_comparison_client_vs_subcont` | Alternative comparison | ✅ |
+
+### User Routes (`/user`)
+| Method | Endpoint | Purpose | Auth |
+|--------|----------|---------|------|
+| GET | `/user/list` | List all users | ✅ |
+| Other | (Check routes) | User management | ✅ |
+
+### File Format Route (`/format`)
+| Method | Endpoint | Purpose | Auth |
+|--------|----------|---------|------|
+| GET/POST | `/format/...` | File format reference | ✅ |
+
+---
+
+## Setup & Installation
+
+### Prerequisites
+- Python 3.7+
+- MySQL/PostgreSQL (or SQLite for development)
+- pip (Python package manager)
+
+### 1. Clone & Install Dependencies
+```bash
+cd d:\New folder\Comparison\Comparison_Project
+pip install -r requirements.txt
+```
+
+### 2. Configure Environment Variables
+Create a `.env` file in the project root:
+```env
+# -----------------------------
+# Flask App Configuration
+# -----------------------------
+FLASK_ENV=development
+FLASK_DEBUG=True
+FLASK_HOST='0.0.0.0'
+FLASK_PORT=5001
+
+# -----------------------------
+# Security
+# -----------------------------
+SECRET_KEY=change-this-to-strong-secret-key
+
+# -----------------------------
+# Database Configuration
+# -----------------------------
+DB_DIALECT=mysql
+DB_DRIVER=pymysql
+DB_HOST=127.0.0.1
+DB_PORT=3306
+DB_NAME=comparisondb
+DB_USER=root
+DB_PASSWORD=your_password
+
+
+```
+
+### 3. Initialize Database
+```bash
+python run.py
+```
+This will:
+- Load environment variables
+- Create Flask app with configuration
+- Initialize SQLAlchemy
+- Create all tables (if not exist)
+- Start Flask development server
+
+### 4. Access Application
+Open browser: `http://127.0.0.1:5000/`
+- First time: Redirect to `/login`
+- Register a new account
+- Login and start using the application
+
+---
+
+## Key Services & Utilities
+
+### FileService (`app/services/file_service.py`)
+**Purpose:** Handles file uploads, parsing, and data validation
+
+**Key Methods:**
+- `handle_file_upload(file, subcontractor_id, RA_Bill_No)` - Upload & process subcontractor file
+- `handle_client_file_upload(file, RA_Bill_No)` - Upload & process client file
+- `process_trench_excavation(df, subcontractor_id, RA_Bill_No)` - Parse & insert trench data
+- `process_manhole_excavation(df, subcontractor_id, RA_Bill_No)` - Parse & insert manhole data
+- `process_manhole_domestic_chamber(df, subcontractor_id, RA_Bill_No)` - Parse & insert chamber data
+
+### UserService (`app/services/user_service.py`)
+**Purpose:** User authentication & management
+
+**Key Methods:**
+- `register_user(name, email, password)` - Register new user
+- `validate_login(email, password)` - Authenticate user
+- `get_all_users()` - Fetch all users
+
+### DBService (`app/services/db_service.py`)
+**Purpose:** Database connection & SQLAlchemy initialization
+
+**Exports:**
+- `db` - SQLAlchemy instance for ORM operations
+- `migrate` - Flask-Migrate for schema migrations
+
+---
+
+## Authentication & Security
+
+### Session Management
+- Uses Flask `session` object
+- Stores `user_id` and `user_name` after successful login
+- `@login_required` decorator validates authenticated requests
+
+### Password Security
+- Passwords hashed using Werkzeug's `generate_password_hash()`
+- Verification via `check_password_hash()`
+- No plaintext passwords stored in database
+
+### File Security
+- File uploads sanitized with `secure_filename()`
+- Only allowed extensions: `.xlsx`, `.xls`, `.csv`
+- Files stored in user-specific subfolders
+
+---
+
+## Error Handling & Validation
+
+### File Import Validation
+1. **File Type Check** - Only CSV/XLS/XLSX allowed
+2. **Sheet Validation** - Required sheets must exist
+3. **Column Normalization** - Auto-fixes column name inconsistencies
+4. **Data Type Conversion** - Converts to appropriate types
+5. **Duplicate Detection** - Prevents duplicate records by (Location, MH_NO, RA_Bill_No)
+6. **Null Handling** - Converts empty/NaN values to None
+
+### Database Constraints
+- Foreign key relationships enforced
+- Unique email constraint on users
+- NOT NULL constraints on critical fields
+
+---
+
+## Example: Typical User Workflow
+
+```
+1. User registers/logs in
+   → POST /auth/register (new user) or /auth/login
+   
+2. User adds subcontractors
+   → GET /subcontractor/add
+   → POST /subcontractor/save
+   
+3. User uploads subcontractor data (Excel file with 3 sheets)
+   → GET /file/import
+   → POST /file/import (file, subcontractor_id, RA_Bill_No)
+   → Database populated with excavation data
+   
+4. User uploads client specifications (Excel file with same 3 sheets)
+   → GET /file/import_client
+   → POST /file/import_client (file, RA_Bill_No)
+   → Database populated with client data
+   
+5. User generates comparison report
+   → GET /report/generate_comparison
+   → POST /report/generate_comparison (RA_Bill_No)
+   → System compares client vs subcontractor data
+   → Generates Excel file showing differences
+   → User downloads report
+   
+6. User logs out
+   → GET /auth/logout
+   → Session cleared, redirected to login
+```
+
+---
+
+## Future Enhancements
+
+- [ ] Email notifications for report generation
+- [ ] Data visualization dashboards
+- [ ] Role-based access control (Admin, User, Viewer)
+- [ ] Bulk report generation
+- [ ] Data export to PDF format
+- [ ] Audit logs for data modifications
+- [ ] API documentation (Swagger/OpenAPI)
+- [ ] Unit & integration tests
+
+---
+
+## Support & Contribution
+
+For issues, feature requests, or contributions, please contact the development team.
+
+**Last Updated:** January 2026

app/__init__.py

@@ -1,47 +1,120 @@
-from flask import Flask
-from flask_sqlalchemy import SQLAlchemy
+import logging
+import logging.handlers
+import os
+from flask import Flask, redirect, url_for
 from app.config import Config
-
-db = SQLAlchemy()
+from app.services.db_service import db
 
 def create_app():
     app = Flask(__name__)
     app.config.from_object(Config)
 
+    # Initialize extensions
     db.init_app(app)
+    
+    # Configure logging
+    setup_logging(app)
 
-    # Register Blueprints
-    from app.routes.dashboard import dashboard_bp
-    from app.routes.file_import import file_import_bp
-    # from app.routes.user import user_bp
+    # Register blueprints
+    register_blueprints(app)
+    # Register error handlers
+    register_error_handlers(app)
 
-    app.register_blueprint(dashboard_bp)
-    app.register_blueprint(file_import_bp)
-    # app.register_blueprint(user_bp)
-
-    from app.routes.subcontractor_routes import subcontractor_bp
-    app.register_blueprint(subcontractor_bp)
+    # ROOT → LOGIN
+    @app.route("/")
+    def index():
+        return redirect(url_for("auth.login"))
 
     return app
 
 
+def setup_logging(app):
+    """Configure comprehensive logging for debugging"""
+    
+    # Ensure logs directory exists
+    logs_dir = os.path.join(os.path.dirname(os.path.dirname(__file__)), 'logs')
+    os.makedirs(logs_dir, exist_ok=True)
+    
+    # Create logger
+    logger = logging.getLogger()
+    logger.setLevel(logging.DEBUG)
+    
+    # Remove existing handlers
+    logger.handlers.clear()
+    
+    # Create formatters
+    detailed_formatter = logging.Formatter(
+        '%(asctime)s - %(name)s - %(levelname)s - [%(filename)s:%(lineno)d] - %(message)s',
+        datefmt='%Y-%m-%d %H:%M:%S'
+    )
+    
+    # File handler for all logs
+    file_handler = logging.handlers.RotatingFileHandler(
+        os.path.join(logs_dir, 'app.log'),
+        maxBytes=10485760,  # 10MB
+        backupCount=5
+    )
+    file_handler.setLevel(logging.DEBUG)
+    file_handler.setFormatter(detailed_formatter)
+    logger.addHandler(file_handler)
+    
+    # File handler for RA bill fetching only
+    ra_bill_handler = logging.handlers.RotatingFileHandler(
+        os.path.join(logs_dir, 'ra_bill_fetch.log'),
+        maxBytes=5242880,  # 5MB
+        backupCount=5
+    )
+    ra_bill_handler.setLevel(logging.DEBUG)
+    ra_bill_handler.setFormatter(detailed_formatter)
+    
+    # Only attach to relevant loggers
+    ra_loggers = ['app.routes.dashboard', 'app.routes.file_report']
+    for logger_name in ra_loggers:
+        logging.getLogger(logger_name).addHandler(ra_bill_handler)
+    
+    # Console handler for important messages
+    console_handler = logging.StreamHandler()
+    console_handler.setLevel(logging.INFO)
+    console_formatter = logging.Formatter(
+        '%(levelname)s - %(name)s - %(message)s'
+    )
+    console_handler.setFormatter(console_formatter)
+    logger.addHandler(console_handler)
+    
+    # Suppress verbose libraries
+    logging.getLogger('werkzeug').setLevel(logging.WARNING)
+    logging.getLogger('sqlalchemy.engine').setLevel(logging.WARNING)
+    
+    app.logger.info("Logging initialized successfully")
+    app.logger.info(f"Log files location: {logs_dir}")
 
 
 
-# from flask import Flask
-# from app.config import Config
+def register_blueprints(app):
+    from app.routes.auth import auth_bp
+    from app.routes.user import user_bp
+    from app.routes.dashboard import dashboard_bp
+    from app.routes.subcontractor_routes import subcontractor_bp
+    from app.routes.file_import import file_import_bp
+    from app.routes.file_report import file_report_bp
+    from app.routes.generate_comparison_report import generate_report_bp
+    from app.routes.file_format import file_format_bp 
 
-# def create_app():
-#     app = Flask(__name__)
-#     app.config.from_object(Config)
+    app.register_blueprint(auth_bp) 
+    app.register_blueprint(user_bp)
+    app.register_blueprint(dashboard_bp)
+    app.register_blueprint(subcontractor_bp)
+    app.register_blueprint(file_import_bp)
+    app.register_blueprint(file_report_bp)
+    app.register_blueprint(generate_report_bp)
+    app.register_blueprint(file_format_bp )
 
-#     # Register Blueprints
-#     from app.routes.dashboard import dashboard_bp
-#     from app.routes.file_import import file_import_bp
-#     from app.routes.user import user_bp
 
-#     app.register_blueprint(dashboard_bp)
-#     app.register_blueprint(file_import_bp)
-#     app.register_blueprint(user_bp)
+def register_error_handlers(app):
+    @app.errorhandler(404)
+    def page_not_found(e):
+        return "Page Not Found", 404
 
-#     return app
+    @app.errorhandler(500)
+    def internal_error(e):
+        return "Internal Server Error", 500

app/config.py

@@ -1,24 +1,29 @@
 import os
+# project base url
+BASE_DIR = os.path.abspath(os.path.dirname(__file__))
 
 class Config:
-    
-    SQLALCHEMY_DATABASE_URI = "mysql+pymysql://root:root@localhost/comparisondb"
+    # secret key
+    SECRET_KEY = os.getenv("SECRET_KEY", "dev-secret-key")
+ 
+    # Database varibles 
+    DB_DIALECT = os.getenv("DB_DIALECT")
+    DB_DRIVER = os.getenv("DB_DRIVER")
+    DB_USER = os.getenv("DB_USER")
+    DB_PASSWORD = os.getenv("DB_PASSWORD")
+    DB_HOST = os.getenv("DB_HOST")
+    DB_PORT = os.getenv("DB_PORT")
+    DB_NAME = os.getenv("DB_NAME")
+    # database connection url
+    SQLALCHEMY_DATABASE_URI = (
+        f"{DB_DIALECT}+{DB_DRIVER}://"
+        f"{DB_USER}:{DB_PASSWORD}@"
+        f"{DB_HOST}:{DB_PORT}/"
+        f"{DB_NAME}"
+    )
 
     SQLALCHEMY_TRACK_MODIFICATIONS = False
-    SECRET_KEY = "secret123"
-
-    UPLOAD_FOLDER = "app/static/uploads/"
+    # uploads folder path
+    UPLOAD_FOLDER = os.path.join(BASE_DIR, "static", "uploads")
+    # file extension
     ALLOWED_EXTENSIONS = {"xlsx", "xls", "csv"}
-
-
-
-# class Config:
-#     SECRET_KEY = os.getenv("SECRET_KEY", "dev_key_12345")
-
-#     UPLOAD_FOLDER = "app/static/uploads/"
-#     ALLOWED_EXTENSIONS = {"xlsx", "xls", "csv"}
-
-#     DB_HOST = "localhost"
-#     DB_USER = "root"
-#     DB_PASSWORD = "root"
-#     DB_NAME = "comparisondb"

app/models/laying_client_model.py

@@ -0,0 +1,58 @@
+from app import db
+from datetime import datetime
+
+class LayingClient(db.Model):
+    __tablename__ = "laying_client"
+
+    id = db.Column(db.Integer, primary_key=True)
+
+    # Basic Fields
+    Location = db.Column(db.String(500))
+    MH_NO = db.Column(db.String(100))
+    CC_length = db.Column(db.Float)
+    # Bedding Qty.
+    Outer_dia_of_MH_m = db.Column(db.Float)
+    Bedding_Length = db.Column(db.Float)
+    Width = db.Column(db.Float)
+    Depth = db.Column(db.Float)
+    Qty = db.Column(db.Float)
+ 
+    #  PIPE LAYING Qty. 
+    Pipe_Dia_mm = db.Column(db.Float)
+    ID_of_MH_m = db.Column(db.Float)
+    Laying_Length = db.Column(db.Float)
+
+    pipe_150_mm = db.Column(db.Float)
+    pipe_200_mm = db.Column(db.Float)
+    pipe_250_mm = db.Column(db.Float)
+    pipe_300_mm = db.Column(db.Float)
+    pipe_350_mm = db.Column(db.Float)
+    pipe_400_mm = db.Column(db.Float)
+    pipe_450_mm = db.Column(db.Float)
+    pipe_500_mm = db.Column(db.Float)
+    pipe_600_mm = db.Column(db.Float)
+    pipe_700_mm = db.Column(db.Float)
+    pipe_900_mm = db.Column(db.Float)
+    pipe_1200_mm = db.Column(db.Float)
+    
+    Total = db.Column(db.Float)
+    Remarks = db.Column(db.String(500))
+    RA_Bill_No=db.Column(db.String(500))
+    
+    created_at = db.Column(db.DateTime, default=datetime.today)
+
+
+    def __repr__(self):
+        return f"<LayingModel {self.Location}>"
+
+    def serialize(self):
+        return {c.name: getattr(self, c.name) for c in self.__table__.columns}
+
+    
+    def sum_laying_fields():
+        return [
+            "pipe_150_mm", "pipe_200_mm", "pipe_250_mm",
+            "pipe_300_mm", "pipe_350_mm", "pipe_400_mm",
+            "pipe_450_mm", "pipe_500_mm", "pipe_600_mm",
+            "pipe_700_mm", "pipe_900_mm", "pipe_1200_mm"
+        ]

app/models/laying_model.py

@@ -0,0 +1,54 @@
+from app import db
+from datetime import datetime
+
+class Laying(db.Model):
+    __tablename__ = "laying"
+
+    id = db.Column(db.Integer, primary_key=True)
+    #  Foreign Key to Subcontractor tables
+    subcontractor_id = db.Column(db.Integer, db.ForeignKey("subcontractors.id"), nullable=False)
+    # Relationship for easy access (subcontractor.subcontractor_name)
+    subcontractor = db.relationship("Subcontractor", backref="laying_records")
+
+    # Pipe Laying Fields
+    Location = db.Column(db.String(500))
+    MH_NO = db.Column(db.String(100))
+    CC_length = db.Column(db.Float)
+    Pipe_Dia_mm = db.Column(db.Float)
+    ID_of_MH_m = db.Column(db.Float)
+    Laying_Length = db.Column(db.Float)
+
+    pipe_150_mm = db.Column(db.Float)
+    pipe_200_mm = db.Column(db.Float)
+    pipe_250_mm = db.Column(db.Float)
+    pipe_300_mm = db.Column(db.Float)
+    pipe_350_mm = db.Column(db.Float)
+    pipe_400_mm = db.Column(db.Float)
+    pipe_450_mm = db.Column(db.Float)
+    pipe_500_mm = db.Column(db.Float)
+    pipe_600_mm = db.Column(db.Float)
+    pipe_700_mm = db.Column(db.Float)
+    pipe_900_mm = db.Column(db.Float)
+    pipe_1200_mm = db.Column(db.Float)
+    
+    Total = db.Column(db.Float)
+    Remarks = db.Column(db.String(500))
+    RA_Bill_No=db.Column(db.String(500))
+    
+    created_at = db.Column(db.DateTime, default=datetime.today)
+
+
+    def __repr__(self):
+        return f"<LayingModel {self.Location}>"
+
+    def serialize(self):
+        return {c.name: getattr(self, c.name) for c in self.__table__.columns}
+    
+
+    def sum_laying_fields():
+        return [
+            "pipe_150_mm", "pipe_200_mm", "pipe_250_mm",
+            "pipe_300_mm", "pipe_350_mm", "pipe_400_mm",
+            "pipe_450_mm", "pipe_500_mm", "pipe_600_mm",
+            "pipe_700_mm", "pipe_900_mm", "pipe_1200_mm"
+        ]

app/models/manhole_domestic_chamber_model.py

@@ -11,13 +11,13 @@ class ManholeDomesticChamber(db.Model):
     subcontractor = db.relationship("Subcontractor", backref="manhole_domestic_chamber_records")
 
     # Basic Fields
-    Location = db.Column(db.String(255))
-    Node_No = db.Column(db.String(100))
+    Location = db.Column(db.String(500))
+    MH_NO = db.Column(db.String(100))
     Depth_of_MH = db.Column(db.Float)
  
-    # Excavation categories
+    # Excavation categories 
     d_0_to_0_75 = db.Column(db.Float)
-    d_1_06_to_1_65 = db.Column(db.Float)
+    d_0_76_to_1_05 = db.Column(db.Float)
     d_1_06_to_1_65 = db.Column(db.Float)
     d_1_66_to_2_15 = db.Column(db.Float)
     d_2_16_to_2_65 = db.Column(db.Float)
@@ -38,8 +38,26 @@ class ManholeDomesticChamber(db.Model):
     d_9_16_to_9_65 = db.Column(db.Float)
 
     Domestic_Chambers = db.Column(db.Float)
+    DWC_Pipe_Length = db.Column(db.Float)
+    UPVC_Pipe_Length = db.Column(db.Float)
+    RA_Bill_No=db.Column(db.String(500))
 
-    created_at = db.Column(db.DateTime, default=datetime.utcnow)
+    created_at = db.Column(db.DateTime, default=datetime.today)
 
     def __repr__(self):
         return f"<HanholeDomesticChamberConstruction {self.Location}>"
+
+    def serialize(self):
+        return {c.name: getattr(self, c.name) for c in self.__table__.columns}
+    
+    
+    def sum_mh_dc_fields():
+        return [
+            "d_0_to_0_75", "d_0_76_to_1_05", "d_1_06_to_1_65",
+            "d_1_66_to_2_15", "d_2_16_to_2_65", "d_2_66_to_3_15",
+            "d_3_16_to_3_65", "d_3_66_to_4_15", "d_4_16_to_4_65",
+            "d_4_66_to_5_15", "d_5_16_to_5_65", "d_5_66_to_6_15",
+            "d_6_16_to_6_65", "d_6_66_to_7_15", "d_7_16_to_7_65",
+            "d_7_66_to_8_15", "d_8_16_to_8_65", "d_8_66_to_9_15",
+            "d_9_16_to_9_65"]
+

app/models/manhole_excavation_model.py

@@ -11,7 +11,7 @@ class ManholeExcavation(db.Model):
     subcontractor = db.relationship("Subcontractor", backref="manhole_records")
 
     # Basic Fields
-    Location = db.Column(db.String(255))
+    Location = db.Column(db.String(500))
     MH_NO = db.Column(db.String(100))
    
     Upto_IL_Depth = db.Column(db.Float)
@@ -49,15 +49,19 @@ class ManholeExcavation(db.Model):
     Soft_Rock_1_5_and_above_total = db.Column(db.Float)
 
     Hard_Rock_0_to_1_5_total = db.Column(db.Float)
-    Hard_Rock_1_5_and_above_total = db.Column(db.Float)
+    Hard_Rock_1_5_to_3_0_total = db.Column(db.Float)
     Hard_Rock_3_0_to_4_5_total = db.Column(db.Float)
     Hard_Rock_4_5_to_6_0_total = db.Column(db.Float)
     Hard_Rock_6_0_to_7_5_total = db.Column(db.Float)
 
-    Remarks = db.Column(db.String(500))
     Total = db.Column(db.Float)
+    Remarks = db.Column(db.String(500))
+    RA_Bill_No=db.Column(db.String(500))
 
-    created_at = db.Column(db.DateTime, default=datetime.utcnow)
+    created_at = db.Column(db.DateTime, default=datetime.today)
 
     def __repr__(self):
         return f"<HanholeExcavation {self.Location}>"
+
+    def serialize(self):
+        return {c.name: getattr(self, c.name) for c in self.__table__.columns}

app/models/mh_dc_client_model.py

@@ -0,0 +1,57 @@
+from app import db
+from datetime import datetime
+
+class ManholeDomesticChamberClient(db.Model):
+    __tablename__ = "mh_dc_client"
+
+    id = db.Column(db.Integer, primary_key=True)
+    #  Foreign Key to Subcontractor table
+    # subcontractor_id = db.Column(db.Integer, db.ForeignKey("subcontractors.id"), nullable=False)
+    # Relationship for easy access (subcontractor.subcontractor_name)
+    # subcontractor = db.relationship("Subcontractor", backref="mh_dc_records")
+
+    # Basic Fields
+    RA_Bill_No=db.Column(db.String(500))
+    Location = db.Column(db.String(500))
+    MH_NO = db.Column(db.String(100))
+    MH_TOP_LEVEL = db.Column(db.Float)
+    MH_IL_LEVEL = db.Column(db.Float)
+    Depth_of_MH = db.Column(db.Float)
+   
+ 
+    # Excavation categories 
+    d_0_to_1_5 = db.Column(db.Float)
+    d_1_5_to_2_0 = db.Column(db.Float)
+    d_2_0_to_2_5 = db.Column(db.Float)
+    d_2_5_to_3_0 = db.Column(db.Float)
+
+    d_3_0_to_3_5 = db.Column(db.Float)
+    d_3_5_to_4_0 = db.Column(db.Float)
+    d_4_0_to_4_5= db.Column(db.Float)
+    d_4_5_to_5_0 = db.Column(db.Float)
+
+    d_5_0_to_5_5 = db.Column(db.Float)
+    d_5_5_to_6_0 = db.Column(db.Float)
+    d_6_0_to_6_5 = db.Column(db.Float)
+
+    Domestic_Chambers = db.Column(db.Float)
+
+    created_at = db.Column(db.DateTime, default=datetime.today)
+
+    def __repr__(self):
+        return f"<HanholeDomesticChamberConstruction {self.Location}>"
+
+
+    def serialize(self):
+        return {c.name: getattr(self, c.name) for c in self.__table__.columns}
+
+    
+    def sum_mh_dc_fields():
+        return [
+            "d_0_to_0_75", "d_0_76_to_1_05", "d_1_06_to_1_65",
+            "d_1_66_to_2_15", "d_2_16_to_2_65", "d_2_66_to_3_15",
+            "d_3_16_to_3_65", "d_3_66_to_4_15", "d_4_16_to_4_65",
+            "d_4_66_to_5_15", "d_5_16_to_5_65", "d_5_66_to_6_15",
+            "d_6_16_to_6_65", "d_6_66_to_7_15", "d_7_16_to_7_65",
+            "d_7_66_to_8_15", "d_8_16_to_8_65", "d_8_66_to_9_15",
+            "d_9_16_to_9_65" ]

app/models/mh_ex_client_model.py

@@ -0,0 +1,83 @@
+from app import db
+from datetime import datetime
+
+class ManholeExcavationClient(db.Model):
+    __tablename__ = "mh_ex_client"
+
+    id = db.Column(db.Integer, primary_key=True)
+    #  Foreign Key to Subcontractor table
+    # subcontractor_id = db.Column(db.Integer, db.ForeignKey("subcontractors.id"), nullable=False)
+    # Relationship for easy access (subcontractor.subcontractor_name)
+    # subcontractor = db.relationship("Subcontractor", backref="mh_ex_records")
+
+    # Basic Fields
+    RA_Bill_No=db.Column(db.String(500))
+    Location = db.Column(db.String(500))
+    MH_NO = db.Column(db.String(100))
+    Ground_Level = db.Column(db.Float)
+    MH_Invert_Level = db.Column(db.Float)
+    MH_Top_Level = db.Column(db.Float)
+    Ex_Level = db.Column(db.Float)
+    Cutting_Depth = db.Column(db.Float)
+    MH_Depth = db.Column(db.Float)
+    ID_of_MH_m = db.Column(db.Float)
+
+    Dia_of_MH_Cutting = db.Column(db.Float)
+    Area_of_Manhole = db.Column(db.Float)
+   
+    # Excavation categories
+    Marshi_Muddy_Slushy_0_to_1_5 = db.Column(db.Float)
+    Marshi_Muddy_Slushy_1_5_to_3_0 = db.Column(db.Float)
+    Marshi_Muddy_Slushy_3_0_to_4_5 = db.Column(db.Float)
+
+    Soft_Murum_0_to_1_5 = db.Column(db.Float)
+    Soft_Murum_1_5_to_3_0 = db.Column(db.Float)
+    Soft_Murum_3_0_to_4_5 = db.Column(db.Float)
+
+    Hard_Murum_0_to_1_5 = db.Column(db.Float)
+    Hard_Murum_1_5_to_3_0 = db.Column(db.Float)
+    Hard_Murum_3_0_to_4_5 = db.Column(db.Float)
+
+    Soft_Rock_0_to_1_5 = db.Column(db.Float)
+    Soft_Rock_1_5_to_3_0 = db.Column(db.Float)
+    Soft_Murum_3_0_to_4_5 = db.Column(db.Float)
+
+    Hard_Rock_0_to_1_5 = db.Column(db.Float)
+    Hard_Rock_1_5_to_3_0 = db.Column(db.Float)
+    Hard_Rock_3_0_to_4_5 = db.Column(db.Float)
+    Hard_Rock_4_5_to_6_0 = db.Column(db.Float)
+    Hard_Rock_6_0_to_7_5 = db.Column(db.Float)
+
+    # Totals
+    Marshi_Muddy_Slushy_0_to_1_5_total = db.Column(db.Float)
+    Marshi_Muddy_Slushy_1_5_to_3_0_total = db.Column(db.Float)
+    Marshi_Muddy_Slushy_3_0_to_4_5_total = db.Column(db.Float)
+
+    Soft_Murum_0_to_1_5_total = db.Column(db.Float)
+    Soft_Murum_1_5_to_3_0_total = db.Column(db.Float)
+    Soft_Murum_3_0_to_4_5_total = db.Column(db.Float)
+
+    Hard_Murum_0_to_1_5_total = db.Column(db.Float)
+    Hard_Murum_1_5_to_3_0_total = db.Column(db.Float)
+    Hard_Murum_3_0_to_4_5_total = db.Column(db.Float)
+
+    Soft_Rock_0_to_1_5_total = db.Column(db.Float)
+    Soft_Rock_1_5_to_3_0_total = db.Column(db.Float)
+    Soft_Rock_3_0_to_4_5_total = db.Column(db.Float)
+
+    Hard_Rock_0_to_1_5_total = db.Column(db.Float)
+    Hard_Rock_1_5_to_3_0_total = db.Column(db.Float)
+    Hard_Rock_3_0_to_4_5_total = db.Column(db.Float)
+    Hard_Rock_4_5_to_6_0_total = db.Column(db.Float)
+    Hard_Rock_6_0_to_7_5_total = db.Column(db.Float)
+
+    Remarks = db.Column(db.String(500))
+    Total = db.Column(db.Float)
+
+    created_at = db.Column(db.DateTime, default=datetime.today)
+
+    def __repr__(self):
+        return f"<HanholeExcavation {self.Location}>"
+
+    def serialize(self):
+        return {c.name: getattr(self, c.name) for c in self.__table__.columns}

app/models/subcontractor_model.py

@@ -14,7 +14,7 @@ class Subcontractor(db.Model):
     email_id = db.Column(db.String(150))
     contact_person = db.Column(db.String(150))
     status = db.Column(db.String(20), default="Active")
-    created_at = db.Column(db.DateTime, default=datetime.utcnow)
+    created_at = db.Column(db.DateTime, default=datetime.today)
 
     def __repr__(self):
         return f"<Subcontractor {self.subcontractor_name}>"

app/models/tr_ex_client_model.py

@@ -0,0 +1,88 @@
+from app import db
+from datetime import datetime
+
+class TrenchExcavationClient(db.Model):
+    __tablename__ = "tr_ex_client"
+
+    id = db.Column(db.Integer, primary_key=True)
+    #  Foreign Key to Subcontractor table
+    # subcontractor_id = db.Column(db.Integer, db.ForeignKey("subcontractors.id"), nullable=False)
+    # Relationship for easy access (subcontractor.subcontractor_name)
+    # subcontractor = db.relationship("Subcontractor", backref="tr_ex_records")
+
+    # Basic Fields
+    RA_Bill_No=db.Column(db.String(500))
+    Location = db.Column(db.String(500))
+    MH_NO = db.Column(db.String(100))
+    CC_length = db.Column(db.Float)
+    Actual_Trench_Length = db.Column(db.Float)
+    Ground_Level = db.Column(db.Float)
+    Invert_Level = db.Column(db.Float)
+    Excavated_level = db.Column(db.Float)
+    Cutting_Depth = db.Column(db.Float)
+    Avg_Depth = db.Column(db.Float)
+    Pipe_Dia_mm = db.Column(db.Float)
+ 
+    # width
+    Width_0_to_1_5_m = db.Column(db.Float)
+    Width_1_5_to_3_0_m = db.Column(db.Float)
+    Width_3_0_to_4_5_m = db.Column(db.Float)
+    Width_4_5_to_6_0_m = db.Column(db.Float)
+    Width_6_0_to_7_5_m = db.Column(db.Float)
+
+    # Excavation categories
+    Marshi_Muddy_Slushy_0_to_1_5 = db.Column(db.Float)
+    Marshi_Muddy_Slushy_1_5_to_3_0 = db.Column(db.Float)
+    Marshi_Muddy_Slushy_3_0_to_4_5 = db.Column(db.Float)
+
+    Soft_Murum_0_to_1_5 = db.Column(db.Float)
+    Soft_Murum_1_5_to_3_0 = db.Column(db.Float)
+    Soft_Murum_3_0_to_4_5 = db.Column(db.Float)
+
+    Hard_Murum_0_to_1_5 = db.Column(db.Float)
+    Hard_Murum_1_5_to_3_0 = db.Column(db.Float)
+    Hard_Murum_3_0_to_4_5 = db.Column(db.Float)
+
+    Soft_Rock_0_to_1_5 = db.Column(db.Float)
+    Soft_Rock_1_5_to_3_0 = db.Column(db.Float)
+    Soft_Rock_3_0_to_4_5 = db.Column(db.Float)
+
+    Hard_Rock_0_to_1_5 = db.Column(db.Float)
+    Hard_Rock_1_5_to_3_0 = db.Column(db.Float)
+    Hard_Rock_3_0_to_4_5 = db.Column(db.Float)
+    Hard_Rock_4_5_to_6_0 = db.Column(db.Float)
+    Hard_Rock_6_0_to_7_5 = db.Column(db.Float)
+
+    # Totals
+    Marshi_Muddy_Slushy_0_to_1_5_total = db.Column(db.Float)
+    Marshi_Muddy_Slushy_1_5_to_3_0_total = db.Column(db.Float)
+    Marshi_Muddy_Slushy_3_0_to_4_5_total = db.Column(db.Float)
+
+    Soft_Murum_0_to_1_5_total = db.Column(db.Float)
+    Soft_Murum_1_5_to_3_0_total = db.Column(db.Float)
+    Soft_Murum_3_0_to_4_5_total = db.Column(db.Float)
+
+    Hard_Murum_0_to_1_5_total = db.Column(db.Float)
+    Hard_Murum_1_5_to_3_0_total = db.Column(db.Float)
+    Hard_Murum_3_0_to_4_5_total = db.Column(db.Float)
+
+    Soft_Rock_0_to_1_5_total = db.Column(db.Float)
+    Soft_Rock_1_5_to_3_0_total = db.Column(db.Float)
+    Soft_Rock_3_0_to_4_5_total = db.Column(db.Float)
+
+    Hard_Rock_0_to_1_5_total = db.Column(db.Float)
+    Hard_Rock_1_5_to_3_0_total = db.Column(db.Float)
+    Hard_Rock_3_0_to_4_5_total = db.Column(db.Float)
+    Hard_Rock_4_5_to_6_0_total = db.Column(db.Float)
+    Hard_Rock_6_0_to_7_5_total = db.Column(db.Float)
+
+    Total = db.Column(db.Float)
+    Remarks = db.Column(db.String(500))
+
+    created_at = db.Column(db.DateTime, default=datetime.today)
+
+    def __repr__(self):
+        return f"<TrenchExcavation {self.Location}>"
+    
+    def serialize(self):
+            return {c.name: getattr(self, c.name) for c in self.__table__.columns}

app/models/trench_excavation_model.py

@@ -11,7 +11,7 @@ class TrenchExcavation(db.Model):
     subcontractor = db.relationship("Subcontractor", backref="trench_records")
 
     # Basic Fields
-    Location = db.Column(db.String(255))
+    Location = db.Column(db.String(500))
     MH_NO = db.Column(db.String(100))
     CC_length = db.Column(db.Float)
     Invert_Level = db.Column(db.Float)
@@ -21,6 +21,7 @@ class TrenchExcavation(db.Model):
     Actual_Trench_Length = db.Column(db.Float)
     Pipe_Dia_mm = db.Column(db.Float)
 
+    # width
     Width_0_to_2_5 = db.Column(db.Float)
     Width_2_5_to_3_0 = db.Column(db.Float)
     Width_3_0_to_4_5 = db.Column(db.Float)
@@ -59,15 +60,51 @@ class TrenchExcavation(db.Model):
     Soft_Rock_1_5_and_above_total = db.Column(db.Float)
 
     Hard_Rock_0_to_1_5_total = db.Column(db.Float)
-    Hard_Rock_1_5_and_above_total = db.Column(db.Float)
+    Hard_Rock_1_5_to_3_0_total = db.Column(db.Float)
     Hard_Rock_3_0_to_4_5_total = db.Column(db.Float)
     Hard_Rock_4_5_to_6_0_total = db.Column(db.Float)
     Hard_Rock_6_0_to_7_5_total = db.Column(db.Float)
 
-    Remarks = db.Column(db.String(500))
     Total = db.Column(db.Float)
+    Remarks = db.Column(db.String(500))
+    RA_Bill_No=db.Column(db.String(500))
+    
+    created_at = db.Column(db.DateTime, default=datetime.today)
 
-    created_at = db.Column(db.DateTime, default=datetime.utcnow)
 
     def __repr__(self):
         return f"<TrenchExcavation {self.Location}>"
+
+    def serialize(self):
+        return {c.name: getattr(self, c.name) for c in self.__table__.columns}
+    
+    def excavation_category_sums(self):
+
+        def safe(val):
+            return val or 0
+
+        return {
+            "Soft_Murum_Total": (
+                safe(self.Soft_Murum_0_to_1_5)
+                + safe(self.Soft_Murum_1_5_to_3_0)
+                + safe(self.Soft_Murum_3_0_to_4_5)
+            ),
+
+            "Hard_Murum_Total": (
+                safe(self.Hard_Murum_0_to_1_5)
+                + safe(self.Hard_Murum_1_5_to_3_0)
+            ),
+
+            "Soft_Rock_Total": (
+                safe(self.Soft_Rock_0_to_1_5)
+                + safe(self.Soft_Rock_1_5_to_3_0)
+            ),
+
+            "Hard_Rock_Total": (
+                safe(self.Hard_Rock_0_to_1_5)
+                + safe(self.Hard_Rock_1_5_to_3_0)
+                + safe(self.Hard_Rock_3_0_to_4_5)
+                + safe(self.Hard_Rock_4_5_to_6_0)
+                + safe(self.Hard_Rock_6_0_to_7_5)
+            ),
+        }

app/models/user_model.py

@@ -1,21 +1,16 @@
-# from app.services.db_service import db
-# from werkzeug.security import generate_password_hash, check_password_hash
+from app.services.db_service import db
+from werkzeug.security import generate_password_hash, check_password_hash
 
-# class User(db.Model):
-#     id = db.Column(db.Integer, primary_key=True)
-#     name = db.Column(db.String(120))
-#     email = db.Column(db.String(120), unique=True)
-#     password_hash = db.Column(db.String(255))
+class User(db.Model):
+    __tablename__ = "users"
 
-#     def set_password(self, password):
-#         self.password_hash = generate_password_hash(password)
+    id = db.Column(db.Integer, primary_key=True)
+    name = db.Column(db.String(120), nullable=False)
+    email = db.Column(db.String(120), unique=True, nullable=False)
+    password_hash = db.Column(db.String(255), nullable=False)
 
-#     def check_password(self, password):
-#         return check_password_hash(self.password_hash, password)
+    def set_password(self, password):
+        self.password_hash = generate_password_hash(password)
 
-
-class User:
-    def __init__(self, id, name, email):
-        self.id = id
-        self.name = name
-        self.email = email
+    def check_password(self, password):
+        return check_password_hash(self.password_hash, password)

app/routes/auth.py

@@ -1,31 +1,48 @@
-from flask import Blueprint, request, render_template, redirect, flash, session
-from app.services.user_service import UserService  
+from flask import Blueprint, render_template, request, redirect, url_for, flash, session
+from app.services.user_service import UserService
 
-auth_bp = Blueprint("auth", __name__, url_prefix="/auth")
+auth_bp = Blueprint("auth", __name__)
 
-# LOGIN PAGE
 @auth_bp.route("/login", methods=["GET", "POST"])
 def login():
+    if session.get("user_id"):
+        return redirect(url_for("dashboard.dashboard"))
+
     if request.method == "POST":
-        user = UserService.validate_login(
-            request.form["email"],
-            request.form["password"]
-        )
+        email = request.form.get("email")
+        password = request.form.get("password")
+
+        user = UserService.validate_login(email, password)
         if user:
             session["user_id"] = user.id
-            return redirect("/dashboard")
-        flash("Invalid credentials", "danger")
-    return render_template("login.html")
+            session["user_name"] = user.name
+            flash("Login successful", "success")
+            return redirect(url_for("dashboard.dashboard"))
+
+        flash("Invalid email or password", "danger")
+
+    return render_template("login.html", title="Login")
 
-# REGISTER API ONLY
-@auth_bp.route("/register", methods=["POST"])
-def register():
-    data = request.json
-    UserService.register_user(data["name"], data["email"], data["password"])
-    return {"message": "User registered successfully"}, 201
 
-# LOGOUT
 @auth_bp.route("/logout")
 def logout():
     session.clear()
-    return redirect("/auth/login")
+    flash("Logged out successfully", "info")
+    return redirect(url_for("auth.login"))
+
+@auth_bp.route("/register", methods=["GET", "POST"])
+def register():
+    if request.method == "POST":
+        name = request.form.get("name")
+        email = request.form.get("email")
+        password = request.form.get("password")
+
+        user = UserService.register_user(name, email, password)
+        if not user:
+            flash("Email already exists", "danger")
+            return redirect(url_for("auth.register"))
+
+        flash("User registered successfully", "success")
+        return redirect(url_for("auth.login"))
+
+    return render_template("register.html", title="Register")

app/routes/dashboard.py

@@ -1,8 +1,308 @@
-from flask import Blueprint, render_template
+import logging
+from flask import Blueprint, render_template, session, redirect, url_for, jsonify, request
+from sqlalchemy import func, union_all
+from app import db
+from app.models.trench_excavation_model import TrenchExcavation
+from app.models.tr_ex_client_model import TrenchExcavationClient
+from app.models.manhole_excavation_model import ManholeExcavation
+from app.models.mh_ex_client_model import ManholeExcavationClient
+from app.models.laying_model import Laying
+from app.models.laying_client_model import LayingClient
+from app.models.subcontractor_model import Subcontractor
 
-dashboard_bp = Blueprint("dashboard", __name__)
+dashboard_bp = Blueprint("dashboard", __name__, url_prefix="/dashboard")
+
+# Configure logging for debugging
+logger = logging.getLogger(__name__)
+
+# API to get dynamic filters from database
+@dashboard_bp.route("/api/filters")
+def get_filters():
+    try:
+        logger.info("=" * 60)
+        logger.info("FETCHING RA BILLS - START")
+        logger.info("=" * 60)
+        
+        # 1. Fetch Subcontractors (Linked to Trench entries)
+        logger.info("Step 1: Fetching Subcontractors...")
+        subcontractors = db.session.query(Subcontractor.subcontractor_name)\
+            .join(TrenchExcavation, Subcontractor.id == TrenchExcavation.subcontractor_id)\
+            .distinct().all()
+        logger.info(f"✓ Subcontractors found: {len(subcontractors)}")
+        logger.debug(f"  Subcontractor list: {[s[0] for s in subcontractors if s[0]]}")
+        
+        # 2. Check total records in TrenchExcavation table
+        logger.info("Step 2: Checking TrenchExcavation table...")
+        total_records = db.session.query(TrenchExcavation).count()
+        logger.info(f"✓ Total TrenchExcavation records: {total_records}")
+        
+        # 3. Check records with RA_Bill_No
+        logger.info("Step 3: Checking records with RA_Bill_No...")
+        records_with_ra = db.session.query(TrenchExcavation).filter(TrenchExcavation.RA_Bill_No != None).count()
+        logger.info(f"✓ Records with RA_Bill_No (not null): {records_with_ra}")
+        
+        # 4. Check for empty strings
+        records_with_ra_and_value = db.session.query(TrenchExcavation).filter(
+            TrenchExcavation.RA_Bill_No != None,
+            TrenchExcavation.RA_Bill_No != ""
+        ).count()
+        logger.info(f"✓ Records with RA_Bill_No (not null & not empty): {records_with_ra_and_value}")
+        
+        # 5. Raw sample of RA_Bill_No values
+        logger.info("Step 4: Sampling RA_Bill_No values from database...")
+        sample_bills = db.session.query(TrenchExcavation.RA_Bill_No).limit(10).all()
+        logger.debug(f"  Sample RA_Bill_No values (Subcontractor): {[str(r[0]) for r in sample_bills]}")
+        
+        sample_bills_client = db.session.query(TrenchExcavationClient.RA_Bill_No).limit(10).all()
+        logger.debug(f"  Sample RA_Bill_No values (Client): {[str(r[0]) for r in sample_bills_client]}")
+        
+        # 6. Fetch RA Bills from BOTH Subcontractor and Client tables
+        logger.info("Step 5: Fetching distinct RA Bills from both Subcontractor and Client data...")
+        
+        # Get RA bills from Subcontractor data
+        subcon_ra_bills = db.session.query(TrenchExcavation.RA_Bill_No)\
+            .filter(TrenchExcavation.RA_Bill_No != None)\
+            .filter(TrenchExcavation.RA_Bill_No != "")\
+            .distinct()
+        
+        logger.debug(f"  Subcontractor RA Bills (before union): {len(subcon_ra_bills.all())}")
+        
+        # Get RA bills from Client data
+        client_ra_bills = db.session.query(TrenchExcavationClient.RA_Bill_No)\
+            .filter(TrenchExcavationClient.RA_Bill_No != None)\
+            .filter(TrenchExcavationClient.RA_Bill_No != "")\
+            .distinct()
+        
+        logger.debug(f"  Client RA Bills (before union): {len(client_ra_bills.all())}")
+        
+        # Union both queries to get all unique RA bills
+        ra_bills_union = db.session.query(TrenchExcavation.RA_Bill_No)\
+            .filter(TrenchExcavation.RA_Bill_No != None)\
+            .filter(TrenchExcavation.RA_Bill_No != "")\
+            .union(
+                db.session.query(TrenchExcavationClient.RA_Bill_No)\
+                .filter(TrenchExcavationClient.RA_Bill_No != None)\
+                .filter(TrenchExcavationClient.RA_Bill_No != "")
+            ).order_by(TrenchExcavation.RA_Bill_No).all()
+        
+        logger.info(f"✓ Distinct RA Bills found (Combined): {len(ra_bills_union)}")
+        ra_bills_list = [r[0] for r in ra_bills_union if r[0]]
+        logger.info(f"  RA Bills list: {ra_bills_list}")
+        
+        # 7. Debug: Check data types
+        if ra_bills_union:
+            logger.debug(f"  First RA Bill value: {ra_bills_union[0][0]}")
+            logger.debug(f"  First RA Bill type: {type(ra_bills_union[0][0])}")
+
+        response = {
+            "subcontractors": [s[0] for s in subcontractors if s[0]],
+            "ra_bills": ra_bills_list
+        }
+        
+        logger.info(f"✓ Response prepared successfully")
+        logger.info("=" * 60)
+        
+        return jsonify(response)
+        
+    except Exception as e:
+        logger.error("=" * 60)
+        logger.error(f"ERROR in get_filters(): {str(e)}")
+        logger.error(f"Error type: {type(e).__name__}")
+        logger.exception("Full traceback:")
+        logger.error("=" * 60)
+        return jsonify({"error": str(e)}), 500
+
+# API for the live abstract data - handles multiple table types
+@dashboard_bp.route("/api/excavation-abstract")
+def excavation_abstract():
+    try:
+        logger.info("=" * 60)
+        logger.info("EXCAVATION ABSTRACT FETCH - START")
+        logger.info("=" * 60)
+        
+        table_type = request.args.get('table_type', 'trench')
+        subcon_name = request.args.get('subcontractor', 'All')
+        ra_bill = request.args.get('ra_bill', 'Cumulative')
+        
+        logger.info(f"Request Parameters:")
+        logger.info(f"  Table Type: {table_type}")
+        logger.info(f"  Subcontractor: {subcon_name}")
+        logger.info(f"  RA Bill: {ra_bill}")
+        
+        # Select models and match keys based on table type
+        if table_type == 'trench':
+            SubconModel = TrenchExcavation
+            ClientModel = TrenchExcavationClient
+            table_label = "Trench Excavation"
+            location_key = 'Location'
+            mh_key = 'MH_NO'
+            excavation_columns = [
+                ("Soft Murum", "0-1.5m", "Soft_Murum_0_to_1_5"),
+                ("Soft Murum", "1.5-3.0m", "Soft_Murum_1_5_to_3_0"),
+                ("Soft Murum", "3.0-4.5m", "Soft_Murum_3_0_to_4_5"),
+                ("Hard Murum", "0-1.5m", "Hard_Murum_0_to_1_5"),
+                ("Hard Murum", "1.5-3.0m", "Hard_Murum_1_5_to_3_0"),
+                ("Soft Rock", "0-1.5m", "Soft_Rock_0_to_1_5"),
+                ("Soft Rock", "1.5-3.0m", "Soft_Rock_1_5_to_3_0"),
+                ("Hard Rock", "0-1.5m", "Hard_Rock_0_to_1_5"),
+                ("Hard Rock", "1.5-3.0m", "Hard_Rock_1_5_to_3_0"),
+                ("Hard Rock", "3.0-4.5m", "Hard_Rock_3_0_to_4_5"),
+                ("Hard Rock", "4.5-6.0m", "Hard_Rock_4_5_to_6_0"),
+                ("Hard Rock", "6.0-7.5m", "Hard_Rock_6_0_to_7_5"),
+            ]
+        elif table_type == 'manhole':
+            SubconModel = ManholeExcavation
+            ClientModel = ManholeExcavationClient
+            table_label = "Manhole Excavation"
+            location_key = 'Location'
+            mh_key = 'MH_NO'
+            excavation_columns = [
+                ("Soft Murum", "0-1.5m", "Soft_Murum_0_to_1_5"),
+                ("Soft Murum", "1.5-3.0m", "Soft_Murum_1_5_to_3_0"),
+                ("Hard Murum", "0-1.5m", "Hard_Murum_0_to_1_5"),
+                ("Hard Murum", "1.5-3.0m", "Hard_Murum_1_5_to_3_0"),
+                ("Soft Rock", "0-1.5m", "Soft_Rock_0_to_1_5"),
+                ("Soft Rock", "1.5-3.0m", "Soft_Rock_1_5_to_3_0"),
+                ("Hard Rock", "0-1.5m", "Hard_Rock_0_to_1_5"),
+                ("Hard Rock", "1.5-3.0m", "Hard_Rock_1_5_to_3_0"),
+            ]
+        elif table_type == 'laying':
+            SubconModel = Laying
+            ClientModel = LayingClient
+            table_label = "Laying"
+            location_key = 'Location'
+            mh_key = 'MH_NO'
+            excavation_columns = [
+                ("Soft Murum", "0-1.5m", "Soft_Murum_0_to_1_5"),
+                ("Soft Murum", "1.5-3.0m", "Soft_Murum_1_5_to_3_0"),
+                ("Hard Murum", "0-1.5m", "Hard_Murum_0_to_1_5"),
+                ("Hard Murum", "1.5-3.0m", "Hard_Murum_1_5_to_3_0"),
+                ("Soft Rock", "0-1.5m", "Soft_Rock_0_to_1_5"),
+                ("Soft Rock", "1.5-3.0m", "Soft_Rock_1_5_to_3_0"),
+                ("Hard Rock", "0-1.5m", "Hard_Rock_0_to_1_5"),
+                ("Hard Rock", "1.5-3.0m", "Hard_Rock_1_5_to_3_0"),
+            ]
+        else:
+            return jsonify({"error": f"Invalid table_type: {table_type}"}), 400
+        
+        logger.info(f"Using table: {table_label}")
+        
+        # ===== FETCH SUBCONTRACTOR DATA =====
+        logger.info(f"Fetching Subcontractor data ({SubconModel.__tablename__})...")
+        subcon_query = db.session.query(SubconModel)
+        
+        # Check if SubconModel has subcontractor relationship
+        if hasattr(SubconModel, 'subcontractor_id'):
+            subcon_query = subcon_query.join(
+                Subcontractor, Subcontractor.id == SubconModel.subcontractor_id
+            )
+            if subcon_name != 'All':
+                subcon_query = subcon_query.filter(Subcontractor.subcontractor_name == subcon_name)
+        
+        subcon_results = subcon_query.all()
+        logger.info(f"  Found {len(subcon_results)} subcontractor records")
+        
+        # ===== FETCH CLIENT DATA =====
+        logger.info(f"Fetching Client data ({ClientModel.__tablename__})...")
+        client_query = db.session.query(ClientModel)
+        
+        if ra_bill != 'Cumulative' and hasattr(ClientModel, 'RA_Bill_No'):
+            client_query = client_query.filter(ClientModel.RA_Bill_No == ra_bill)
+        
+        client_results = client_query.all()
+        logger.info(f"  Found {len(client_results)} client records")
+        
+        # ===== MATCH RECORDS BY MH_NO AND LOCATION =====
+        logger.info("Matching records by MH_NO and Location...")
+        matched_data = {}
+        
+        # Build a map of client records by MH_NO + Location
+        client_map = {}
+        for client_record in client_results:
+            mh_no = getattr(client_record, mh_key)
+            location = getattr(client_record, location_key)
+            key = f"{location}|{mh_no}"
+            client_map[key] = client_record
+        
+        logger.info(f"  Client map has {len(client_map)} unique MH_NO+Location combinations")
+        
+        # Match subcontractor records with client records
+        match_count = 0
+        for subcon_record in subcon_results:
+            mh_no = getattr(subcon_record, mh_key)
+            location = getattr(subcon_record, location_key)
+            key = f"{location}|{mh_no}"
+            
+            # Only process if matching client record exists
+            if key in client_map:
+                match_count += 1
+                client_record = client_map[key]
+                
+                # Aggregate excavation data for this matched pair
+                for soil, depth, col_name in excavation_columns:
+                    record_key = f"{soil}|{depth}|{location}|{mh_no}"
+                    
+                    # Get values
+                    subcon_val = 0
+                    client_val = 0
+                    
+                    if hasattr(subcon_record, col_name):
+                        subcon_val = getattr(subcon_record, col_name) or 0
+                    
+                    if hasattr(client_record, col_name):
+                        client_val = getattr(client_record, col_name) or 0
+                    
+                    # Only add if at least one has data
+                    if subcon_val > 0 or client_val > 0:
+                        if record_key not in matched_data:
+                            matched_data[record_key] = {
+                                "soil_type": soil,
+                                "depth": depth,
+                                "location": location,
+                                "mh_no": mh_no,
+                                "client_qty": 0,
+                                "subcon_qty": 0
+                            }
+                        matched_data[record_key]["client_qty"] += client_val
+                        matched_data[record_key]["subcon_qty"] += subcon_val
+        
+        logger.info(f"  Matched {match_count} subcontractor records with client records")
+        logger.info(f"  Found {len(matched_data)} excavation items with data")
+        
+        # Calculate differences and format response
+        data = []
+        for key, item in matched_data.items():
+            difference = item["subcon_qty"] - item["client_qty"]
+            # Format label as: "Soft Murum 0-1.5m (Location - MH_NO)"
+            label = f"{item['soil_type']} {item['depth']}"
+            data.append({
+                "label": label,
+                "soil_type": item["soil_type"],
+                "depth": item["depth"],
+                "location": item["location"],
+                "mh_no": item["mh_no"],
+                "client_qty": round(item["client_qty"], 2),
+                "subcon_qty": round(item["subcon_qty"], 2),
+                "difference": round(difference, 2)
+            })
+        
+        # Sort by location and mh_no for consistency
+        data.sort(key=lambda x: (x["location"], x["mh_no"], x["soil_type"], x["depth"]))
+        
+        logger.info(f"Response prepared with {len(data)} matched records")
+        logger.info("=" * 60)
+        
+        return jsonify(data)
+    except Exception as e:
+        logger.error("=" * 60)
+        logger.error(f"ERROR in excavation_abstract(): {str(e)}")
+        logger.error(f"Error type: {type(e).__name__}")
+        logger.exception("Full traceback:")
+        logger.error("=" * 60)
+        return jsonify({"error": str(e)}), 500
 
 @dashboard_bp.route("/")
-@dashboard_bp.route("/dashboard")
 def dashboard():
-    return render_template("dashboard.html", title="Dashboard")
+    if not session.get("user_id"):
+        return redirect(url_for("auth.login"))
+    return render_template("dashboard.html", title="Live Excavation Dashboard")

app/routes/file_format.py

@@ -0,0 +1,30 @@
+from flask import Blueprint, render_template, send_from_directory, abort, current_app
+from app.utils.helpers import login_required
+import os
+
+file_format_bp  = Blueprint("file_format", __name__)
+
+@file_format_bp .route("/file_format")
+@login_required
+def download_format():
+    return render_template("file_format.html", title="Download File Formats")
+
+
+@file_format_bp .route("/file_format/download/<filename>")
+@login_required
+def download_excel_format(filename):
+
+    download_folder = os.path.join(
+        current_app.root_path, "static", "downloads/format"
+    )
+
+    file_path = os.path.join(download_folder, filename)
+
+    if not os.path.exists(file_path):
+        abort(404)
+
+    return send_from_directory(
+        directory=download_folder,
+        path=filename,
+        as_attachment=True
+    )

app/routes/file_import.py

@@ -1,21 +1,46 @@
 from flask import Blueprint, render_template, request, flash
 from app.services.file_service import FileService
 from app.models.subcontractor_model import Subcontractor
+from app.utils.helpers import login_required
 
 file_import_bp = Blueprint("file_import", __name__, url_prefix="/file")
 
-@file_import_bp.route("/import", methods=["GET", "POST"])
+# this is contractractor immport routes
+@file_import_bp.route("/import_Subcontractor", methods=["GET", "POST"])
+@login_required
 def import_file():
     subcontractors = Subcontractor.query.all()
 
     if request.method == "POST":
         file = request.files.get("file")
         subcontractor_id = request.form.get("subcontractor_id")
-        file_type = request.form.get("file_type")
+        RA_Bill_No = request.form.get("RA_Bill_No")
 
         service = FileService()
-        success, msg = service.handle_file_upload(file, subcontractor_id, file_type)
+        success, msg = service.handle_file_upload(file, subcontractor_id, RA_Bill_No)
 
         flash(msg, "success" if success else "danger")
 
-    return render_template("file_import.html", title="File Import", subcontractors=subcontractors)
+    return render_template(
+        "file_import_subcontractor.html",
+        title="Sub-cont. File Import",
+        subcontractors=subcontractors
+    )
+
+
+# this is client import routes
+@file_import_bp.route("/import_client", methods=["GET", "POST"])
+@login_required
+def client_import_file():
+    subcontractors = Subcontractor.query.all()
+
+    if request.method == "POST":
+        file = request.files.get("file")
+        RA_Bill_No = request.form.get("RA_Bill_No")
+
+        service = FileService()
+        success, msg = service.handle_client_file_upload(file, RA_Bill_No)
+
+        flash(msg, "success" if success else "danger")
+
+    return render_template("file_import_client.html", title="Client File Import", subcontractors=subcontractors)

app/routes/file_report.py

@@ -0,0 +1,297 @@
+import pandas as pd
+import io
+import logging
+from flask import Blueprint, render_template, request, send_file, flash
+from app.utils.helpers import login_required
+
+from app.models.subcontractor_model import Subcontractor
+
+from app.models.manhole_excavation_model import ManholeExcavation
+from app.models.trench_excavation_model import TrenchExcavation
+from app.models.manhole_domestic_chamber_model import ManholeDomesticChamber
+from app.models.laying_model import Laying
+
+from app.models.mh_ex_client_model import ManholeExcavationClient
+from app.models.tr_ex_client_model import TrenchExcavationClient
+from app.models.mh_dc_client_model import ManholeDomesticChamberClient
+from app.models.laying_client_model import LayingClient
+
+ 
+# --- BLUEPRINT DEFINITION ---
+file_report_bp = Blueprint("file_report", __name__, url_prefix="/file")
+
+# Configure logging for debugging
+logger = logging.getLogger(__name__)
+ 
+# --- Client class ---
+class ClientBill:
+    def __init__(self):
+        self.df_tr = pd.DataFrame()
+        self.df_mh = pd.DataFrame()
+        self.df_dc = pd.DataFrame()
+        self.df_laying = pd.DataFrame()
+ 
+    def Fetch(self, RA_Bill_No):
+        logger.info("=" * 60)
+        logger.info("ClientBill.Fetch() - START")
+        logger.info("=" * 60)
+        logger.info(f"Fetching data for RA_Bill_No: '{RA_Bill_No}'")
+        logger.debug(f"  Type of RA_Bill_No: {type(RA_Bill_No)}")
+        
+        try:
+            logger.info("Step 1: Fetching TrenchExcavationClient records...")
+            trench = TrenchExcavationClient.query.filter_by(RA_Bill_No=RA_Bill_No).all()
+            logger.info(f"✓ Trench records found: {len(trench)}")
+            
+            logger.info("Step 2: Fetching ManholeExcavationClient records...")
+            mh = ManholeExcavationClient.query.filter_by(RA_Bill_No=RA_Bill_No).all()
+            logger.info(f"✓ Manhole records found: {len(mh)}")
+            
+            logger.info("Step 3: Fetching ManholeDomesticChamberClient records...")
+            dc = ManholeDomesticChamberClient.query.filter_by(RA_Bill_No=RA_Bill_No).all()
+            logger.info(f"✓ Domestic Chamber records found: {len(dc)}")
+            
+            logger.info("Step 4: Fetching LayingClient records...")
+            lay = LayingClient.query.filter_by(RA_Bill_No=RA_Bill_No).all()
+            logger.info(f"✓ Laying records found: {len(lay)}")
+            
+            logger.info("Step 5: Converting to DataFrames...")
+            self.df_tr = pd.DataFrame([c.serialize() for c in trench])
+            self.df_mh = pd.DataFrame([c.serialize() for c in mh])
+            self.df_dc = pd.DataFrame([c.serialize() for c in dc])
+            self.df_laying = pd.DataFrame([c.serialize() for c in lay])
+            
+            logger.debug(f"  Trench DF shape: {self.df_tr.shape}")
+            logger.debug(f"  Manhole DF shape: {self.df_mh.shape}")
+            logger.debug(f"  Domestic Chamber DF shape: {self.df_dc.shape}")
+            logger.debug(f"  Laying DF shape: {self.df_laying.shape}")
+            
+            logger.info("Step 6: Cleaning DataFrames...")
+            drop_cols = ["id", "created_at", "_sa_instance_state"]
+            for df in [self.df_tr, self.df_mh, self.df_dc, self.df_laying]:
+                if not df.empty:
+                    df.drop(columns=drop_cols, errors="ignore", inplace=True)
+                    logger.debug(f"  Cleaned DF with shape: {df.shape}")
+            
+            logger.info("✓ ClientBill.Fetch() completed successfully")
+            logger.info("=" * 60)
+            
+        except Exception as e:
+            logger.error("=" * 60)
+            logger.error(f"ERROR in ClientBill.Fetch(): {str(e)}")
+            logger.error(f"Error type: {type(e).__name__}")
+            logger.exception("Full traceback:")
+            logger.error("=" * 60)
+            raise
+ 
+# --- Subcontractor class ---
+class SubcontractorBill:
+    def __init__(self):
+        self.df_tr = pd.DataFrame()
+        self.df_mh = pd.DataFrame()
+        self.df_dc = pd.DataFrame()
+        self.df_laying = pd.DataFrame()
+ 
+    def Fetch(self, RA_Bill_No=None, subcontractor_id=None):
+        logger.info("=" * 60)
+        logger.info("SubcontractorBill.Fetch() - START")
+        logger.info("=" * 60)
+        logger.info(f"Parameters:")
+        logger.info(f"  RA_Bill_No: '{RA_Bill_No}' (type: {type(RA_Bill_No)})")
+        logger.info(f"  subcontractor_id: '{subcontractor_id}' (type: {type(subcontractor_id)})")
+        
+        try:
+            filters = {}
+            if subcontractor_id:
+                filters["subcontractor_id"] = subcontractor_id
+                logger.debug(f"  Added filter - subcontractor_id: {subcontractor_id}")
+            if RA_Bill_No:
+                filters["RA_Bill_No"] = RA_Bill_No
+                logger.debug(f"  Added filter - RA_Bill_No: {RA_Bill_No}")
+            
+            logger.info(f"Applied filters: {filters}")
+            
+            logger.info("Step 1: Fetching TrenchExcavation records...")
+            trench = TrenchExcavation.query.filter_by(**filters).all()
+            logger.info(f"✓ Trench records found: {len(trench)}")
+            
+            logger.info("Step 2: Fetching ManholeExcavation records...")
+            mh = ManholeExcavation.query.filter_by(**filters).all()
+            logger.info(f"✓ Manhole records found: {len(mh)}")
+            
+            logger.info("Step 3: Fetching ManholeDomesticChamber records...")
+            dc = ManholeDomesticChamber.query.filter_by(**filters).all()
+            logger.info(f"✓ Domestic Chamber records found: {len(dc)}")
+            
+            logger.info("Step 4: Fetching Laying records...")
+            lay = Laying.query.filter_by(**filters).all()
+            logger.info(f"✓ Laying records found: {len(lay)}")
+            
+            logger.info("Step 5: Converting to DataFrames...")
+            self.df_tr = pd.DataFrame([c.serialize() for c in trench])
+            self.df_mh = pd.DataFrame([c.serialize() for c in mh])
+            self.df_dc = pd.DataFrame([c.serialize() for c in dc])
+            self.df_laying = pd.DataFrame([c.serialize() for c in lay])
+            
+            logger.debug(f"  Trench DF shape: {self.df_tr.shape}")
+            logger.debug(f"  Manhole DF shape: {self.df_mh.shape}")
+            logger.debug(f"  Domestic Chamber DF shape: {self.df_dc.shape}")
+            logger.debug(f"  Laying DF shape: {self.df_laying.shape}")
+            
+            logger.info("Step 6: Cleaning DataFrames...")
+            drop_cols = ["id", "created_at", "_sa_instance_state"]
+            for df in [self.df_tr, self.df_mh, self.df_dc, self.df_laying]:
+                if not df.empty:
+                    df.drop(columns=drop_cols, errors="ignore", inplace=True)
+                    logger.debug(f"  Cleaned DF with shape: {df.shape}")
+            
+            logger.info("✓ SubcontractorBill.Fetch() completed successfully")
+            logger.info("=" * 60)
+            
+        except Exception as e:
+            logger.error("=" * 60)
+            logger.error(f"ERROR in SubcontractorBill.Fetch(): {str(e)}")
+            logger.error(f"Error type: {type(e).__name__}")
+            logger.exception("Full traceback:")
+            logger.error("=" * 60)
+            raise
+
+
+# --- subcontractor report only  ---
+@file_report_bp.route("/Subcontractor_report", methods=["GET", "POST"])
+@login_required
+def report_file():
+    subcontractors = Subcontractor.query.all()
+    tables = None 
+    selected_sc_id = None
+    ra_bill_no = None
+    download_all = False
+ 
+    if request.method == "POST":
+        subcontractor_id = request.form.get("subcontractor_id")
+        ra_bill_no = request.form.get("ra_bill_no")
+        download_all = request.form.get("download_all") == "true"
+        action = request.form.get("action")
+ 
+        if not subcontractor_id:
+            flash("Please select a subcontractor.", "danger")
+            return render_template("subcontractor_report.html", subcontractors=subcontractors)
+ 
+        subcontractor = Subcontractor.query.get(subcontractor_id)
+        bill_gen = SubcontractorBill()
+ 
+        if download_all:
+            bill_gen.Fetch(subcontractor_id=subcontractor_id)
+            file_name = f"{subcontractor.subcontractor_name}_ALL_BILLS.xlsx"
+        else:
+            if not ra_bill_no:
+                flash("Please enter an RA Bill Number.", "danger")
+                return render_template("subcontractor_report.html", subcontractors=subcontractors)
+            bill_gen.Fetch(RA_Bill_No=ra_bill_no, subcontractor_id=subcontractor_id)
+            file_name = f"{subcontractor.subcontractor_name}_RA_{ra_bill_no}_Report.xlsx"
+ 
+        if bill_gen.df_tr.empty and bill_gen.df_mh.empty and bill_gen.df_dc.empty:
+            flash("No data found for this selection.", "warning")
+            return render_template("subcontractor_report.html", subcontractors=subcontractors)
+ 
+        # If download is clicked, return file immediately
+        if action == "download":
+            output = io.BytesIO()
+            with pd.ExcelWriter(output, engine="xlsxwriter") as writer:
+                bill_gen.df_tr.to_excel(writer, index=False, sheet_name="Tr.Ex.")
+                bill_gen.df_mh.to_excel(writer, index=False, sheet_name="MH.Ex.")
+                bill_gen.df_dc.to_excel(writer, index=False, sheet_name="MH & DC")
+                bill_gen.df_laying.to_excel(writer, index=False, sheet_name="Laying")
+            output.seek(0)
+            return send_file(output, download_name=file_name, as_attachment=True)
+ 
+        # We add bootstrap classes directly to the pandas output
+        table_classes = "table table-bordered table-striped table-hover table-sm mb-0"
+        tables = {
+            "tr": bill_gen.df_tr.to_html(classes=table_classes, index=False),
+            "mh": bill_gen.df_mh.to_html(classes=table_classes, index=False),
+            "dc": bill_gen.df_dc.to_html(classes=table_classes, index=False),
+            "laying": bill_gen.df_laying.to_html(classes=table_classes, index=False)
+        }
+        selected_sc_id = subcontractor_id
+ 
+    return render_template(
+        "subcontractor_report.html",
+        subcontractors=subcontractors,
+        tables=tables,
+        selected_sc_id=selected_sc_id,
+        ra_bill_no=ra_bill_no,
+        download_all=download_all
+    )
+ 
+# --- client report only --- 
+@file_report_bp.route("/client_report", methods=["GET", "POST"])
+@login_required
+def client_vs_all_subcontractor():
+    tables = {"tr": None, "mh": None, "dc": None}
+    ra_val = ""
+ 
+    if request.method == "POST":
+        RA_Bill_No = request.form.get("RA_Bill_No")
+        ra_val = RA_Bill_No
+ 
+        if not RA_Bill_No:
+            flash("Please enter RA Bill No.", "danger")
+            return render_template("client_report.html", tables=tables, ra_val=ra_val)
+ 
+        clientBill = ClientBill()
+        clientBill.Fetch(RA_Bill_No=RA_Bill_No)
+        contractorBill = SubcontractorBill()
+        contractorBill.Fetch(RA_Bill_No=RA_Bill_No)
+ 
+        # --- SAFETY CHECK: Verify data exists before merging ---
+        if clientBill.df_tr.empty and clientBill.df_mh.empty:
+            flash(f"No Client records found for RA Bill {RA_Bill_No}", "warning")
+            return render_template("client_report.html", tables=tables, ra_val=ra_val)
+ 
+        qty_cols = [...] # (Keep your existing list)
+        mh_dc_qty_cols = [...] # (Keep your existing list)
+        mh_lay_qty_cols =[...]
+
+        def aggregate_df(df, group_cols, sum_cols):
+            if df.empty:
+                # Create an empty DF with the correct columns to avoid Merge/Key Errors
+                return pd.DataFrame(columns=group_cols + sum_cols)
+            existing_cols = [c for c in sum_cols if c in df.columns]
+            # Ensure group_cols exist in the DF
+            for col in group_cols:
+                if col not in df.columns:
+                    df[col] = "N/A" # Fill missing join keys
+            return df.groupby(group_cols, as_index=False)[existing_cols].sum()
+ 
+        # Aggregate data
+        df_sub_tr_grp = aggregate_df(contractorBill.df_tr, ["Location", "MH_NO"], qty_cols)
+        df_sub_mh_grp = aggregate_df(contractorBill.df_mh, ["Location", "MH_NO"], qty_cols)
+        df_sub_dc_grp = aggregate_df(contractorBill.df_dc, ["Location", "MH_NO"], mh_dc_qty_cols)
+        df_sub_lay_grp = aggregate_df(contractorBill.df_dc, ["Location", "MH_NO"], mh_lay_qty_cols)
+ 
+        # --- FINAL MERGE LOGIC ---
+        # We check if "Location" exists in the client data. If not, we add it to prevent the KeyError.
+        for df_client in [clientBill.df_tr, clientBill.df_mh, clientBill.df_dc, clientBill.df_laying ]:
+            if not df_client.empty and "Location" not in df_client.columns:
+                df_client["Location"] = "Unknown"
+ 
+        try:
+            df_tr_cmp = clientBill.df_tr.merge(df_sub_tr_grp, on=["Location", "MH_NO"], how="left", suffixes=("_Client", "_Sub"))
+            df_mh_cmp = clientBill.df_mh.merge(df_sub_mh_grp, on=["Location", "MH_NO"], how="left", suffixes=("_Client", "_Sub"))
+            df_dc_cmp = clientBill.df_dc.merge(df_sub_dc_grp, on=["Location", "MH_NO"], how="left", suffixes=("_Client", "_Sub"))
+            df_lay_cmp = clientBill.df_laying.merge(df_sub_lay_grp, on=["Location", "MH_NO"], how="left", suffixes=("_Client", "_Sub"))
+        except KeyError as e:
+            flash(f"Merge Error: Missing column {str(e)}. Check if 'Location' is defined in your database models.", "danger")
+            return render_template("client_report.html", tables=tables, ra_val=ra_val)
+ 
+       
+        # Convert to HTML for preview
+        tables["tr"] = df_tr_cmp.to_html(classes='table table-striped table-hover table-sm', index=False)
+        tables["mh"] = df_mh_cmp.to_html(classes='table table-striped table-hover table-sm', index=False)
+        tables["dc"] = df_dc_cmp.to_html(classes='table table-striped table-hover table-sm', index=False)
+        tables["laying"] = df_lay_cmp.to_html(classes='table table-striped table-hover table-sm', index=False)
+ 
+   
+    return render_template("client_report.html", tables=tables, ra_val=ra_val)   
+ 

app/routes/generate_comparison_report.py

@@ -0,0 +1,352 @@
+from flask import Blueprint, render_template, request, send_file, flash
+from collections import defaultdict
+import pandas as pd
+import io
+ 
+from app.models.subcontractor_model import Subcontractor
+from app.models.trench_excavation_model import TrenchExcavation
+from app.models.manhole_excavation_model import ManholeExcavation
+from app.models.manhole_domestic_chamber_model import ManholeDomesticChamber
+from app.models.laying_model import Laying
+ 
+from app.models.tr_ex_client_model import TrenchExcavationClient
+from app.models.mh_ex_client_model import ManholeExcavationClient
+from app.models.mh_dc_client_model import ManholeDomesticChamberClient
+from app.models.laying_client_model import LayingClient
+ 
+from app.utils.helpers import login_required
+import re
+ 
+ 
+generate_report_bp = Blueprint("generate_report", __name__, url_prefix="/report")
+ 
+ 
+# sum field  of pipe laying (pipe_150_mm)
+PIPE_MM_PATTERN = re.compile(r"^pipe_\d+_mm$")
+# sum fields of MH dc (d_0_to_0_75)
+D_RANGE_PATTERN = re.compile( r"^d_\d+(?:_\d+)?_to_\d+(?:_\d+)?$")
+ 
+ 
+# NORMALIZER
+def normalize_key(value):
+    if value is None:
+        return None
+    return str(value).strip().upper()
+ 
+ 
+# HEADER FORMATTER
+def format_header(header):
+    if "-" in header:
+        prefix, rest = header.split("-", 1)
+        prefix = prefix.title()
+    else:
+        prefix, rest = None, header
+ 
+    parts = rest.split("_")
+    result = []
+    i = 0
+ 
+    while i < len(parts):
+        if i + 1 < len(parts) and parts[i].isdigit() and parts[i + 1].isdigit():
+            result.append(f"{parts[i]}.{parts[i + 1]}")
+            i += 2
+        else:
+            result.append(parts[i].title())
+            i += 1
+ 
+    final_text = " ".join(result)
+    return f"{prefix}-{final_text}" if prefix else final_text
+ 
+ 
+# LOOKUP CREATOR
+def make_lookup(rows, key_field):
+    lookup = {}
+    for r in rows:
+        location = normalize_key(r.get("Location"))
+        key_val = normalize_key(r.get(key_field))
+ 
+        if location and key_val:
+            lookup.setdefault((location, key_val), []).append(r)
+ 
+    return lookup
+ 
+ 
+ 
+# COMPARISON BUILDER
+def build_comparison(client_rows, contractor_rows, key_field):
+    contractor_lookup = make_lookup(contractor_rows, key_field)
+    output = []
+ 
+    used_index = defaultdict(int)  # 🔥 THIS FIXES YOUR ISSUE
+ 
+    for c in client_rows:
+        client_location = normalize_key(c.get("Location"))
+        client_key = normalize_key(c.get(key_field))
+ 
+        if not client_location or not client_key:
+            continue
+ 
+        subs = contractor_lookup.get((client_location, client_key))
+        if not subs:
+            continue
+ 
+        idx = used_index[(client_location, client_key)]
+ 
+        # ❗ If subcontractor rows are exhausted, skip
+        if idx >= len(subs):
+            continue
+ 
+        s = subs[idx]                     # ✅ take NEXT subcontractor row
+        used_index[(client_location, client_key)] += 1
+ 
+        # ---- totals ----
+        client_total = sum(
+            float(v or 0)
+            for k, v in c.items()
+            if k.endswith("_total")
+            or D_RANGE_PATTERN.match(k)
+            or PIPE_MM_PATTERN.match(k)
+        )
+ 
+        sub_total = sum(
+            float(v or 0)
+            for k, v in s.items()
+            if k.endswith("_total")
+            or D_RANGE_PATTERN.match(k)
+            or PIPE_MM_PATTERN.match(k)
+        )
+ 
+        row = {
+            "Location": client_location,
+            key_field.replace("_", " "): client_key
+        }
+ 
+        for k, v in c.items():
+            if k not in ["id", "created_at"]:
+                row[f"Client-{k}"] = v
+ 
+        row["Client-Total"] = round(client_total, 2)
+        row[" "] = ""
+ 
+        for k, v in s.items():
+            if k not in ["id", "created_at", "subcontractor_id"]:
+                row[f"Subcontractor-{k}"] = v
+ 
+        row["Subcontractor-Total"] = round(sub_total, 2)
+        row["Diff"] = round(client_total - sub_total, 2)
+ 
+        output.append(row)
+ 
+    df = pd.DataFrame(output)
+    df.columns = [format_header(col) for col in df.columns]
+    return df
+ 
+ 
+ 
+ 
+# EXCEL SHEET WRITER
+def write_sheet(writer, df, sheet_name, subcontractor_name):
+    workbook = writer.book
+    df.to_excel(writer, sheet_name=sheet_name, index=False, startrow=3)
+    ws = writer.sheets[sheet_name]
+ 
+    title_fmt = workbook.add_format({"bold": True, "font_size": 14})
+    client_fmt = workbook.add_format({"bold": True, "border": 1, "bg_color": "#B6DAED"})
+    sub_fmt = workbook.add_format({"bold": True, "border": 1, "bg_color": "#F3A081"})
+    total_fmt = workbook.add_format({"bold": True, "border": 1, "bg_color": "#F7D261"})
+    diff_fmt = workbook.add_format({"bold": True, "border": 1, "bg_color": "#82DD49"})
+    default_header_fmt = workbook.add_format({"bold": True,"border": 1,"bg_color": "#E7E6E6","align": "center","valign": "vcenter"})
+ 
+ 
+    ws.merge_range(
+        0, 0, 0, len(df.columns) - 1,
+        "CLIENT vs SUBCONTRACTOR",
+        title_fmt
+    )
+    ws.merge_range(
+        1, 0, 1, len(df.columns) - 1,
+        f"Subcontractor Name - {subcontractor_name}",
+        title_fmt
+    )
+ 
+ 
+    for col_num, col_name in enumerate(df.columns):
+        if col_name.startswith("Client-"):
+            ws.write(3, col_num, col_name, client_fmt)
+        elif col_name.startswith("Subcontractor-"):
+            ws.write(3, col_num, col_name, sub_fmt)
+        elif col_name.endswith("_total") or col_name.endswith("_total") :
+            ws.write(3, col_num, col_name, total_fmt)
+        elif col_name == "Diff":
+            ws.write(3, col_num, col_name, diff_fmt)
+        else:
+            ws.write(3, col_num, col_name, default_header_fmt)
+ 
+        ws.set_column(col_num, col_num, 20)
+ 
+ 
+# REPORT ROUTE
+@generate_report_bp.route("/comparison_report", methods=["GET", "POST"])
+@login_required
+def comparison_report():
+    subcontractors = Subcontractor.query.all()
+ 
+    if request.method == "POST":
+        subcontractor_id = request.form.get("subcontractor_id")
+        if not subcontractor_id:
+            flash("Please select subcontractor", "danger")
+            return render_template("generate_comparison_report.html",subcontractors=subcontractors)
+ 
+        subcontractor = Subcontractor.query.get_or_404(subcontractor_id)
+ 
+        # -------- DATA --------
+        tr_client = [r.serialize() for r in TrenchExcavationClient.query.all()]
+        tr_sub = [r.serialize() for r in TrenchExcavation.query.filter_by(
+            subcontractor_id=subcontractor_id
+        ).all()]
+        df_tr = build_comparison(tr_client, tr_sub, "MH_NO")
+ 
+        mh_client = [r.serialize() for r in ManholeExcavationClient.query.all()]
+        mh_sub = [r.serialize() for r in ManholeExcavation.query.filter_by(
+            subcontractor_id=subcontractor_id
+        ).all()]
+        df_mh = build_comparison(mh_client, mh_sub, "MH_NO")
+ 
+        dc_client = [r.serialize() for r in ManholeDomesticChamberClient.query.all()]
+        dc_sub = [r.serialize() for r in ManholeDomesticChamber.query.filter_by(
+            subcontractor_id=subcontractor_id
+        ).all()]
+        df_dc = build_comparison(dc_client, dc_sub, "MH_NO")
+        # df_dc = build_comparison_mh_dc(dc_client, dc_sub, "MH_NO")
+ 
+        lay_client = [r.serialize() for r in LayingClient.query.all()]
+        lay_sub = [r.serialize() for r in Laying.query.filter_by(
+            subcontractor_id=subcontractor_id
+        ).all()]
+        df_lay = build_comparison(lay_client, lay_sub, "MH_NO")
+        # df_lay = build_comparison_laying(lay_client, lay_sub, "MH_NO")
+ 
+ 
+        # -------- EXCEL --------
+        output = io.BytesIO()
+        filename = f"{subcontractor.subcontractor_name}_Comparison_Report.xlsx"
+ 
+        with pd.ExcelWriter(output, engine="xlsxwriter") as writer:
+            write_sheet(writer, df_tr, "Tr.Ex", subcontractor.subcontractor_name)
+            write_sheet(writer, df_mh, "Mh.Ex", subcontractor.subcontractor_name)
+            write_sheet(writer, df_dc, "MH & DC", subcontractor.subcontractor_name)
+            write_sheet(writer, df_lay, "Laying", subcontractor.subcontractor_name)
+ 
+        output.seek(0)
+        return send_file(
+            output,
+            as_attachment=True,
+            download_name=filename,
+            mimetype="application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"
+        )
+ 
+    return render_template("generate_comparison_report.html",subcontractors=subcontractors)
+ 
+ 
+# def build_comparison_mh_dc(client_rows, contractor_rows, key_field):
+#     contractor_lookup = make_lookup(contractor_rows, key_field)
+#     mh_dc_fields = ManholeDomesticChamberClient.sum_mh_dc_fields()
+ 
+#     output = []
+ 
+#     for c in client_rows:
+#         loc = normalize_key(c.get("Location"))
+#         key = normalize_key(c.get(key_field))
+#         if not loc or not key:
+#             continue
+ 
+#         s = contractor_lookup.get((loc, key))
+#         if not s:
+#             continue
+ 
+#         client_total = sum(float(c.get(f, 0) or 0) for f in mh_dc_fields)
+#         sub_total = sum(float(s.get(f, 0) or 0) for f in mh_dc_fields)
+ 
+#         row = {
+#             "Location": loc,
+#             key_field.replace("_", " "): key
+#         }
+ 
+#         # CLIENT – ALL FIELDS
+#         for k, v in c.items():
+#             if k in ["id", "created_at"]:
+#                 continue
+#             row[f"Client-{k}"] = v
+ 
+#         row["Client-Total"] = round(client_total, 2)
+#         row[" "] = ""
+ 
+#         # SUBCONTRACTOR – ALL FIELDS
+#         for k, v in s.items():
+#             if k in ["id", "created_at", "subcontractor_id"]:
+#                 continue
+#             row[f"Subcontractor-{k}"] = v
+ 
+#         row["Subcontractor-Total"] = round(sub_total, 2)
+#         row["Diff"] = round(client_total - sub_total, 2)
+ 
+#         output.append(row)
+ 
+#     df = pd.DataFrame(output)
+#     df.columns = [format_header(col) for col in df.columns]
+#     return df
+ 
+ 
+# def build_comparison_laying(client_rows, contractor_rows, key_field):
+#     contractor_lookup = make_lookup(contractor_rows, key_field)
+#     laying_fields = Laying.sum_laying_fields()
+ 
+#     output = []
+ 
+#     for c in client_rows:
+#         loc = normalize_key(c.get("Location"))
+#         key = normalize_key(c.get(key_field))
+#         if not loc or not key:
+#             continue
+ 
+#         s = contractor_lookup.get((loc, key))
+#         if not s:
+#             continue
+ 
+#         client_total = sum(float(c.get(f, 0) or 0) for f in laying_fields)
+#         sub_total = sum(float(s.get(f, 0) or 0) for f in laying_fields)
+ 
+#         print("--------------",key,"----------")
+#         print("sum -client_total ",client_total)
+#         print("sum -sub_total ",sub_total)
+#         print("Diff ---- ",client_total - sub_total)
+#         print("------------------------")
+#         row = {
+#             "Location": loc,
+#             key_field.replace("_", " "): key
+#         }
+ 
+#         # CLIENT – ALL FIELDS
+#         for k, v in c.items():
+#             if k in ["id", "created_at"]:
+#                 continue
+#             row[f"Client-{k}"] = v
+ 
+#         row["Client-Total"] = round(client_total, 2)
+#         row[" "] = ""
+ 
+#         # SUBCONTRACTOR – ALL FIELDS
+#         for k, v in s.items():
+#             if k in ["id", "created_at", "subcontractor_id"]:
+#                 continue
+#             row[f"Subcontractor-{k}"] = v
+ 
+#         row["Subcontractor-Total"] = round(sub_total, 2)
+#         row["Diff"] = round(client_total - sub_total, 2)
+ 
+#         output.append(row)
+ 
+#     df = pd.DataFrame(output)
+#     df.columns = [format_header(col) for col in df.columns]
+#     return df
+ 

app/routes/subcontractor_routes.py

@@ -1,59 +1,186 @@
 from flask import Blueprint, render_template, request, redirect, flash
 from app import db
 from app.models.subcontractor_model import Subcontractor
-
+from app.utils.helpers import login_required
+ 
 subcontractor_bp = Blueprint("subcontractor", __name__, url_prefix="/subcontractor")
-
+ 
 # ---------------- ADD -----------------
 @subcontractor_bp.route("/add")
+@login_required
+def add_subcontractor():
+    return render_template("subcontractor/add.html")
+@subcontractor_bp.route("/save", methods=["POST"])
+@login_required
+def save_subcontractor():
+    # 1. Get and clean the name from the form
+    name = request.form.get("subcontractor_name", "").strip()
+ 
+    # 2. Basic validation: Ensure the name isn't empty
+    if not name:
+        flash("Subcontractor name cannot be empty.", "danger")
+        return redirect("/subcontractor/add")
+ 
+    # 3. Check if a subcontractor with this name already exists
+    existing_sub = Subcontractor.query.filter_by(subcontractor_name=name).first()
+ 
+    if existing_sub:
+        flash(f"Subcontractor with name '{name}' already exists!", "danger")
+        return redirect("/subcontractor/add")
+ 
+    # 4. If no duplicate is found, proceed to save
+    try:
+        subcontractor = Subcontractor(
+            subcontractor_name=name,
+            contact_person=request.form.get("contact_person"),
+            mobile_no=request.form.get("mobile_no"),
+            email_id=request.form.get("email_id"),
+            gst_no=request.form.get("gst_no")
+        )
+       
+        db.session.add(subcontractor)
+        db.session.commit()
+        flash("Subcontractor added successfully!", "success")
+       
+    except Exception as e:
+        db.session.rollback()
+        flash("An error occurred while saving. Please try again.", "danger")
+ 
+    return redirect("/subcontractor/list")
+ 
+# ---------------- LIST -----------------
+@subcontractor_bp.route("/list")
+@login_required
+def subcontractor_list():
+    subcontractors = Subcontractor.query.all()
+    return render_template("subcontractor/list.html", subcontractors=subcontractors)
+ 
+# ---------------- EDIT -----------------
+@subcontractor_bp.route("/edit/<int:id>")
+@login_required
+def edit_subcontractor(id):
+    subcontractor = Subcontractor.query.get_or_404(id)
+    return render_template("subcontractor/edit.html", subcontractor=subcontractor)
+ 
+# ---------------- UPDATE -----------------
+@subcontractor_bp.route("/update/<int:id>", methods=["POST"])
+@login_required
+def update_subcontractor(id):
+    subcontractor = Subcontractor.query.get_or_404(id)
+    new_name = request.form.get("subcontractor_name")
+ 
+    # Check if the new name is taken by someone ELSE (not this current ID)
+    duplicate = Subcontractor.query.filter(
+        Subcontractor.subcontractor_name == new_name,
+        Subcontractor.id != id
+    ).first()
+ 
+    if duplicate:
+        flash("Another subcontractor already uses this name.", "danger")
+        return redirect(f"/subcontractor/edit/{id}")
+ 
+    subcontractor.subcontractor_name = new_name
+   
+    db.session.commit()
+   
+    flash("Subcontractor updated successfully!", "success")
+    return redirect("/subcontractor/list")
+ 
+# ---------------- DELETE -----------------
+@subcontractor_bp.route("/delete/<int:id>")
+@login_required
+def delete_subcontractor(id):
+    subcontractor = Subcontractor.query.get_or_404(id)
+ 
+    db.session.delete(subcontractor)
+    db.session.commit()
+ 
+    flash("Subcontractor deleted successfully!", "success")
+    return redirect("/subcontractor/list")
+from flask import Blueprint, render_template, request, redirect, flash
+from app import db
+from app.models.subcontractor_model import Subcontractor
+from app.utils.helpers import login_required
+ 
+subcontractor_bp = Blueprint("subcontractor", __name__, url_prefix="/subcontractor")
+ 
+# ---------------- ADD -----------------
+@subcontractor_bp.route("/add")
+@login_required
 def add_subcontractor():
     return render_template("subcontractor/add.html")
 
 @subcontractor_bp.route("/save", methods=["POST"])
+@login_required
 def save_subcontractor():
-    subcontractor = Subcontractor(
-        subcontractor_name=request.form.get("subcontractor_name"),
-        contact_person=request.form.get("contact_person"),
-        mobile_no=request.form.get("mobile_no"),
-        email_id=request.form.get("email_id"),
-        gst_no=request.form.get("gst_no")
-    )
-    db.session.add(subcontractor)
-    db.session.commit()
-
-    flash("Subcontractor added successfully!", "success")
+    name = request.form.get("subcontractor_name", "").strip()
+    if not name:
+        flash("Subcontractor name cannot be empty.", "danger")
+        return redirect("/subcontractor/add")
+    existing_sub = Subcontractor.query.filter_by(subcontractor_name=name).first()
+ 
+    if existing_sub:
+        flash(f"Subcontractor with name '{name}' already exists!", "danger")
+        return redirect("/subcontractor/add")
+    try:
+        subcontractor = Subcontractor(
+            subcontractor_name=name,
+            contact_person=request.form.get("contact_person"),
+            mobile_no=request.form.get("mobile_no"),
+            email_id=request.form.get("email_id"),
+            gst_no=request.form.get("gst_no")
+        )
+       
+        db.session.add(subcontractor)
+        db.session.commit()
+        flash("Subcontractor added successfully!", "success")
+       
+    except Exception as e:
+        db.session.rollback()
+        flash("An error occurred while saving. Please try again.", "danger")
+ 
     return redirect("/subcontractor/list")
-
+ 
 # ---------------- LIST -----------------
 @subcontractor_bp.route("/list")
+@login_required
 def subcontractor_list():
     subcontractors = Subcontractor.query.all()
     return render_template("subcontractor/list.html", subcontractors=subcontractors)
-
+ 
 # ---------------- EDIT -----------------
 @subcontractor_bp.route("/edit/<int:id>")
+@login_required
 def edit_subcontractor(id):
     subcontractor = Subcontractor.query.get_or_404(id)
     return render_template("subcontractor/edit.html", subcontractor=subcontractor)
-
+ 
 # ---------------- UPDATE -----------------
 @subcontractor_bp.route("/update/<int:id>", methods=["POST"])
+@login_required
 def update_subcontractor(id):
     subcontractor = Subcontractor.query.get_or_404(id)
-
-    subcontractor.subcontractor_name = request.form.get("subcontractor_name")
-    subcontractor.contact_person = request.form.get("contact_person")
-    subcontractor.mobile_no = request.form.get("mobile_no")
-    subcontractor.email_id = request.form.get("email_id")
-    subcontractor.gst_no = request.form.get("gst_no")
-
+    new_name = request.form.get("subcontractor_name")
+ 
+    # Check if the new name is taken by someone ELSE (not this current ID)
+    duplicate = Subcontractor.query.filter(
+        Subcontractor.subcontractor_name == new_name,
+        Subcontractor.id != id
+    ).first()
+ 
+    if duplicate:
+        flash("Another subcontractor already uses this name.", "danger")
+        return redirect(f"/subcontractor/edit/{id}")
+ 
+    subcontractor.subcontractor_name = new_name
     db.session.commit()
-
+   
     flash("Subcontractor updated successfully!", "success")
     return redirect("/subcontractor/list")
-
+ 
 # ---------------- DELETE -----------------
 @subcontractor_bp.route("/delete/<int:id>")
+@login_required
 def delete_subcontractor(id):
     subcontractor = Subcontractor.query.get_or_404(id)
 
@@ -61,4 +188,4 @@ def delete_subcontractor(id):
     db.session.commit()
 
     flash("Subcontractor deleted successfully!", "success")
-    return redirect("/subcontractor/list")
+    return redirect("/subcontractor/list")

app/routes/user.py

@@ -1,9 +1,11 @@
 from flask import Blueprint, render_template
 from app.services.user_service import UserService
+from app.utils.helpers import login_required
 
 user_bp = Blueprint("user", __name__, url_prefix="/user")
 
 @user_bp.route("/list")
+@login_required
 def list_users():
-    users = UserService().get_all_users()
+    users = UserService.get_all_users()
     return render_template("users.html", users=users, title="Users")

app/services/dashboard_service.py

@@ -0,0 +1,49 @@
+from app.config import Config
+from app import db
+import matplotlib
+matplotlib.use("Agg")
+from app.routes.dashboard import plot_to_base64
+import matplotlib.pyplot as plt
+import io
+import base64
+
+from app.utils.plot_utils import plot_to_base64
+
+# Subcontractor models import
+from app.models.trench_excavation_model import TrenchExcavation
+
+# Client models import
+# from app.models.tr_ex_client_model import TrenchExcavationClient
+
+
+class DashboardService:
+
+
+    # bar chart
+    def bar_chart_of_tr_ex():
+        categories = ["Soft Murum", "Hard Murum", "Soft Rock", "Hard Rock"]
+        values = [120, 80, 150, 60]
+
+        tr = TrenchExcavation()
+
+        record = TrenchExcavation.query.first()
+        print(" RA_Bill_No::",record["RA_Bill_No"])
+
+        totals = tr.excavation_category_sums()
+
+        # print(totals["Soft_Murum_Total"])
+        # print(totals["Hard_Rock_Total"])
+
+
+        plt.figure()
+        plt.bar(categories, values)
+        plt.title("Trench Excavation Work Category Report")
+        plt.xlabel("Excavation category")
+        plt.ylabel("Quantity")
+
+        return plot_to_base64(plt)
+
+
+    def subcontractor_dash():
+            return True
+        

app/services/file_service.py

@@ -1,28 +1,53 @@
 import os
 import pandas as pd
 from werkzeug.utils import secure_filename
+from app.utils.file_utils import ensure_upload_folder
+
 from app.config import Config
 from app import db
+
+# Subcontractor models import
 from app.models.trench_excavation_model import TrenchExcavation
 from app.models.manhole_excavation_model import ManholeExcavation
 from app.models.manhole_domestic_chamber_model import ManholeDomesticChamber
+from app.models.laying_model import Laying
+
+# Client models import
+from app.models.tr_ex_client_model import TrenchExcavationClient
+from app.models.mh_ex_client_model import ManholeExcavationClient
+from app.models.mh_dc_client_model import ManholeDomesticChamberClient
+from app.models.laying_client_model import LayingClient
 
-from app.utils.file_utils import ensure_upload_folder
 
 
 class FileService:
 
+    # ---------------- COMMON HELPERS ----------------
     def allowed_file(self, filename):
-        return "." in filename and filename.rsplit(".", 1)[1].lower() in Config.ALLOWED_EXTENSIONS
+        return ("." in filename and filename.rsplit(".", 1)[1].lower() in Config.ALLOWED_EXTENSIONS)
 
-    def handle_file_upload(self, file, subcontractor_id, file_type):
+    def normalize(self, val):
+        if val is None or pd.isna(val):
+            return None
+        
+        val = str(val).strip()
+        if val.lower() in ["", "nan", "none", "-", "—"]:
+            return None
+
+        return val.upper()
+
+    # ---------------- SUBCONTRACTOR FILE UPLOAD ----------------
+    def handle_file_upload(self, file, subcontractor_id, RA_Bill_No):
 
         if not subcontractor_id:
             return False, "Please select subcontractor."
-        if not file_type:
-            return False, "Please select file type."
+
+        if not RA_Bill_No:
+            return False, "Please Enter RA Bill No."
+
         if not file or file.filename == "":
             return False, "No file selected."
+
         if not self.allowed_file(file.filename):
             return False, "Invalid file type! Allowed: CSV, XLSX, XLS"
 
@@ -36,190 +61,315 @@ class FileService:
         file.save(filepath)
 
         try:
-            df = pd.read_csv(filepath) if filename.endswith(".csv") else pd.read_excel(filepath)
+            df_tr_ex = pd.read_excel(filepath, sheet_name="Tr.Ex.", header=12, dtype={"MH No": str})
+            df_mh_ex = pd.read_excel(filepath, sheet_name="MH Ex.", header=12, dtype={"MH No": str})
+            df_mh_dc = pd.read_excel(filepath, sheet_name="MH & DC", header=11, dtype={"MH No": str})
+            df_laying = pd.read_excel(filepath, sheet_name="Laying", header=11, dtype={"MH No": str})
 
-            print("\n=== Uploaded File Preview ===")
-            print(df.head())
-            print("=============================\n")
+            self.process_trench_excavation(df_tr_ex, subcontractor_id, RA_Bill_No)
+            self.process_manhole_excavation(df_mh_ex, subcontractor_id, RA_Bill_No)
+            self.process_manhole_domestic_chamber(df_mh_dc, subcontractor_id, RA_Bill_No)
+            self.process_laying(df_laying, subcontractor_id, RA_Bill_No)
 
-            # Trench Excavation save 
-            if file_type == "trench_excavation":
-                return self.process_trench_excavation(df, subcontractor_id)
-            
-            # Manhole Excavation save
-            if file_type == "manhole_excavation":
-                return self.process_manhole_excavation(df, subcontractor_id)
-            
-            # Manhole and Domestic Chamber Construction save
-            if file_type == "manhole_domestic_chamber":
-                return self.process_manhole_excavation(df, subcontractor_id)
-
-
-            return True, "File uploaded successfully."
-
-        except Exception as e:
-            return False, f"Processing failed: {e}"
-
-
-
-    #   Trench Excavation save method (TrenchExcavation model)
-    def process_trench_excavation(self, df, subcontractor_id):
-
-        # Clean column names (strip whitespace)
-        df.columns = [str(c).strip() for c in df.columns]
-
-        # If the sheet has merged cells -> forward fill Location
-        if "Location" in df.columns:
-            df["Location"] = df["Location"].ffill()
-
-        # REMOVE empty rows
-        df = df.dropna(how="all")
-
-        # Identify missing location rows before insert
-        missing_loc = df[df["Location"].isna() | (df["Location"].astype(str).str.strip() == "")]
-        if not missing_loc.empty:
-            return False, f"Error: Some rows have empty Location. Rows: {missing_loc.index.tolist()}"
-
-        saved_count = 0
-
-        try:
-            for index, row in df.iterrows():
-
-                record_data = {}
-
-                # Insert only fields that exist in model
-                for col in df.columns:
-                    if hasattr(TrenchExcavation, col):
-                        value = row[col]
-
-                        # Normalize empty values
-                        if pd.isna(value) or str(value).strip() in ["", "-", "—", "nan", "NaN"]:
-                            value = None
-
-                        record_data[col] = value
-
-                record = TrenchExcavation(
-                    subcontractor_id=subcontractor_id,
-                    **record_data
-                )
-
-                db.session.add(record)
-                saved_count += 1
-
-            db.session.commit()
-            return True, f"Trench Excavation data saved successfully. Total rows: {saved_count}"
+            return True, "SUBCONTRACTOR File uploaded successfully."
 
         except Exception as e:
             db.session.rollback()
-            return False, f"Trench Excavation Save Failed: {e}"
-        
+            return False, f"Import failed: {e}"
 
-    #   Manhole Excavation save method (ManholeExcavation model)
-    def process_manhole_excavation(self, df, subcontractor_id):
+    # ---------------- Trench Excavation (Subcontractor) ----------------
+    def process_trench_excavation(self, df, subcontractor_id, RA_Bill_No):
 
-        # Clean column names (strip whitespace)
-        df.columns = [str(c).strip() for c in df.columns]
+        df.columns = (
+            df.columns.astype(str)
+            .str.strip()
+            .str.replace(r"[^\w]", "_", regex=True)
+            .str.replace("__+", "_", regex=True)
+            .str.strip("_")
+        )
+
+        df = df.dropna(how="all")
 
-        # If the sheet has merged cells -> forward fill Location
         if "Location" in df.columns:
             df["Location"] = df["Location"].ffill()
 
-        # REMOVE empty rows
+        errors = []
+
+        for idx, row in df.iterrows():
+            location = self.normalize(row.get("Location"))
+            mh_no = self.normalize(row.get("MH_NO"))
+
+            if not location or not mh_no:
+                continue
+
+            # exists = TrenchExcavation.query.filter_by(
+            #     subcontractor_id=subcontractor_id,
+            #     RA_Bill_No=RA_Bill_No,
+            #     Location=location,
+            #     MH_NO=mh_no,
+            # ).first()
+
+            # if exists:
+            #     errors.append(
+            #         f"Model-Tr.Ex. (Row {idx+1}): Duplicate → Location={location}, MH_NO={mh_no}"
+            #     )
+            #     continue
+
+            record_data = {}
+            for col in df.columns:
+                if hasattr(TrenchExcavation, col):
+                    val = row[col]
+                    if pd.isna(val) or str(val).strip() in ["", "-", "—", "nan"]:
+                        val = None
+                    record_data[col] = val
+
+            record = TrenchExcavation(
+                subcontractor_id=subcontractor_id,
+                RA_Bill_No=RA_Bill_No,
+                **record_data,
+            )
+            db.session.add(record)
+
+        if errors:
+            raise Exception(" | ".join(errors))
+
+        db.session.commit()
+
+    # ---------------- Manhole Excavation (Subcontractor) ----------------
+    def process_manhole_excavation(self, df, subcontractor_id, RA_Bill_No):
+
+        df.columns = (
+            df.columns.astype(str)
+            .str.strip()
+            .str.replace(r"[^\w]", "_", regex=True)
+            .str.replace("__+", "_", regex=True)
+            .str.strip("_")
+        )
+
         df = df.dropna(how="all")
 
-        # Identify missing location rows before insert
-        missing_loc = df[df["Location"].isna() | (df["Location"].astype(str).str.strip() == "")]
-        if not missing_loc.empty:
-            return False, f"Error: Some rows have empty Location. Rows: {missing_loc.index.tolist()}"
-
-        saved_count = 0
-
-        try:
-            for index, row in df.iterrows():
-
-                record_data = {}
-
-                # Insert only fields that exist in model
-                for col in df.columns:
-                    if hasattr(ManholeExcavation, col):
-                        value = row[col]
-
-                        # Normalize empty values
-                        if pd.isna(value) or str(value).strip() in ["", "-", "—", "nan", "NaN"]:
-                            value = None
-
-                        record_data[col] = value
-
-                record = ManholeExcavation(
-                    subcontractor_id=subcontractor_id,
-                    **record_data
-                )
-
-                db.session.add(record)
-                saved_count += 1
-
-            db.session.commit()
-            return True, f"Manhole Excavation data saved successfully. Total rows: {saved_count}"
-
-        except Exception as e:
-            db.session.rollback()
-            return False, f"Manhole Excavation Save Failed: {e}"
-
-
-    #   Manhole and Domestic Chamber Construction save method (ManholeDomesticChamber  model)
-    def process_manhole_excavation(self, df, subcontractor_id):
-
-        # Clean column names (strip whitespace)
-        df.columns = [str(c).strip() for c in df.columns]
-
-        # If the sheet has merged cells -> forward fill Location
         if "Location" in df.columns:
             df["Location"] = df["Location"].ffill()
 
-        # REMOVE empty rows
+        errors = []
+
+        for idx, row in df.iterrows():
+            location = self.normalize(row.get("Location"))
+            mh_no = self.normalize(row.get("MH_NO"))
+
+            if not location or not mh_no:
+                continue
+
+            # exists = ManholeExcavation.query.filter_by(
+            #     subcontractor_id=subcontractor_id,
+            #     RA_Bill_No=RA_Bill_No,
+            #     Location=location,
+            #     MH_NO=mh_no,
+            # ).first()
+
+            # if exists:
+            #     errors.append(
+            #         f"Model-MH Ex. (Row {idx+1}): Duplicate → Location={location}, MH_NO={mh_no}"
+            #     )
+            #     continue
+
+            record_data = {}
+            for col in df.columns:
+                if hasattr(ManholeExcavation, col):
+                    val = row[col]
+                    if pd.isna(val) or str(val).strip() in ["", "-", "—", "nan"]:
+                        val = None
+                    record_data[col] = val
+
+            record = ManholeExcavation(
+                subcontractor_id=subcontractor_id,
+                RA_Bill_No=RA_Bill_No,
+                **record_data,
+            )
+            db.session.add(record)
+
+        if errors:
+            raise Exception(" | ".join(errors))
+
+        db.session.commit()
+
+    # ---------------- Manhole & Domestic Chamber (Subcontractor) ----------------
+    def process_manhole_domestic_chamber(self, df, subcontractor_id, RA_Bill_No):
+
+        df.columns = (
+            df.columns.astype(str)
+            .str.strip()
+            .str.replace(r"[^\w]", "_", regex=True)
+            .str.replace("__+", "_", regex=True)
+            .str.strip("_")
+        )
+
         df = df.dropna(how="all")
 
-        # Identify missing location rows before insert
-        missing_loc = df[df["Location"].isna() | (df["Location"].astype(str).str.strip() == "")]
-        if not missing_loc.empty:
-            return False, f"Error: Some rows have empty Location. Rows: {missing_loc.index.tolist()}"
+        if "Location" in df.columns:
+            df["Location"] = df["Location"].ffill()
 
-        saved_count = 0
+        errors = []
+
+        for idx, row in df.iterrows():
+            location = self.normalize(row.get("Location"))
+            mh_no = self.normalize(row.get("MH_NO"))
+
+            if not location or not mh_no:
+                continue
+
+            # exists = ManholeDomesticChamber.query.filter_by(
+            #     subcontractor_id=subcontractor_id,
+            #     RA_Bill_No=RA_Bill_No,
+            #     Location=location,
+            #     MH_NO=mh_no,
+            # ).first()
+
+            # if exists:
+            #     errors.append(
+            #         f"Model-MH & DC (Row {idx+1}): Duplicate → Location={location}, MH_NO={mh_no}"
+            #     )
+            #     continue
+
+            record_data = {}
+            for col in df.columns:
+                if hasattr(ManholeDomesticChamber, col):
+                    val = row[col]
+                    if pd.isna(val) or str(val).strip() in ["", "-", "—", "nan"]:
+                        val = None
+                    record_data[col] = val
+
+            record = ManholeDomesticChamber(
+                subcontractor_id=subcontractor_id,
+                RA_Bill_No=RA_Bill_No,
+                **record_data,
+            )
+            db.session.add(record)
+
+        if errors:
+            raise Exception(" | ".join(errors))
+
+        db.session.commit()
+
+    # ---------------- Laying (Subcontractor) ----------------
+    def process_laying(self, df, subcontractor_id, RA_Bill_No):
+
+        df.columns = (
+            df.columns.astype(str)
+            .str.strip()
+            .str.replace(r"[^\w]", "_", regex=True)
+            .str.replace("__+", "_", regex=True)
+            .str.strip("_")
+        )
+
+        df = df.dropna(how="all")
+
+        if "Location" in df.columns:
+            df["Location"] = df["Location"].ffill()
+
+        errors = []
+
+        for idx, row in df.iterrows():
+            location = self.normalize(row.get("Location"))
+            mh_no = self.normalize(row.get("MH_NO"))
+
+            if not location or not mh_no:
+                continue
+
+            # exists = ManholeDomesticChamber.query.filter_by(
+            #     subcontractor_id=subcontractor_id,
+            #     RA_Bill_No=RA_Bill_No,
+            #     Location=location,
+            #     MH_NO=mh_no,
+            # ).first()
+
+            # if exists:
+            #     errors.append(
+            #         f"Model-MH & DC (Row {idx+1}): Duplicate → Location={location}, MH_NO={mh_no}"
+            #     )
+            #     continue
+
+            record_data = {}
+            for col in df.columns:
+                if hasattr(Laying, col):
+                    val = row[col]
+                    if pd.isna(val) or str(val).strip() in ["", "-", "—", "nan"]:
+                        val = None
+                    record_data[col] = val
+
+            record = Laying(
+                subcontractor_id=subcontractor_id,
+                RA_Bill_No=RA_Bill_No,
+                **record_data,
+            )
+            db.session.add(record)
+
+        if errors:
+            raise Exception(" | ".join(errors))
+
+        db.session.commit()
+
+
+
+    # ---------------- CLIENT FILE UPLOAD ----------------
+    def handle_client_file_upload(self, file, RA_Bill_No):
+
+        if not RA_Bill_No:
+            return False, "Please Enter RA Bill No."
+
+        if not file or file.filename == "":
+            return False, "No file selected."
+
+        if not self.allowed_file(file.filename):
+            return False, "Invalid file type! Allowed: CSV, XLSX, XLS"
+
+        ensure_upload_folder()
+
+        folder = os.path.join(Config.UPLOAD_FOLDER, f"Client_Bill_{RA_Bill_No}")
+        os.makedirs(folder, exist_ok=True)
+
+        filename = secure_filename(file.filename)
+        filepath = os.path.join(folder, filename)
+        file.save(filepath)
 
         try:
-            for index, row in df.iterrows():
+            df_tr_ex = pd.read_excel(filepath, sheet_name="Tr.Ex.", header=4)
+            df_mh_ex = pd.read_excel(filepath, sheet_name="MH Ex.", header=4)
+            df_mh_dc = pd.read_excel(filepath, sheet_name="MH & DC", header=3)
+            df_lay = pd.read_excel(filepath, sheet_name="Laying & Bedding", header=3)
 
-                record_data = {}
-
-                # Insert only fields that exist in model
-                for col in df.columns:
-                    if hasattr(ManholeDomesticChamber, col):
-                        value = row[col]
-
-                        # Normalize empty values
-                        if pd.isna(value) or str(value).strip() in ["", "-", "—", "nan", "NaN"]:
-                            value = None
-
-                        record_data[col] = value
-
-                record = ManholeDomesticChamber(
-                    subcontractor_id=subcontractor_id,
-                    **record_data
-                )
-
-                db.session.add(record)
-                saved_count += 1
+            self.save_client_data(df_tr_ex, TrenchExcavationClient, RA_Bill_No)
+            self.save_client_data(df_mh_ex, ManholeExcavationClient, RA_Bill_No)
+            self.save_client_data(df_mh_dc, ManholeDomesticChamberClient, RA_Bill_No)
+            self.save_client_data(df_lay, LayingClient, RA_Bill_No)
 
             db.session.commit()
-            return True, f"Manhole Domestic Chamber Construction data saved successfully. Total rows: {saved_count}"
+            return True, "Client file uploaded successfully."
 
         except Exception as e:
             db.session.rollback()
-            return False, f"Manhole Domestic Chamber Construction Save Failed: {e}"
-        
+            return False, f"Client import failed: {e}"
 
+    # ---------------- CLIENT SAVE METHOD ----------------
+    def save_client_data(self, df, model, RA_Bill_No):
 
+        df.columns = [str(c).strip() for c in df.columns]
 
+        if "Location" in df.columns:
+            df["Location"] = df["Location"].ffill()
 
+        df = df.dropna(how="all")
+
+        for idx, row in df.iterrows():
+            record_data = {}
+
+            for col in df.columns:
+                if hasattr(model, col):
+                    val = row[col]
+                    if pd.isna(val) or str(val).strip() in ["", "-", "—", "nan"]:
+                        val = None
+                    record_data[col] = val
+
+            record = model(RA_Bill_No=RA_Bill_No, **record_data)
+            db.session.add(record)
 
-        

app/services/user_service.py

@@ -1,47 +1,27 @@
-# from app.models.user_model import User
-# from app.services.db_service import db
-# import logging
-
-# class UserService:
-
-#     @staticmethod
-#     def register_user(name, email, password):
-#         user = User(name=name, email=email)
-#         user.set_password(password)
-#         db.session.add(user)
-#         db.session.commit()
-#         logging.info(f"New user registered: {email}")
-#         return user
-
-#     @staticmethod
-#     def validate_login(email, password):
-#         user = User.query.filter_by(email=email).first()
-#         if user and user.check_password(password):
-#             logging.info(f"Login success: {email}")
-#             return user
-#         logging.warning(f"Login failed for: {email}")
-#         return None
-
-#     @staticmethod
-#     def get_all_users():
-#         return User.query.all()
-
-
-from app.services.db_service import DBService
 from app.models.user_model import User
+from app.services.db_service import db
 
 class UserService:
 
-    def get_all_users(self):
-        db = DBService().connect()
-        cursor = db.cursor(dictionary=True)
+    @staticmethod
+    def register_user(name, email, password):
+        if User.query.filter_by(email=email).first():
+            return None
 
-        cursor.execute("SELECT id, name, email FROM users")
-        rows = cursor.fetchall()
+        user = User(name=name, email=email)
+        user.set_password(password)
 
-        users = [User(**row) for row in rows]
+        db.session.add(user)
+        db.session.commit()
+        return user
 
-        cursor.close()
-        db.close()
+    @staticmethod
+    def validate_login(email, password):
+        user = User.query.filter_by(email=email).first()
+        if user and user.check_password(password):
+            return user
+        return None
 
-        return users
+    @staticmethod
+    def get_all_users():
+        return User.query.all()

app/static/css/subcontractor.css

@@ -1,35 +0,0 @@
-body {
-    background: #f5f7fa;
-    font-family: Arial;
-    padding: 40px;
-}
-
-form {
-    width: 420px;
-    padding: 20px;
-    background: #fff;
-    border-radius: 8px;
-    box-shadow: 0 0 10px rgba(0, 0, 0, 0.1);
-}
-
-label {
-    margin-top: 10px;
-    display: block;
-    font-weight: bold;
-}
-
-input,
-textarea {
-    width: 100%;
-    padding: 8px;
-    margin-top: 4px;
-}
-
-button {
-    margin-top: 20px;
-    padding: 10px 20px;
-    background: #0055ff;
-    color: #fff;
-    border: 0;
-    cursor: pointer;
-}

app/templates/base.html

@@ -3,62 +3,211 @@
 
 <head>
     <meta charset="UTF-8">
-    <meta name="viewport" content="width=device-width, initial-scale=1.0">
-    <title>{{ title if title else "ComparisonSoftware" }}</title>
+    <meta name="viewport" content="width=device-width, initial-scale=1">
+    <title>{{ title if title else "Comparison Software" }}</title>
 
+    <!-- Bootstrap CSS -->
     <link href="https://cdn.jsdelivr.net/npm/bootstrap@5.3.0/dist/css/bootstrap.min.css" rel="stylesheet">
-    <link rel="stylesheet" href="{{ url_for('static', filename='css/style.css') }}">
+
+    <!-- Bootstrap Icons -->
+    <link href="https://cdn.jsdelivr.net/npm/bootstrap-icons/font/bootstrap-icons.css" rel="stylesheet">
 </head>
 
-<body style="background:#f5f7fa;">
+<body class="bg-light">
+
+    <!-- NAVBAR -->
+    <!-- <nav class="navbar navbar-expand-lg navbar-dark bg-dark shadow-sm"> -->
+    <nav class="navbar navbar-expand-lg navbar-dark bg-dark shadow-sm fixed-top">
 
-    <nav class="navbar navbar-expand-lg navbar-dark bg-dark">
         <div class="container-fluid">
-            <a class="navbar-brand" href="/">LCEPL</a>
 
+            <!-- Brand -->
+            <a class="navbar-brand fw-bold" href="/">
+                <i class="bi bi-buildings me-1"></i> LCEPL
+            </a>
+
+            <!-- Mobile toggle -->
             <button class="navbar-toggler" type="button" data-bs-toggle="collapse" data-bs-target="#navMenu">
                 <span class="navbar-toggler-icon"></span>
             </button>
 
+            <!-- Menu -->
             <div class="collapse navbar-collapse" id="navMenu">
-                <ul class="navbar-nav ms-auto">
+                <ul class="navbar-nav ms-auto align-items-lg-center">
 
-                    <li class="nav-item"><a class="nav-link" href="/dashboard">Dashboard</a></li>
-                    <li class="nav-item"><a class="nav-link" href="/file/import">File Import</a></li>
-                    <!-- <li class="nav-item"><a class="nav-link" href="/user/list">Users</a></li> -->
-                    <!-- <li class="nav-item"><a class="nav-link" href="/subcontractor/add">Subcontractor</a></li> -->
+                    <!-- Dashboard -->
+                    <li class="nav-item">
+                        <a class="nav-link" href="/dashboard">
+                            <i class="bi bi-speedometer2 me-1"></i> Dashboard - Anish
+                        </a>
+                    </li>
+
+                    <!-- Subcontractor Model -->
                     <li class="nav-item dropdown">
-                        <a class="nav-link dropdown-toggle" data-bs-toggle="dropdown" href="#">Subcontractor</a>
-
+                        <a class="nav-link dropdown-toggle" data-bs-toggle="dropdown" href="#">
+                            <i class="bi bi-people-fill me-1"></i> Subcontractor Model
+                        </a>
                         <ul class="dropdown-menu dropdown-menu-dark">
-                            <li><a class="dropdown-item" href="/subcontractor/add">Add Subcontractor</a></li>
-                            <li><a class="dropdown-item" href="/subcontractor/list">Subcontractor List</a></li>
+                            <li>
+                                <a class="dropdown-item" href="/subcontractor/add">
+                                    <i class="bi bi-plus-circle me-2"></i> Add Subcontractor
+                                </a>
+                            </li>
+                            <li>
+                                <a class="dropdown-item" href="/subcontractor/list">
+                                    <i class="bi bi-list-ul me-2"></i> Subcontractor List
+                                </a>
+                            </li>
                         </ul>
                     </li>
-                    <li class="nav-item"><a class="nav-link" href="/user/list">Users</a></li>
+
+                    <!-- Subcontractor File System -->
+                    <li class="nav-item dropdown">
+                        <a class="nav-link dropdown-toggle" data-bs-toggle="dropdown" href="#">
+                            <i class="bi bi-folder-fill me-1"></i>Subcontractor File System
+                        </a>
+                        <ul class="dropdown-menu dropdown-menu-dark">
+                            <li>
+                                <a class="dropdown-item" href="/file/import_Subcontractor">
+                                    <i class="bi bi-upload me-2"></i> Import File
+                                </a>
+                            </li>
+                            <li>
+                                <a class="dropdown-item" href="/file/Subcontractor_report">
+                                    <i class="bi bi-download me-2"></i> Show Reports
+                                </a>
+                            </li>
+
+                            <li>
+                                <a class="dropdown-item" href="/dashboard/subcontractor_dashboard">
+                                    <i class="bi bi-speedometer2 me-2"></i> Subcontractor Dashboard
+                                </a>
+                            </li>
+
+                        </ul>
+                    </li>
+
+                    <!-- Client System -->
+                    <li class="nav-item dropdown">
+                        <a class="nav-link dropdown-toggle" data-bs-toggle="dropdown" href="#">
+                            <i class="bi bi-building me-1"></i> Client File System
+                        </a>
+                        <ul class="dropdown-menu dropdown-menu-dark">
+                            <li>
+                                <a class="dropdown-item" href="/file/import_client">
+                                    <i class="bi bi-upload me-2"></i> Import Client File
+                                </a>
+                            </li>
+
+                            <li>
+                                <a class="dropdown-item" href="/file/client_report">
+                                    <i class="bi bi-arrow-left-right me-2"></i> Show Report
+                                </a>
+                            </li>
+                        </ul>
+                    </li>
+
+                    <!-- Reports -->
+                    <li class="nav-item dropdown">
+                        <a class="nav-link dropdown-toggle" data-bs-toggle="dropdown" href="#">
+                            <i class="bi bi-building me-1"></i> Reports
+                        </a>
+                        <ul class="dropdown-menu dropdown-menu-dark">
+
+                            <li>
+                                <a class="dropdown-item" href="/report/comparison_report">
+                                    <i class="bi bi-arrow-left-right me-2"></i> client vs sub-cont. Comparison Report
+                                </a>
+                            </li>
+                            <!-- <li>
+                                <a class="dropdown-item" href="/file/client_vs_subcont">
+                                    <i class="bi bi-arrow-left-right me-2"></i> Comparison Report
+                                </a>
+                            </li> -->
+                        </ul>
+                    </li>
+
+
+                    <!-- Formats -->
+                    <li class="nav-item">
+                        <a class="nav-link" href="/file_format">
+                            <i class="bi bi-file-earmark-text me-1"></i> Formats
+                        </a>
+                    </li>
+
+                    <!-- USER DROPDOWN -->
+                    {% if session.get("user_id") %}
+                    <li class="nav-item dropdown ms-lg-3">
+
+                        <a class="nav-link dropdown-toggle d-flex align-items-center gap-2" href="#"
+                            data-bs-toggle="dropdown">
+                            <i class="bi bi-person-circle fs-5"></i>
+                            <span class="d-none d-lg-inline">
+                                {{ session.get("user_name") }}
+                            </span>
+                        </a>
+
+                        <ul class="dropdown-menu dropdown-menu-end dropdown-menu-dark shadow">
+
+                            <!-- User card -->
+                            <li class="px-3 py-3 text-center border-bottom">
+                                <i class="bi bi-person-circle fs-1"></i>
+                                <div class="fw-semibold mt-1">
+                                    {{ session.get("user_name") }}
+                                </div>
+                                <small class="text-muted">Logged in user</small>
+                            </li>
+
+                            <li>
+                                <a class="dropdown-item" href="/dashboard">
+                                    <i class="bi bi-speedometer2 me-2"></i> Dashboard
+                                </a>
+                            </li>
+
+                            <li>
+                                <a class="dropdown-item text-warning" href="/logout">
+                                    <i class="bi bi-box-arrow-right me-2"></i> Logout
+                                </a>
+                            </li>
+
+                        </ul>
+                    </li>
+                    {% endif %}
 
                 </ul>
             </div>
         </div>
     </nav>
 
-    <div class="container mt-3">
-        {% with messages = get_flashed_messages(with_categories=true) %}
-        {% for category, message in messages %}
-        <div class="alert alert-{{ category }} alert-dismissible fade show">
-            {{ message }}
-            <button class="btn-close" data-bs-dismiss="alert"></button>
+
+    <!-- PAGE CONTENT -->
+    <div class="container-fluid vh-100 pt-5 overflow-hidden">
+        <!-- FLASH MESSAGES -->
+        <div class="container mt-3">
+            {% with messages = get_flashed_messages(with_categories=true) %}
+            {% if messages %}
+            {% for category, message in messages %}
+            <div class="alert alert-{{ category }} alert-dismissible fade show">
+                {{ message }}
+                <button class="btn-close" data-bs-dismiss="alert"></button>
+            </div>
+            {% endfor %}
+            {% endif %}
+            {% endwith %}
         </div>
-        {% endfor %}
-        {% endwith %}
-    </div>
 
-    <div class="container mt-4">
-        {% block content %}{% endblock %}
+
+        <div class="overflow-auto h-100">
+            <div class="container mt-4">
+                {% block content %}{% endblock %}
+            </div>
+        </div>
     </div>
 
 
+    <!-- Bootstrap JS -->
     <script src="https://cdn.jsdelivr.net/npm/bootstrap@5.3.0/dist/js/bootstrap.bundle.min.js"></script>
+
 </body>
 
 </html>

app/templates/client_report.html

@@ -0,0 +1,75 @@
+{% extends "base.html" %}
+
+{% block content %}
+<div class="container-fluid mt-4">
+    <h2 class="mb-4">Client File Reports</h2>
+
+    <div class="card p-4 shadow-sm mb-5">
+        <form method="POST">
+            <label class="form-label fw-bold">RA Bill No</label>
+            <input type="text" name="RA_Bill_No" class="form-control mb-3" value="{{ ra_val }}" required>
+
+            <div class="row">
+                <div class="col-md-6">
+                    <button type="submit" name="action" value="preview" class="btn btn-secondary w-100">Preview
+                        Data</button>
+                </div>
+                <div class="col-md-6">
+                    <button type="submit" name="action" value="download" class="btn btn-primary w-100">Download Excel
+                        Report Anish anassdfsdfdfasfd Prajakta</button>
+                </div> 
+            </div>
+        </form>
+    </div>
+
+
+    {% if tables.tr or tables.mh or tables.dc or tables.laying %}
+    <div class="card shadow-sm p-3">
+        <h4 class="mb-3">Comparison Preview</h4>
+
+        <ul class="nav nav-tabs" id="reportTabs" role="tablist">
+            <li class="nav-item">
+                <button class="nav-link active" id="tr-tab" data-bs-toggle="tab" data-bs-target="#tr"
+                    type="button">Tr.Ex Comparison</button>
+            </li>
+            <li class="nav-item">
+                <button class="nav-link" id="mh-tab" data-bs-toggle="tab" data-bs-target="#mh" type="button">Mh.Ex
+                    Comparison</button>
+            </li>
+            <li class="nav-item">
+                <button class="nav-link" id="dc-tab" data-bs-toggle="tab" data-bs-target="#dc" type="button">MH & DC
+                    Comparison</button>
+            </li>
+            <li class="nav-item">
+                <button class="nav-link" id="laying-tab" data-bs-toggle="tab" data-bs-target="#laying"
+                    type="button">Laying
+                    & Bedding Comparison</button>
+        </ul>
+
+        <div class="tab-content mt-3" id="reportTabsContent">
+            <div class="tab-pane fade show active" id="tr" role="tabpanel">
+                <div class="table-responsive" style="max-height: 500px;">
+                    {{ tables.tr|safe }}
+                </div>
+            </div>
+            <div class="tab-pane fade" id="mh" role="tabpanel">
+                <div class="table-responsive" style="max-height: 500px;">
+                    {{ tables.mh|safe }}
+                </div>
+            </div>
+            <div class="tab-pane fade" id="dc" role="tabpanel">
+                <div class="table-responsive" style="max-height: 500px;">
+                    {{ tables.dc|safe }}
+                </div>
+            </div>
+            <div class="tab-pane fade" id="laying" role="tabpanel">
+                <div class="table-responsive" style="max-height: 500px;">
+                    {{ tables.laying|safe }}
+                </div>
+
+            </div>
+        </div>
+        {% endif %}
+
+    </div>
+    {% endblock %}

app/templates/dashboard.html

@@ -1,10 +1,375 @@
 {% extends "base.html" %}
 
 {% block content %}
-<h2 class="mb-4">Dashboard</h2>
+<div class="container-fluid py-4" style="background-color: #f8f9fa;">
+    <h3 class="mb-4 fw-bold text-uppercase">Abstract Excavation Dashboard</h3>
 
-<div class="card p-4 shadow-sm">
-    <h5>Welcome to Comparison Project</h5>
-    <p>This is dashboard panel.</p>
+    <div class="card shadow-sm mb-4">
+        <div class="card-body bg-white">
+            <div class="row g-3">
+                <div class="col-md-2">
+                    <label class="form-label fw-bold">Comparison Type</label>
+                    <select id="filter-table" class="form-select" onchange="loadDashboardData()">
+                        <option value="trench">Trench Excavation</option>
+                        <option value="manhole">Manhole Excavation</option>
+                        <option value="laying">Laying</option>
+                    </select>
+                </div>
+                <div class="col-md-3">
+                    <label class="form-label fw-bold">Subcontractor</label>
+                    <select id="filter-subcon" class="form-select" onchange="loadDashboardData()">
+                        <option value="All">All Subcontractors</option>
+                    </select>
+                </div>
+                <div class="col-md-3">
+                    <label class="form-label fw-bold">RA Bill No</label>
+                    <select id="filter-ra" class="form-select" onchange="loadDashboardData()">
+                        <option value="Cumulative">Cumulative (All Bills)</option>
+                    </select>
+                </div>
+                <div class="col-md-4 d-flex align-items-end gap-2">
+                    <button class="btn btn-primary flex-grow-1" onclick="loadDashboardData()">🔄 Refresh</button>
+                    <button class="btn btn-secondary flex-grow-1" onclick="clearDashboard()">🗑️ Clear</button>
+                </div>
+            </div>
+        </div>
+    </div>
+
+    <!-- Empty State (Shown on page load) -->
+    <div id="empty-state" class="alert alert-info text-center py-5">
+        <h5>📊 Select filters to display data</h5>
+        <p>Choose a Subcontractor and/or RA Bill to see the excavation abstract comparison.</p>
+    </div>
+
+    <!-- Data Display Area (Hidden by default) -->
+    <div id="data-area" style="display: none;">
+        <div class="row">
+            <div class="col-lg-8">
+                <div class="card shadow-sm h-100">
+                    <div class="card-header bg-primary text-white fw-bold d-flex justify-content-between align-items-center">
+                        <span id="chart-title">Excavation Comparison: Client vs Subcontractor Qty</span>
+                        <small class=\"fw-normal\">(Horizontal Bar Chart)</small>
+                    </div>
+                    <div class=\"card-body\" style=\"position: relative; height: 700px; overflow-y: auto;\">
+                        <canvas id="groupedBarChart"></canvas>
+                    </div>
+                </div>
+            </div>
+
+            <div class="col-lg-4">
+                <div class="card shadow-sm h-100">
+                    <div class="card-header bg-success text-white fw-bold">Excavation Abstract Table</div>
+                    <div class="card-body p-0">
+                        <div class="table-responsive" style="max-height: 500px; overflow-y: auto;">
+                            <table class="table table-hover mb-0" id="abstract-table">
+                                <thead class="table-light sticky-top">
+                                    <tr>
+                                        <th class="small">Soil / Depth</th>
+                                        <th class="small">Client (m³)</th>
+                                        <th class="small">Subcon (m³)</th>
+                                        <th class="small">Diff</th>
+                                    </tr>
+                                </thead>
+                                <tbody></tbody>
+                                <tfoot class="table-light fw-bold position-sticky bottom-0">
+                                    <tr id="table-totals" class="bg-light"></tr>
+                                </tfoot>
+                            </table>
+                        </div>
+                    </div>
+                </div>
+            </div>
+        </div>
+    </div>
 </div>
-{% endblock %}
+
+<script src="https://cdn.jsdelivr.net/npm/chart.js"></script>
+
+<script>
+let comparisonChart;
+
+// Define color palette - ONLY 2 COLORS
+const colorPalette = {
+    'client': '#003D7A',      // Dark Blue for Client (RA Bill)
+    'subcon': '#87CEEB'       // Light Sky Blue for Subcontractor
+};
+
+// 1. Function to Initialize or Update the Chart (VERTICAL BARS - 2 COLORS ONLY)
+function updateChartUI(labels, clientData, subconData) {
+    const ctx = document.getElementById('groupedBarChart').getContext('2d');
+    if (comparisonChart) comparisonChart.destroy();
+
+    comparisonChart = new Chart(ctx, {
+        type: 'bar', // Vertical bar chart
+        data: {
+            labels: labels,
+            datasets: [
+                { 
+                    label: 'Client Qty (m³)',
+                    data: clientData,
+                    backgroundColor: colorPalette.client,
+                    borderColor: '#001F4D',
+                    borderWidth: 1,
+                    borderRadius: 4,
+                    hoverBackgroundColor: '#002A5C',
+                    hoverBorderWidth: 2
+                },
+                { 
+                    label: 'Subcontractor Qty (m³)',
+                    data: subconData,
+                    backgroundColor: colorPalette.subcon,
+                    borderColor: '#4A90B8',
+                    borderWidth: 1,
+                    borderRadius: 4,
+                    hoverBackgroundColor: '#6BB3D9',
+                    hoverBorderWidth: 2
+                }
+            ]
+        },
+        options: {
+            indexAxis: 'x', // Vertical bars (default)
+            responsive: true,
+            maintainAspectRatio: false,
+            interaction: {
+                intersect: false,
+                mode: 'index'
+            },
+            plugins: {
+                legend: {
+                    position: 'top',
+                    labels: {
+                        font: { size: 14, weight: 'bold' },
+                        padding: 15,
+                        usePointStyle: true,
+                        boxWidth: 15
+                    }
+                },
+                tooltip: {
+                    backgroundColor: '#2c3e50',
+                    padding: 12,
+                    titleFont: { size: 13, weight: 'bold' },
+                    bodyFont: { size: 12 },
+                    borderColor: '#fff',
+                    borderWidth: 1,
+                    displayColors: true,
+                    callbacks: {
+                        label: function(context) {
+                            let label = context.dataset.label || '';
+                            if (label) label += ': ';
+                            label += Number(context.parsed.y).toLocaleString('en-IN', {
+                                minimumFractionDigits: 2,
+                                maximumFractionDigits: 2
+                            }) + ' m³';
+                            return label;
+                        }
+                    }
+                }
+            },
+            scales: {
+                x: {
+                    stacked: false,
+                    grid: {
+                        display: false
+                    },
+                    ticks: {
+                        font: { size: 11 },
+                        maxRotation: 45,
+                        minRotation: 0
+                    }
+                },
+                y: {
+                    stacked: false,
+                    beginAtZero: true,
+                    grid: {
+                        color: '#ecf0f1',
+                        drawBorder: false
+                    },
+                    ticks: {
+                        font: { size: 11 },
+                        callback: function(value) {
+                            return Number(value).toLocaleString('en-IN');
+                        }
+                    },
+                    title: {
+                        display: true,
+                        text: 'Excavation Quantity (m³)',
+                        font: { size: 12, weight: 'bold' }
+                    }
+                }
+            }
+        }
+    });
+}
+
+// 2. Function to fetch unique filters (Subcontractors & RA Bills) from DB
+function loadFilters() {
+    console.log("🔄 Loading filters from /dashboard/api/filters...");
+    fetch('/dashboard/api/filters')
+        .then(res => {
+            console.log(`Response status: ${res.status}`);
+            return res.json();
+        })
+        .then(data => {
+            console.log("✓ Filter data received:", data);
+            
+            const raSelect = document.getElementById('filter-ra');
+            
+            // CRITICAL: This clears the "RA-01", "RA-02" you typed in manually
+            raSelect.innerHTML = '<option value="Cumulative">Cumulative (All Bills)</option>';
+
+            if (data.ra_bills && data.ra_bills.length > 0) {
+                console.log(`Adding ${data.ra_bills.length} RA bills to dropdown`);
+                data.ra_bills.forEach(billNo => {
+                    let opt = document.createElement('option');
+                    opt.value = billNo;
+                    opt.innerText = billNo; // This will show exactly what's in the DB
+                    raSelect.appendChild(opt);
+                    console.log(`  + Added RA Bill: ${billNo}`);
+                });
+            } else {
+                console.warn("❌ No RA bills found in response");
+            }
+            
+            // Repeat same for subcontractor dropdown
+            const subconSelect = document.getElementById('filter-subcon');
+            subconSelect.innerHTML = '<option value="All">All Subcontractors</option>';
+            if (data.subcontractors && data.subcontractors.length > 0) {
+                data.subcontractors.forEach(name => {
+                    let opt = document.createElement('option');
+                    opt.value = name;
+                    opt.innerText = name;
+                    subconSelect.appendChild(opt);
+                });
+            }
+            console.log("✓ Filters loaded successfully");
+        })
+        .catch(err => {
+            console.error("❌ Error loading filters:", err);
+        });
+}
+// 3. Main function to load data and reflect in UI
+function loadDashboardData() {
+    const tableType = document.getElementById('filter-table').value;
+    const subcon = document.getElementById('filter-subcon').value;
+    const ra = document.getElementById('filter-ra').value;
+
+    console.log(`📊 Filter values: Table="${tableType}", Subcon="${subcon}", RA="${ra}"`);
+
+    // If still on default values, don't load
+    if (subcon === 'All' && ra === 'Cumulative') {
+        console.warn("⚠️  Please select filters first");
+        return;
+    }
+
+    // Update chart title
+    const tableNames = {
+        'trench': 'Trench Excavation',
+        'manhole': 'Manhole Excavation',
+        'laying': 'Laying'
+    };
+    const chartTitle = document.getElementById('chart-title');
+    if (chartTitle) {
+        chartTitle.textContent = `${tableNames[tableType]}: Client (RA Bill) vs Subcontractor Qty`;
+    }
+
+    console.log(`📊 Loading dashboard data: Table="${tableType}", Subcon="${subcon}", RA="${ra}"`);
+    
+    const url = `/dashboard/api/excavation-abstract?table_type=${encodeURIComponent(tableType)}&subcontractor=${encodeURIComponent(subcon)}&ra_bill=${encodeURIComponent(ra)}`;
+    console.log(`Fetching from URL: ${url}`);
+    
+    fetch(url)
+        .then(res => {
+            console.log(`Response status: ${res.status}`);
+            if (!res.ok) {
+                throw new Error(`HTTP Error: ${res.status}`);
+            }
+            return res.json();
+        })
+        .then(data => {
+            console.log("✓ Dashboard data received:", data);
+            
+            if (!Array.isArray(data)) {
+                console.error("❌ Response is not an array:", data);
+                return;
+            }
+            
+            if (data.length === 0) {
+                console.warn("⚠️  No data returned for this filter combination");
+                alert("No data found for selected filters");
+                return;
+            }
+            
+            const labels = [];
+            const clientData = [];
+            const subconData = [];
+            const tableBody = document.querySelector("#abstract-table tbody");
+            tableBody.innerHTML = "";
+
+            let tClient = 0, tSub = 0, tDiff = 0;
+
+            data.forEach(item => {
+                // Label format: "Soil Type Depth"
+                const label = `${item.soil_type}\n${item.depth}`;
+                labels.push(label);
+                clientData.push(item.client_qty || 0);
+                subconData.push(item.subcon_qty || 0);
+
+                tClient += item.client_qty || 0;
+                tSub += item.subcon_qty || 0;
+                tDiff += (item.difference || 0);
+
+                const diffColor = (item.difference || 0) < 0 ? 'text-danger' : 'text-success';
+                tableBody.innerHTML += `
+                    <tr>
+                        <td class="small">
+                            <strong>${item.soil_type}</strong>
+                            <br>
+                            <span class="text-muted small">${item.depth}</span>
+                        </td>
+                        <td class="small text-primary fw-bold">${(item.client_qty || 0).toLocaleString('en-IN', {maximumFractionDigits: 2})}</td>
+                        <td class="small text-success fw-bold">${(item.subcon_qty || 0).toLocaleString('en-IN', {maximumFractionDigits: 2})}</td>
+                        <td class="small fw-bold ${diffColor}">${(item.difference || 0).toLocaleString('en-IN', {maximumFractionDigits: 2})}</td>
+                    </tr>
+                `;
+            });
+
+            const totalDiffColor = tDiff < 0 ? 'text-danger' : 'text-success';
+            document.getElementById('table-totals').innerHTML = `
+                <td class="small fw-bold">TOTAL</td>
+                <td class="small fw-bold text-primary">${tClient.toLocaleString('en-IN', {maximumFractionDigits: 2})}</td>
+                <td class="small fw-bold text-success">${tSub.toLocaleString('en-IN', {maximumFractionDigits: 2})}</td>
+                <td class="small fw-bold ${totalDiffColor}">${tDiff.toLocaleString('en-IN', {maximumFractionDigits: 2})}</td>
+            `;
+
+            // Show data area
+            document.getElementById('empty-state').style.display = 'none';
+            document.getElementById('data-area').style.display = 'block';
+
+            updateChartUI(labels, clientData, subconData);
+            console.log("✓ Chart and table updated successfully");
+        })
+        .catch(err => {
+            console.error("❌ Error loading dashboard data:", err);
+            alert(`Failed to load dashboard data: ${err.message}`);
+        });
+}
+
+// Clear dashboard
+function clearDashboard() {
+    console.log("🗑️  Clearing dashboard...");
+    document.getElementById('filter-table').value = 'trench';
+    document.getElementById('filter-subcon').value = 'All';
+    document.getElementById('filter-ra').value = 'Cumulative';
+    document.getElementById('empty-state').style.display = 'block';
+    document.getElementById('data-area').style.display = 'none';
+    if (comparisonChart) comparisonChart.destroy();
+}
+
+// Start: Load filters only, don't auto-load data
+document.addEventListener("DOMContentLoaded", () => {
+    console.log("🚀 Dashboard initialized");
+    loadFilters();
+    // Don't auto-load data - keep dashboard blank until filters selected
+});
+</script>
+{% endblock %}  

app/templates/file_format.html

@@ -0,0 +1,66 @@
+{% extends "base.html" %}
+
+{% block content %}
+<h2 class="mb-4">Download File Formats</h2>
+
+<div class="row g-4">
+
+    <!-- Subcontractor Format -->
+    <div class="col-md-4">
+        <div class="card text-center shadow-sm h-100">
+            <div class="card-body d-flex flex-column justify-content-center">
+                <div class="mb-3 fs-1 text-primary">
+                    ⬇️
+                </div>
+                <h5 class="card-title">Subcontractor Upload Format</h5>
+                <p class="text-muted mb-2">Excel (.xlsx)</p>
+                <p class="small text-secondary">File size: 245 KB</p>
+
+                <a href="{{ url_for('file_format.download_excel_format', filename='subcontractor_format.xlsx') }}"
+                    class="btn btn-outline-primary mt-auto">
+                    Download
+                </a>
+            </div>
+        </div>
+    </div>
+
+    <!-- Client Format -->
+    <div class="col-md-4">
+        <div class="card text-center shadow-sm h-100">
+            <div class="card-body d-flex flex-column justify-content-center">
+                <div class="mb-3 fs-1 text-success">
+                    ⬇️
+                </div>
+                <h5 class="card-title">Client Upload Format</h5>
+                <p class="text-muted mb-2">Excel (.xlsx)</p>
+                <p class="small text-secondary">File size: 310 KB</p>
+
+                <a href="{{ url_for('file_format.download_excel_format', filename='client_format.xlsx') }}"
+                    class="btn btn-outline-success mt-auto">
+                    Download
+                </a>
+            </div>
+        </div>
+    </div>
+
+    <!-- MH & DC Format -->
+    <!-- <div class="col-md-4">
+        <div class="card text-center shadow-sm h-100">
+            <div class="card-body d-flex flex-column justify-content-center">
+                <div class="mb-3 fs-1 text-warning">
+                    ⬇️
+                </div>
+                <h5 class="card-title">Manhole & DC Format</h5>
+                <p class="text-muted mb-2">Excel (.xlsx)</p>
+                <p class="small text-secondary">File size: 190 KB</p>
+
+                <a href="{{ url_for('file_format.download_excel_format', filename='mh_dc_format.xlsx') }}"
+                    class="btn btn-outline-warning mt-auto">
+                    Download
+                </a>
+            </div>
+        </div>
+    </div> -->
+
+</div>
+{% endblock %}

app/templates/file_import_client.html

@@ -1,33 +1,35 @@
 {% extends "base.html" %}
 
 {% block content %}
-<h2 class="mb-4">File Import</h2>
+<h2 class="mb-4">Client File Import</h2>
 
 <div class="card p-4 shadow-sm">
 
     <form method="POST" enctype="multipart/form-data">
 
         <!-- 1. SELECT SUBCONTRACTOR -->
-        <label class="form-label">Select Subcontractor</label>
+        <!-- <label class="form-label">Select Subcontractor vs Client</label>
         <select name="subcontractor_id" id="subcontractor_id" class="form-select mb-3" required>
             <option value="">-- Select Subcontractor --</option>
 
             {% for sc in subcontractors %}
             <option value="{{ sc.id }}">{{ sc.subcontractor_name }}</option>
             {% endfor %}
-        </select>
+        </select> -->
 
         <!-- 2. FILE TYPE (MODEL NAME) -->
-        <label class="form-label">Select File Type</label>
+        <!-- <label class="form-label">Select File Type</label>
         <select name="file_type" id="file_type" class="form-select mb-3" required>
             <option value="">-- Select File Type --</option>
-            <option value="">Subcontractor Sheet</option>
-            <option value="trench_excavation">Trench Excavation (Tr.Ex)</option>
-            <option value="manhole_excavation">Manhole Excavation (Mh.Ex)</option>
-            <option value="manhole_domestic_chamber">Manhole & Domestic Chambers Construction (MH & DC)</option>
+            <option value="">Sheet</option>
+            <option value="tr_ex_client">Tr. Ex</option>
+            <option value="mh_ex_client">Mh. Ex </option>
+            <option value="mh_dc_client">MH & DC </option>
             <option value="">Laying Sheet</option>
-        </select>
+        </select> -->
 
+        <label class="form-label">RA Bill No</label>
+        <input type="text" name="RA_Bill_No" class="form-control mb-3" required>
         <!-- 3. FILE UPLOAD -->
         <label class="form-label">Choose File</label>
         <input type="file" name="file" class="form-control mb-3" required>

app/templates/file_import_subcontractor.html

@@ -0,0 +1,32 @@
+{% extends "base.html" %}
+
+{% block content %}
+<h2 class="mb-4">Sub-Contractor File Import</h2>
+
+<div class="card p-4 shadow-sm">
+
+    <form method="POST" enctype="multipart/form-data">
+
+        <!-- 1. SELECT SUBCONTRACTOR -->
+        <label class="form-label">Select Subcontractor</label>
+        <select name="subcontractor_id" id="subcontractor_id" class="form-select mb-3" required>
+            <option value="">-- Select Subcontractor --</option>
+
+            {% for sc in subcontractors %}
+            <option value="{{ sc.id }}">{{ sc.subcontractor_name }}</option>
+            {% endfor %}
+        </select>
+
+        <!-- 2. RA bill no -->
+        <label class="form-label">RA Bill No</label>
+        <input type="text" name="RA_Bill_No" class="form-control mb-3" required>
+
+        <!-- 3. FILE UPLOAD -->
+        <label class="form-label">Choose File</label>
+        <input type="file" name="file" class="form-control mb-3" required>
+
+        <button class="btn btn-primary w-100">Upload</button>
+    </form>
+
+</div>
+{% endblock %}

app/templates/generate_comparison_report.html

@@ -0,0 +1,39 @@
+{% extends "base.html" %}
+
+{% block content %}
+<div class="container mt-4">
+
+    <h2 class="mb-4">Subcontractor vs Client Comparison</h2>
+
+    <!-- FLASH MESSAGES -->
+    {% with messages = get_flashed_messages(with_categories=true) %}
+    {% if messages %}
+    {% for category, message in messages %}
+    <div class="alert alert-{{ category }} alert-dismissible fade show" role="alert">
+        {{ message }}
+        <button type="button" class="btn-close" data-bs-dismiss="alert"></button>
+    </div>
+    {% endfor %}
+    {% endif %}
+    {% endwith %}
+
+    <div class="card p-4 shadow-sm">
+
+        <form method="POST">
+
+            <!-- SELECT SUBCONTRACTOR -->
+            <label class="form-label fw-semibold">Select Subcontractor</label>
+            <select name="subcontractor_id" id="subcontractor_id" class="form-select mb-3" required>
+                <option value="">-- Select Subcontractor --</option>
+
+                {% for sc in subcontractors %}
+                <option value="{{ sc.id }}">{{ sc.subcontractor_name }}</option>
+                {% endfor %}
+            </select>
+
+            <button class="btn btn-primary w-100">Generate Report</button>
+        </form>
+
+    </div>
+</div>
+{% endblock %}

app/templates/login.html

@@ -1,17 +1,82 @@
-{% extends "base.html" %}
-{% block content %}
-<h2 class="mb-4">Login</h2>
+<!DOCTYPE html>
+<html lang="en">
 
-<div class="card p-4 shadow-sm">
-    <form method="POST">
+<head>
+    <meta charset="UTF-8">
+    <title>LCEPL | Login</title>
+    <meta name="viewport" content="width=device-width, initial-scale=1">
 
-        <label>Email:</label>
-        <input type="email" name="email" class="form-control mb-3" required>
+    <!-- Bootstrap CSS -->
+    <link href="https://cdn.jsdelivr.net/npm/bootstrap@5.3.0/dist/css/bootstrap.min.css" rel="stylesheet">
+</head>
 
-        <label>Password:</label>
-        <input type="password" name="password" class="form-control mb-3" required>
+<body class="bg-light">
 
-        <button class="btn btn-success">Login</button>
-    </form>
-</div>
-{% endblock %}
+    <div class="container-fluid vh-100">
+        <div class="row h-100 justify-content-center align-items-center">
+
+            <!-- Increased column width -->
+            <div class="col-12 col-sm-10 col-md-8 col-lg-5 col-xl-4">
+
+                <div class="card shadow-lg border-0">
+                    <!-- Increased padding -->
+                    <div class="card-body p-5">
+
+                        <!-- Branding -->
+                        <div class="text-center mb-4">
+                            <img src="{{ url_for('static', filename='images/lcepl.png') }}" alt="LCEPL Logo"
+                                class="img-fluid mb-3" style="max-height:80px;">
+
+                            <h4 class="fw-bold mb-1">
+                                Laxmi Civil Engineering Services Pvt Ltd
+                            </h4>
+                            <p class="text-muted mb-0">
+                                Data Comparison Software Solapur(UGD)
+                            </p>
+                        </div>
+
+                        <!-- Flash messages -->
+                        {% with messages = get_flashed_messages(with_categories=true) %}
+                        {% if messages %}
+                        {% for category, message in messages %}
+                        <div class="alert alert-{{ category }} alert-dismissible fade show" role="alert">
+                            {{ message }}
+                            <button type="button" class="btn-close" data-bs-dismiss="alert"></button>
+                        </div>
+                        {% endfor %}
+                        {% endif %}
+                        {% endwith %}
+
+                        <!-- Login Form -->
+                        <form method="POST">
+
+                            <div class="mb-4">
+                                <label class="form-label fw-semibold">User Name</label>
+                                <input type="email" name="email" class="form-control " placeholder="Enter email"
+                                    required>
+                            </div>
+
+                            <div class="mb-4">
+                                <label class="form-label fw-semibold">Password</label>
+                                <input type="password" name="password" class="form-control" placeholder="Enter password"
+                                    required>
+                            </div>
+
+                            <button class="btn btn-success btn-lg w-100">
+                                Login
+                            </button>
+                        </form>
+
+                    </div>
+                </div>
+
+            </div>
+
+        </div>
+    </div>
+
+    <!-- Bootstrap JS -->
+    <script src="https://cdn.jsdelivr.net/npm/bootstrap@5.3.0/dist/js/bootstrap.bundle.min.js"></script>
+</body>
+
+</html>

app/templates/register.html

@@ -1,18 +1,17 @@
 {% extends "base.html" %}
 {% block content %}
-<h2 class="mb-4">User Registration</h2>
+<h2>Register User</h2>
 
 <div class="card p-4 shadow-sm">
     <form method="POST">
+        <label>Name</label>
+        <input class="form-control mb-3" name="name" required>
 
-        <label>Name:</label>
-        <input type="text" name="name" class="form-control mb-3" required>
+        <label>Email</label>
+        <input type="email" class="form-control mb-3" name="email" required>
 
-        <label>Email:</label>
-        <input type="email" name="email" class="form-control mb-3" required>
-
-        <label>Password:</label>
-        <input type="password" name="password" class="form-control mb-3" required>
+        <label>Password</label>
+        <input type="password" class="form-control mb-3" name="password" required>
 
         <button class="btn btn-primary">Register</button>
     </form>

app/templates/subcontractor/add.html

@@ -13,7 +13,7 @@
         </div>
 
         <div class="mb-3">
-            <label class="form-label">Contact Person</label>
+            <label class="form-label">Contact Person Name</label>
             <input type="text" class="form-control" name="contact_person">
         </div>
 

app/templates/subcontractor/list.html

@@ -1,38 +1,91 @@
 {% extends "base.html" %}
 {% block content %}
 
-<div class="card shadow-sm p-4">
-    <h4 class="mb-3">Subcontractor List</h4>
+<div class="card shadow-sm p-3 p-md-4">
 
-    <table class="table table-bordered table-striped">
-        <thead class="table-dark">
-            <tr>
-                <th>ID</th>
-                <th>Name</th>
-                <th>Mobile</th>
-                <th>Email</th>
-                <th>GST No</th>
-                <th>Action</th>
-            </tr>
-        </thead>
+    <!-- Header -->
+    <div class="row mb-3 align-items-center">
+        <div class="col-12 col-md-6 ">
+            <h4 class="mb-2 mb-md-0 text-center text-md-start">
+                Subcontractor List
+            </h4>
+        </div>
+
+        <div class="col-12 col-md-3 text-center text-md-end">
+            <a href="/subcontractor/add" class="btn btn-success w-100 w-md-auto">
+                ➕ Add Subcontractor
+            </a>
+        </div>
+    </div>
+
+
+    <div class="table-responsive">
+        <table class="table table-bordered table-striped align-middle text-nowrap">
+            <thead class="table-dark">
+                <tr>
+                    <th>ID</th>
+                    <th>Name</th>
+                    <th class="d-none d-md-table-cell">Mobile</th>
+                    <th class="d-none d-lg-table-cell">Email</th>
+                    <th class="d-none d-lg-table-cell">GST No</th>
+                    <th class="text-center">Action</th>
+                </tr>
+            </thead>
+
+            <tbody>
+                {% for s in subcontractors %}
+                <tr>
+                    <td>{{ s.id }}</td>
+
+                    <td class="fw-semibold text-wrap">
+                        {{ s.subcontractor_name }}
+                    </td>
+
+                    <td class="d-none d-md-table-cell">
+                        {{ s.mobile_no }}
+                    </td>
+
+                    <td class="d-none d-lg-table-cell">
+                        {{ s.email_id }}
+                    </td>
+
+                    <td class="d-none d-lg-table-cell">
+                        {{ s.gst_no }}
+                    </td>
+
+                    <!-- Action Buttons -->
+                    <td class="text-center">
+                        <div class="d-flex flex-column gap-1">
+                            <a href="/subcontractor/edit/{{ s.id }}" class="btn btn-sm btn-warning">
+                                Edit
+                            </a>
+                            <a href="/subcontractor/delete/{{ s.id }}" class="btn btn-sm btn-danger"
+                                onclick="return confirm('Are you sure?')">
+                                Delete
+                            </a>
+                        </div>
+                    </td>
+                </tr>
+                {% endfor %}
+            </tbody>
+
+            <!-- TOTAL ROW -->
+            <tfoot>
+                <tr class="table-secondary fw-bold">
+                    <td colspan="2" class="text-end"> Total Subcontractors</td>
+
+                    <td class="d-none d-lg-table-cell">-</td>
+                    <td class="d-none d-lg-table-cell">-</td>
+                    <td class="d-none d-lg-table-cell">-</td>
+
+                    <td class="text-center">{{ subcontractors|length }}</td>
+
+                </tr>
+            </tfoot>
+
+        </table>
+    </div>
 
-        <tbody>
-            {% for s in subcontractors %}
-            <tr>
-                <td>{{ s.id }}</td>
-                <td>{{ s.subcontractor_name }}</td>
-                <td>{{ s.mobile_no }}</td>
-                <td>{{ s.email_id }}</td>
-                <td>{{ s.gst_no }}</td>
-                <td>
-                    <a href="/subcontractor/edit/{{ s.id }}" class="btn btn-sm btn-warning">Edit</a>
-                    <a href="/subcontractor/delete/{{ s.id }}" class="btn btn-sm btn-danger"
-                        onclick="return confirm('Are you sure?')">Delete</a>
-                </td>
-            </tr>
-            {% endfor %}
-        </tbody>
-    </table>
 </div>
 
 {% endblock %}

app/templates/subcontractor_dashboard.html

@@ -0,0 +1,29 @@
+{% extends "base.html" %}
+
+{% block content %}
+
+<div class="container-fluid px-2 px-md-4">
+
+    <h4 class="mb-3 text-center text-md-start">Subcontractor Dashboard </h4>
+
+    <!-- Charts -->
+    <div class="row g-3">
+
+        <!-- Bar Chart -->
+        <div class="col-12 col-md-6">
+            <div class="card shadow-sm h-100">
+                <div class="card-header bg-dark text-white text-center text-md-start">
+                    Work Category Bar Chart
+                </div>
+                <div class="card-body text-center">
+                    <img src="data:image/png;base64,{{ bar_chart }}" class="img-fluid" style="max-height:300px;">
+                </div>
+            </div>
+        </div>
+
+
+    </div>
+
+</div>
+
+{% endblock %}

app/templates/subcontractor_report.html

@@ -0,0 +1,142 @@
+{% extends "base.html" %}
+
+{% block content %}
+<div class="container my-4">
+
+    <h2 class="text-center mb-4">Generate Subcontractor Report</h2>
+
+    <!-- FORM -->
+    <div class="card shadow-sm p-3 p-md-4 mx-auto" style="max-width:600px;">
+        <form method="POST">
+
+            <div class="mb-3">
+                <label class="form-label fw-semibold">Select Subcontractor</label>
+                <select name="subcontractor_id" class="form-select" required>
+                    <option value="">-- Select Subcontractor --</option>
+                    {% for sc in subcontractors %}
+                    <option value="{{ sc.id }}" {% if selected_sc_id==sc.id|string %}selected{% endif %}>
+                        {{ sc.subcontractor_name }}
+                    </option>
+                    {% endfor %}
+                </select>
+            </div>
+
+            <div class="form-check form-switch mb-3">
+                <input class="form-check-input" type="checkbox" id="downloadAllSwitch" name="download_all" value="true"
+                    {% if download_all %}checked{% endif %}>
+                <label class="form-check-label fw-bold text-primary">
+                    Download All RA Bills
+                </label>
+            </div>
+
+            <div class="mb-4" id="ra_bill_container">
+                <label class="form-label fw-semibold">RA Bill Number</label>
+                <input type="text" name="ra_bill_no" id="ra_bill_input" class="form-control"
+                    value="{{ ra_bill_no or '' }}">
+            </div>
+
+            <div class="row g-2">
+                <div class="col-12 col-md-6">
+                    <button class="btn btn-outline-primary w-100" name="action" value="preview">
+                        Preview Data
+                    </button>
+                </div>
+                <div class="col-12 col-md-6">
+                    <button class="btn btn-primary w-100" name="action" value="download">
+                        Download Excel
+                    </button>
+                </div>
+            </div>
+
+        </form>
+    </div>
+
+    {% if tables %}
+    <!-- REPORT PREVIEW -->
+    <div class="mt-5">
+
+        <h3 class="text-center mb-3">Report Preview</h3>
+
+        <div class="card shadow-sm">
+
+            <!-- MOBILE SCROLLABLE TABS -->
+            <div class="card-header p-0">
+                <ul class="nav nav-tabs flex-nowrap overflow-auto">
+                    <li class="nav-item">
+                        <button class="nav-link active" data-bs-toggle="tab" data-bs-target="#tr">
+                            Trench Excavation
+                        </button>
+                    </li>
+                    <li class="nav-item">
+                        <button class="nav-link" data-bs-toggle="tab" data-bs-target="#mh">
+                            Manhole Excavation
+                        </button>
+                    </li>
+                    <li class="nav-item">
+                        <button class="nav-link" data-bs-toggle="tab" data-bs-target="#dc">
+                            Manhole & Domestic Chambers
+                        </button>
+                    </li>
+                    <li class="nav-item">
+                        <button class="nav-link" data-bs-toggle="tab" data-bs-target="#laying">
+                            Pipe Laying
+                        </button>
+                    </li>
+                </ul>
+            </div>
+
+            <!-- TAB CONTENT -->
+            <div class="card-body tab-content">
+
+                <div class="tab-pane fade show active" id="tr">
+                    <div class="table-responsive overflow-auto">
+                        {{ tables.tr | safe }}
+                    </div>
+                </div>
+
+                <div class="tab-pane fade" id="mh">
+                    <div class="table-responsive overflow-auto">
+                        {{ tables.mh | safe }}
+                    </div>
+                </div>
+
+                <div class="tab-pane fade" id="dc">
+                    <div class="table-responsive overflow-auto">
+                        {{ tables.dc | safe }}
+                    </div>
+                </div>
+
+                <div class="tab-pane fade" id="laying">
+                    <div class="table-responsive overflow-auto">
+                        {{ tables.laying | safe }}
+                    </div>
+                </div>
+
+            </div>
+        </div>
+    </div>
+    {% endif %}
+
+</div>
+
+<!-- BOOTSTRAP-ONLY JS -->
+<script>
+    function toggleRAInput() {
+        const checkbox = document.getElementById("downloadAllSwitch");
+        const input = document.getElementById("ra_bill_input");
+        const container = document.getElementById("ra_bill_container");
+
+        if (checkbox.checked) {
+            input.value = "";
+            input.disabled = true;
+            container.classList.add("opacity-50");
+        } else {
+            input.disabled = false;
+            container.classList.remove("opacity-50");
+        }
+    }
+
+    document.addEventListener("DOMContentLoaded", toggleRAInput);
+    document.getElementById("downloadAllSwitch").addEventListener("change", toggleRAInput);
+</script>
+{% endblock %}

app/utils/helpers.py

@@ -1,2 +1,14 @@
-def is_logged_in(session):
-    return session.get("user_id") is not None
+# def is_logged_in(session):
+#     return session.get("user_id") is not None
+
+from functools import wraps
+from flask import session, redirect, url_for, flash
+
+def login_required(view):
+    @wraps(view)
+    def wrapped_view(*args, **kwargs):
+        if "user_id" not in session:
+            flash("Please login first", "warning")
+            return redirect(url_for("auth.login"))
+        return view(*args, **kwargs)
+    return wrapped_view

app/utils/plot_utils.py

@@ -0,0 +1,10 @@
+import base64
+from io import BytesIO
+import matplotlib
+matplotlib.use("Agg")
+
+def plot_to_base64(plt):
+    img = BytesIO()
+    plt.savefig(img, format="png", bbox_inches="tight")
+    img.seek(0)
+    return base64.b64encode(img.read()).decode("utf-8")

docker-compose.yml

@@ -0,0 +1,46 @@
+version: '3.8'
+
+services:
+  db:
+    image: mysql:8.0
+    container_name: comparison_db
+    restart: always
+    environment:
+      MYSQL_ROOT_PASSWORD: admin
+      MYSQL_DATABASE: comparisondb
+    ports:
+      - "3307:3306"
+    volumes:
+      - mysql_data:/var/lib/mysql
+
+  app:
+    build: .
+    container_name: comparison_app
+    restart: always
+    environment:
+      FLASK_ENV: development
+      FLASK_DEBUG: "True"
+      FLASK_HOST: "0.0.0.0"
+      FLASK_PORT: "5001"
+
+      DB_DIALECT: mysql
+      DB_DRIVER: pymysql
+      DB_HOST: db
+      DB_PORT: 3306
+      DB_NAME: comparisondb
+      DB_USER: root
+      DB_PASSWORD: admin
+
+    ports:
+      - "5001:5001"
+
+    depends_on:
+      - db
+
+    volumes:
+      - ./app/logs:/app/app/logs
+      - ./app/static/uploads:/app/app/static/uploads
+      - ./app/static/downloads:/app/app/static/downloads
+
+volumes:
+  mysql_data:

requirements.txt

@@ -4,4 +4,8 @@ openpyxl
 xlrd
 Werkzeug
 python-dotenv
-cryptography
+cryptography
+xlsxwriter
+matplotlib 
+flask_sqlalchemy
+flask_migrate

run.py

@@ -1,9 +1,17 @@
-from app import create_app, db
+from dotenv import load_dotenv
+load_dotenv()
+from app import create_app
+from app.services.db_service import db
+import os
 
 app = create_app()
 
-with app.app_context():
-    db.create_all()
-
 if __name__ == "__main__":
-    app.run(debug=True, port=5001)
+    with app.app_context():
+        db.create_all() 
+
+    app.run(
+        host=os.getenv("FLASK_HOST"),
+        port=int(os.getenv("FLASK_PORT")),
+        debug=os.getenv("FLASK_DEBUG") == "True"
+    )

test_ra_bills.py

@@ -0,0 +1,68 @@
+#!/usr/bin/env python3
+"""
+Test script to verify RA bill fetching from both client and subcontractor tables
+"""
+import sys
+import os
+sys.path.insert(0, os.path.dirname(__file__))
+
+from app import create_app, db
+from app.models.trench_excavation_model import TrenchExcavation
+from app.models.tr_ex_client_model import TrenchExcavationClient
+
+app = create_app()
+
+with app.app_context():
+    print("=" * 70)
+    print("RA BILL FETCHING TEST")
+    print("=" * 70)
+    
+    # Test 1: Count total records
+    print("\n✓ TEST 1: Total Records in Both Tables")
+    subcon_count = db.session.query(TrenchExcavation).count()
+    client_count = db.session.query(TrenchExcavationClient).count()
+    print(f"  Subcontractor (TrenchExcavation): {subcon_count} records")
+    print(f"  Client (TrenchExcavationClient): {client_count} records")
+    
+    # Test 2: Check RA bills in Subcontractor table
+    print("\n✓ TEST 2: RA Bills in Subcontractor Table")
+    subcon_bills = db.session.query(TrenchExcavation.RA_Bill_No)\
+        .filter(TrenchExcavation.RA_Bill_No != None)\
+        .filter(TrenchExcavation.RA_Bill_No != "")\
+        .distinct().all()
+    print(f"  Distinct RA Bills found: {len(subcon_bills)}")
+    print(f"  Bills: {[str(r[0]) for r in subcon_bills]}")
+    
+    # Test 3: Check RA bills in Client table
+    print("\n✓ TEST 3: RA Bills in Client Table")
+    client_bills = db.session.query(TrenchExcavationClient.RA_Bill_No)\
+        .filter(TrenchExcavationClient.RA_Bill_No != None)\
+        .filter(TrenchExcavationClient.RA_Bill_No != "")\
+        .distinct().all()
+    print(f"  Distinct RA Bills found: {len(client_bills)}")
+    print(f"  Bills: {[str(r[0]) for r in client_bills]}")
+    
+    # Test 4: Combined unique RA bills
+    print("\n✓ TEST 4: Combined Unique RA Bills (Union)")
+    combined_bills = db.session.query(TrenchExcavation.RA_Bill_No)\
+        .filter(TrenchExcavation.RA_Bill_No != None)\
+        .filter(TrenchExcavation.RA_Bill_No != "")\
+        .union(
+            db.session.query(TrenchExcavationClient.RA_Bill_No)\
+            .filter(TrenchExcavationClient.RA_Bill_No != None)\
+            .filter(TrenchExcavationClient.RA_Bill_No != "")
+        ).order_by(TrenchExcavation.RA_Bill_No).all()
+    
+    print(f"  Total unique RA Bills: {len(combined_bills)}")
+    print(f"  Bills: {[str(r[0]) for r in combined_bills]}")
+    
+    # Test 5: Sample data from both tables
+    print("\n✓ TEST 5: Sample RA Bills from Tables")
+    sample_subcon = db.session.query(TrenchExcavation.RA_Bill_No).limit(5).all()
+    sample_client = db.session.query(TrenchExcavationClient.RA_Bill_No).limit(5).all()
+    print(f"  Subcontractor samples: {[str(r[0]) for r in sample_subcon]}")
+    print(f"  Client samples: {[str(r[0]) for r in sample_client]}")
+    
+    print("\n" + "=" * 70)
+    print("✅ TEST COMPLETE")
+    print("=" * 70)