Comparison_Project/app/services/file_service.py

import os
import pandas as pd
from werkzeug.utils import secure_filename
from app.config import Config
from app import db
from app.models.trench_excavation_model import TrenchExcavation
from app.models.manhole_excavation_model import ManholeExcavation
from app.models.manhole_domestic_chamber_model import ManholeDomesticChamber

from app.models.tr_ex_client_model import TrenchExcavationClient
from app.models.mh_ex_client_model import ManholeExcavationClient
from app.models.mh_dc_client_model import ManholeDomesticChamberClient


from app.utils.file_utils import ensure_upload_folder


class FileService:

    def allowed_file(self, filename):
        return "." in filename and filename.rsplit(".", 1)[1].lower() in Config.ALLOWED_EXTENSIONS

    # def handle_file_upload(self, file, subcontractor_id, file_type):
    def handle_file_upload(self, file, subcontractor_id, RA_Bill_No):

        if not subcontractor_id:
            return False, "Please select subcontractor."
        if not RA_Bill_No:
            return False, "Please Enter RA Bill No)."
        if not file or file.filename == "":
            return False, "No file selected."
        if not self.allowed_file(file.filename):
            return False, "Invalid file type! Allowed: CSV, XLSX, XLS"

        ensure_upload_folder()

        folder = os.path.join(Config.UPLOAD_FOLDER, f"sub_{subcontractor_id}")
        os.makedirs(folder, exist_ok=True)

        filename = secure_filename(file.filename)
        filepath = os.path.join(folder, filename)
        file.save(filepath)

        try:

            df_tr_ex = pd.read_excel(filepath, sheet_name ="Tr.Ex.", header=12)
            df_mh_ex = pd.read_excel(filepath, sheet_name="MH Ex.", header=12)
            df_mh_dc = pd.read_excel(filepath, sheet_name="MH & DC", header=11)

            self.process_trench_excavation(df_tr_ex, subcontractor_id, RA_Bill_No)
            self.process_manhole_excavation(df_mh_ex, subcontractor_id, RA_Bill_No)
            self.process_manhole_domestic_chamber(df_mh_dc, subcontractor_id, RA_Bill_No)

            return True, "File uploaded successfully."

        except Exception as e:
            return False, f"Processing failed: {e}"
      

    #   Trench Excavation save method (TrenchExcavation model)
    def process_trench_excavation(self, df, subcontractor_id,RA_Bill_No):

        print("trench_excavation RA_Bill_No :",RA_Bill_No)

        print("=== trench_excavation ===")
        print(df.columns.tolist())
        print("===================")

        # Clean column names
        df.columns = (
            df.columns.astype(str)
            .str.strip()
            .str.replace(r"[^\w]", "_", regex=True)
            .str.replace("__+", "_", regex=True)
            .str.strip("_")
        )

        # Remove completely empty rows
        df = df.dropna(how="all")

        # Forward fill merged Location
        if "Location" in df.columns:
            df["Location"] = df["Location"].ffill()

        saved_count = 0
        skipped_count = 0

        try:
            for index, row in df.iterrows():
                record_data = {}
                location = row.get("Location")
                mh_no = row.get("MH_NO")

                if (pd.isna(location) or str(location).strip() == "" or pd.isna(mh_no) or str(mh_no).strip() == ""):
                    skipped_count += 1
                    continue

                # Map only model columns
                for col in df.columns:
                    if hasattr(TrenchExcavation, col):
                        value = row[col]

                        # Normalize empty values
                        if pd.isna(value) or str(value).strip() in ["", "-", "—", "nan"]:
                            value = None

                        record_data[col] = value
                       
                # If all mapped fields are None → skip
                if all(v is None for v in record_data.values()):
                    skipped_count += 1
                    continue

                record = TrenchExcavation(
                    subcontractor_id=subcontractor_id, RA_Bill_No=RA_Bill_No,
                    **record_data
                )

                print("trench_excavation Saving Row → Location:", record.Location, " MH_NO:", record.MH_NO)

                db.session.add(record)
                saved_count += 1

            db.session.commit()

            return True, (
                f"Trench Excavation saved successfully. "
                f"Inserted: {saved_count}, Skipped: {skipped_count}"
            )

        except Exception as e:
            db.session.rollback()
            return False, f"Trench Excavation save failed: {e}"
        

    #   Manhole Excavation save method (ManholeExcavation  model)
    def process_manhole_excavation(self, df, subcontractor_id, RA_Bill_No):

            print("manhole_excavation RA_Bill_No:",RA_Bill_No)

            print("=== manhole_excavation ===")
            print(df.columns.tolist())
            print("===================")

            # Clean column names
            df.columns = (
                df.columns.astype(str)
                .str.strip()
                .str.replace(r"[^\w]", "_", regex=True)
                .str.replace("__+", "_", regex=True)
                .str.strip("_")
            )

            # Remove completely empty rows
            df = df.dropna(how="all")

            # Forward fill merged Location
            if "Location" in df.columns:
                df["Location"] = df["Location"].ffill()

            saved_count = 0
            skipped_count = 0

            try:
                for index, row in df.iterrows():
                    record_data = {}
                    location = row.get("Location")
                    mh_no = row.get("MH_NO")

                    if (pd.isna(location) or str(location).strip() == "" or pd.isna(mh_no) or str(mh_no).strip() == ""):
                        skipped_count += 1
                        continue

                    # Map only model columns
                    for col in df.columns:
                        if hasattr(ManholeExcavation, col):
                            value = row[col]

                            # Normalize empty values
                            if pd.isna(value) or str(value).strip() in ["", "-", "—", "nan"]:
                                value = None

                            record_data[col] = value
                        
                    # If all mapped fields are None → skip
                    if all(v is None for v in record_data.values()):
                        skipped_count += 1
                        continue

                    record = ManholeExcavation(
                        subcontractor_id=subcontractor_id, RA_Bill_No=RA_Bill_No,
                        **record_data
                    )

                    print("manhole_excavation Saving Row → Location:", record.Location, " MH_NO:", record.MH_NO)

                    db.session.add(record)
                    saved_count += 1

                db.session.commit()

                return True, (
                    f"Manhole Excavation saved successfully. "
                    f"Inserted: {saved_count}, Skipped: {skipped_count}"
                )

            except Exception as e:
                db.session.rollback()
                return False, f"Manhole Excavation save failed: {e}"

    #   Manhole and Domestic Chamber Construction save method (ManholeDomesticChamber  model)
    def process_manhole_domestic_chamber(self, df, subcontractor_id, RA_Bill_No):

            print("manhole_domestic_chamber RA_Bill_No :",RA_Bill_No)

            print("=== manhole_domestic_chamber ===")
            print(df.columns.tolist())
            print("===================")

            # Clean column names
            df.columns = (
                df.columns.astype(str)
                .str.strip()
                .str.replace(r"[^\w]", "_", regex=True)
                .str.replace("__+", "_", regex=True)
                .str.strip("_")
            )

            # Remove completely empty rows
            df = df.dropna(how="all")

            # Forward fill merged Location
            if "Location" in df.columns:
                df["Location"] = df["Location"].ffill()

            saved_count = 0
            skipped_count = 0

            try:
                for index, row in df.iterrows():
                    record_data = {}
                    location = row.get("Location")
                    Node_No = row.get("Node_No")

                    if (pd.isna(location) or str(location).strip() == "" or pd.isna(Node_No) or str(Node_No).strip() == ""):
                        skipped_count += 1
                        continue

                    # Map only model columns
                    for col in df.columns:
                        if hasattr(ManholeDomesticChamber, col):
                            value = row[col]

                            # Normalize empty values
                            if pd.isna(value) or str(value).strip() in ["", "-", "—", "nan"]:
                                value = None

                            record_data[col] = value
                        
                    # If all mapped fields are None → skip
                    if all(v is None for v in record_data.values()):
                        skipped_count += 1
                        continue

                    record = ManholeDomesticChamber(
                        subcontractor_id=subcontractor_id, RA_Bill_No=RA_Bill_No,
                        **record_data
                    )

                    print("manhole_domestic_chamber Saving Row → Location:", record.Location, " Node_No:", record.Node_No)

                    db.session.add(record)
                    saved_count += 1

                db.session.commit()

                return True, (
                    f"Manhole Domestic Chamber saved successfully. "
                    f"Inserted: {saved_count}, Skipped: {skipped_count}"
                )

            except Exception as e:
                db.session.rollback()
                return False, f"Manhole Domestic Chamber save failed: {e}"
       


    # ---------------------- client ---------------------------------- 

    # def handle_client_file_upload(self, file, RA_Bill_No):

    #         if not RA_Bill_No:
    #             return False, "Please Enter RA Bill No ."

    #         if not file or file.filename == "":
    #             return False, "No file selected."
    #         if not self.allowed_file(file.filename):
    #             return False, "Invalid file type! Allowed: CSV, XLSX, XLS"

    #         ensure_upload_folder()

    #         folder = os.path.join(Config.UPLOAD_FOLDER, f"Client_Bill_{RA_Bill_No}")
    #         os.makedirs(folder, exist_ok=True)

    #         filename = secure_filename(file.filename)
    #         filepath = os.path.join(folder, filename)
    #         file.save(filepath)

    #         try:
    #             df_tr_ex = pd.read_excel(filepath, sheet_name ="Tr.Ex.", header=12)
    #             df_mh_ex = pd.read_excel(filepath, sheet_name="MH Ex.", header=12)
    #             df_mh_dc = pd.read_excel(filepath, sheet_name="MH & DC", header=11)

    #             print("\n=== Uploaded File tr ex ===")
    #             print(df_tr_ex.head())
    #             print("=============================\n")
    #             print("\n=== Uploaded File mh ex ===")
    #             print(df_mh_ex.head())
    #             print("========================================\n")
    #             print("\n=== Uploaded File MH DC ===")
    #             print(df_mh_dc.head())
    #             print("========================================\n")


    #             self.client_trench_excavation(df_tr_ex, RA_Bill_No)
    #             self.client_manhole_excavation(df_mh_ex, RA_Bill_No)
    #             self.client_manhole_domestic_chamber(df_mh_dc, RA_Bill_No)

    #             return True, "File uploaded successfully."

    #         except Exception as e:
    #             return False, f"Processing failed: {e}"
        
    # # Tr Ex save method (TrenchExcavationClient model)
    # def client_trench_excavation(self, df, RA_Bill_No):
    #     df.columns = [str(c).strip() for c in df.columns]
    #     # If the sheet has merged cells -> forward fill Location
    #     if "Location" in df.columns:
    #         df["Location"] = df["Location"].ffill()

    #     df = df.dropna(how="all") # REMOVE empty rows
    #     # Identify missing location rows before insert
    #     missing_loc = df[df["Location"].isna() | (df["Location"].astype(str).str.strip() == "")]
    #     if not missing_loc.empty:
    #         return False, f"Error: Some rows have empty Location. Rows: {missing_loc.index.tolist()}"

    #     saved_count = 0

    #     try:
    #         for index, row in df.iterrows():
    #             record_data = {}
    #             # Insert only fields that exist in model
    #             for col in df.columns:
    #                 if hasattr(TrenchExcavationClient, col):
    #                     value = row[col]
    #                     # Normalize empty values
    #                     if pd.isna(value) or str(value).strip() in ["", "-", "—", "nan", "NaN"]:
    #                         value = None

    #                     record_data[col] = value

    #             record = TrenchExcavationClient(
    #                 RA_Bill_No=RA_Bill_No,
    #                 **record_data
    #             )

    #             db.session.add(record)
    #             saved_count += 1

    #         db.session.commit()
    #         return True, f"Clinnt Tr Ex data saved successfully. Total rows: {saved_count}"

    #     except Exception as e:
    #         db.session.rollback()
    #         return False, f"Clinnt Tr Ex Save Failed: {e}"
        

    # #   Mh Ex save method (ManholeExcavationClient model)
    # def client_manhole_excavation(self, df, RA_Bill_No):
    #     # Clean column names (strip whitespace)
    #     df.columns = [str(c).strip() for c in df.columns]
    #     # If the sheet has merged cells -> forward fill Location
    #     if "Location" in df.columns:
    #         df["Location"] = df["Location"].ffill()

    #     # REMOVE empty rows
    #     df = df.dropna(how="all")
    #     # Identify missing location rows before insert
    #     missing_loc = df[df["Location"].isna() | (df["Location"].astype(str).str.strip() == "")]
    #     if not missing_loc.empty:
    #         return False, f"Error: Some rows have empty Location. Rows: {missing_loc.index.tolist()}"

    #     saved_count = 0

    #     try:
    #         for index, row in df.iterrows():
    #             record_data = {}
    #             # Insert only fields that exist in model
    #             for col in df.columns:
    #                 if hasattr(ManholeExcavationClient, col):
    #                     value = row[col]

    #                     # Normalize empty values
    #                     if pd.isna(value) or str(value).strip() in ["", "-", "—", "nan", "NaN"]:
    #                         value = None

    #                     record_data[col] = value

    #             record = ManholeExcavationClient(
    #                 RA_Bill_No=RA_Bill_No,
    #                 **record_data
    #             )

    #             db.session.add(record)
    #             saved_count += 1

    #         db.session.commit()
    #         return True, f" Client Mh Ex. data saved successfully. Total rows: {saved_count}"

    #     except Exception as e:
    #         db.session.rollback()
    #         return False, f"Client Mh Ex. Save Failed: {e}"


    # #   Mh and Dc save method (ManholeDomesticChamberClient  model)
    # def client_manhole_domestic_chamber(self, df, RA_Bill_No):
        # Clean column names (strip whitespace)
        # df.columns = [str(c).strip() for c in df.columns]
        # # If the sheet has merged cells -> forward fill Location
        # if "Location" in df.columns:
        #     df["Location"] = df["Location"].ffill()

        # # REMOVE empty rows
        # df = df.dropna(how="all")
        # # Identify missing location rows before insert
        # missing_loc = df[df["Location"].isna() | (df["Location"].astype(str).str.strip() == "")]
        # if not missing_loc.empty:
        #     return False, f"Error: Some rows have empty Location. Rows: {missing_loc.index.tolist()}"

        # saved_count = 0

        # try:
        #     for index, row in df.iterrows():
        #         record_data = {}

        #         # Insert only fields that exist in model
        #         for col in df.columns:
        #             if hasattr(ManholeDomesticChamberClient, col):
        #                 value = row[col]
        #                 # Normalize empty values
        #                 if pd.isna(value) or str(value).strip() in ["", "-", "—", "nan", "NaN"]:
        #                     value = None

        #                 record_data[col] = value

        #         record = ManholeDomesticChamberClient(
        #             RA_Bill_No=RA_Bill_No,
        #             **record_data
        #         )

        #         db.session.add(record)
        #         saved_count += 1

        #     db.session.commit()
        #     return True, f"Mh and Dc data saved successfully. Total rows: {saved_count}"

        # except Exception as e:
        #     db.session.rollback()
        #     return False, f"Mh and Dc data Save Failed: {e}"
        
    def handle_client_file_upload(self, file, RA_Bill_No):

        if not RA_Bill_No:
            return False, "Please Enter RA Bill No."

        if not file or file.filename == "":
            return False, "No file selected."

        if not self.allowed_file(file.filename):
            return False, "Invalid file type! Allowed: CSV, XLSX, XLS"

        ensure_upload_folder()

        folder = os.path.join(Config.UPLOAD_FOLDER, f"Client_Bill_{RA_Bill_No}")
        os.makedirs(folder, exist_ok=True)

        filename = secure_filename(file.filename)
        filepath = os.path.join(folder, filename)
        file.save(filepath)

        try:
            df_tr_ex = pd.read_excel(filepath, sheet_name="Tr.Ex.", header=12)
            df_mh_ex = pd.read_excel(filepath, sheet_name="MH Ex.", header=12)
            df_mh_dc = pd.read_excel(filepath, sheet_name="MH & DC", header=11)

            # Single transaction
            self._save_client_data(df_tr_ex, TrenchExcavationClient, "Trench Excavation", RA_Bill_No)
            self._save_client_data(df_mh_ex, ManholeExcavationClient, "Manhole Excavation", RA_Bill_No)
            self._save_client_data(df_mh_dc, ManholeDomesticChamberClient, "MH & DC", RA_Bill_No)

            db.session.commit()
            return True, "Client file uploaded and all data saved successfully."

        except ClientDataSaveError as e:
            db.session.rollback()
            return False, str(e)

        except Exception as e:
            db.session.rollback()
            return False, f"Unexpected system error: {e}"


    def _save_client_data(self, df, model, RA_Bill_No):
            
            # Clean column names
            df.columns = [str(c).strip() for c in df.columns]

            # Forward fill Location (merged cells issue)
            if "Location" in df.columns:
                df["Location"] = df["Location"].ffill()

            # Remove fully empty rows
            df = df.dropna(how="all")

            # Validate Location
            if "Location" in df.columns:
                missing_loc = df[
                    df["Location"].isna() |
                    (df["Location"].astype(str).str.strip() == "")
                ]
                if not missing_loc.empty:
                    raise Exception(
                        f"{model.__name__}: Empty Location at rows {missing_loc.index.tolist()}"
                    )

            # Insert rows
            for _, row in df.iterrows():
                record_data = {}

                for col in df.columns:
                    if hasattr(model, col):
                        value = row[col]

                        if pd.isna(value) or str(value).strip() in ["", "-", "—", "nan", "NaN"]:
                            value = None

                        record_data[col] = value

                record = model(
                    RA_Bill_No=RA_Bill_No,
                    **record_data
                )

                db.session.add(record)