From 18a402edd42eb26f62a1f483098d74bf303e3f66 Mon Sep 17 00:00:00 2001 From: pjpatil12 Date: Fri, 19 Dec 2025 17:53:45 +0530 Subject: [PATCH] model as per sheet variable imort route call --- .gitignore | 15 +- ...ole_domestic_chamber_model.cpython-313.pyc | Bin 3501 -> 3792 bytes app/models/manhole_domestic_chamber_model.py | 3 + .../__pycache__/file_import.cpython-313.pyc | Bin 2305 -> 2409 bytes app/routes/file_import.py | 3 +- .../__pycache__/file_service.cpython-313.pyc | Bin 17364 -> 18712 bytes app/services/file_service.py | 169 +++++++++++++----- app/templates/file_import.html | 3 + 8 files changed, 144 insertions(+), 49 deletions(-) diff --git a/.gitignore b/.gitignore index 1b846b5..0a2f069 100644 --- a/.gitignore +++ b/.gitignore @@ -1,11 +1,20 @@ +# Python +app/__pycache__/ +*.pyc +*.pyo +*.pyd + # Ingnor upload files app/static/uploads/ # Ignore files venv -# Ignore Log files -logs +# Ignore Log files ss +logs/ # Ignore db folders -instance +instance/ + + +*.__pycache__ \ No newline at end of file diff --git a/app/models/__pycache__/manhole_domestic_chamber_model.cpython-313.pyc b/app/models/__pycache__/manhole_domestic_chamber_model.cpython-313.pyc index de81f1840eccd55f1cacce5b9a2c65eefb934ba5..b5bbc55c34ea8528d078c5e07b956430cf005ee5 100644 GIT binary patch delta 280 zcmZ20eL1`7D`=bb|S%7>aa*1&Z|21vP~x-(_3NocV=u za|gQ+BcuN0*&J$&`kN1NurTTIyM#N(2V@qc#`~n^rI%#f5(o_lLlxl)a*TJ%%*l!O z%b$FdD~2&>vl90oM(Z1564Ns$WiDV^&asf=vY2Lr{|#};>6Mcz7YHqvSSWE>TrY(8A9_VXGtzFo-KEoU!%cmvK(&$qxt G-U9%~2~>*! delta 159 zcmca0yH=X-GcPX}0}uo+cgmc_Hj!^4qs77nY~ns3F;ozv9uzR~qOdTpB~y`RFrO4d zkybE&k#@R(rr_j>>}xkWa|kdp>P!ydRAbcHT+PYCGV1ERguX%*e=imqF+*gX|X;O-6Ud8A0o$uB)PeQ9OIobb8_PS@+bdbTB*lg zBnDJi#1A3_K!hNWxW!(aT2z*qoLVFV;xXqIrx&vWSqhU+GDmVL0a+YCTs(KO4vW9; z9Z{L-F%x61i>h7}Rb7yCS=6|L>kAu$u-s(b4)zD)a@WNTFNzy(P&uJ>Sv+*}Bo=Na zHjt@B!jl)X8QO?|nBqV}6T>B{ASrne0amX7WIhlLgo%Sk-{SGLv=LmoO?% zzQo?42y$H!NN^=Xkur#_2qbQC*yQG?l;)(`73oc`=5Q0_XUt{%%*?=K^??b*{Gc}Z K5{EVmNId{HNlQ=w delta 428 zcmaDU)F{OJnU|M~0SNBSbjmE)$eYW^7&WUiSLlzO{%i^A!f3a{gv4NBq2~Fl^H;e=MqDTx#XtJZ3UZet&k^>Q7 z_3}VgF*}e@fPf+eAp4e3a!zJyUP*jrZb5!gNqkynPHK@VP(m7ri)RC^4w(Fk-Jelu zvK2>%BFIf(*_8}MN+7lZkhsNRlbfGXnv-f*q&xXBhnpZDV=m)oW(Fp!4@@BD2bIZI JoZ2iP^#IfgO#J`= diff --git a/app/routes/file_import.py b/app/routes/file_import.py index 85ffddf..d16071f 100644 --- a/app/routes/file_import.py +++ b/app/routes/file_import.py @@ -12,11 +12,12 @@ def import_file(): if request.method == "POST": file = request.files.get("file") subcontractor_id = request.form.get("subcontractor_id") + RA_Bill_No = request.form.get("RA_Bill_No") # file_type = request.form.get("file_type") service = FileService() # success, msg = service.handle_file_upload(file, subcontractor_id, file_type) - success, msg = service.handle_file_upload(file, subcontractor_id) + success, msg = service.handle_file_upload(file, subcontractor_id, RA_Bill_No) flash(msg, "success" if success else "danger") diff --git a/app/services/__pycache__/file_service.cpython-313.pyc b/app/services/__pycache__/file_service.cpython-313.pyc index 676152a93bb708b5b2080cb2a710ab13e9c5ecf4..2acc6dc8714fdcd23de8f50af62ee494e8ab2b80 100644 GIT binary patch delta 2542 zcmb`JYfMvT7=X{$%js!3?D1@_kF+H zd-=X|8aqbbpCE~IMx%ja&&}_e{PMuH#LMau4eARI&ssJ6%{^s2$91a;7dm`|;|2*l zQZG$XRrhe52X%Ef*Qo}NJrznc#Y6*}G&0e`4UHp-ck8ND64#qLKOJV&7FeSZz^Xl_ z7u)sPyZ5vlcez+==k5wgnpbG>+(d#?dRM|}mX0Ibj{=u0!6QA8 zRdB#inD#I|ES>JrYykk|Fo7BPLcn6jRO=o?m`;Ih4l4T&6`# zburP&hD1!S=%3*^W6HEx92JYNthzNG6FcvV$dnqv@KQPtLZL2d@f}< zAs@n5rb6DqVx<-Ez3DxngNa2MbQ!#z+Oe#dDRGTIG~$WthCCt9*chdFj_F$L;P-g~ zd*u*yp=I5K*+$AiGG{zr&1v32s^MdEHnGDEb1~T!er&ero9l426R`$S&(K91P;5fr z%Cs4wEUQ9k9U>cHLbNhKj;SG+ZbTJ5rf)KYaIhp%3mUo@;#Kga*$g+WOYM)DI%j5h z!b%y*Y`kl4xRNrdxlqM~pXer>u^GYcA4j)9vaOPA1(&VhMLv$&3O4P;UN6Fj!1PcV zF@Vq^6#oZNnokQ#TEq~-kJyDMN9;xfFh_hT6XW>{{+h_=;X3J4mF6I`hOecs;*$zG z>-!q~fk0n-Pznx65G}B*ucrS#nQ>iv&?mE1{!>Do3l0h{UvO9sjrqNO-XYKMPMO+k z0zt1Q5UR7+49X+&fl*qwg54W>2t)jHe#iCFf)CZ>l`N19u%}uDx|NM=9?wU^qL62w zEDd1)yknK`*eYtAQw z9}G_Vr>tdXT3}sHiLG~f{kG`(ZLxlD#J78@KQP@-qy2QMe=Krfe5!vO&gNw6tFAky zvz${|PWU2ch&VvX%{RUk>DoHowKLkaGiLUJJGZ~IjXX>?oj05{OeZgiCNGI4FTJ0> zI3hLQ+H=brv9?CiTkj>eKAX|1jBVs+dJwLuGs4euGc^gyc~(-}3{pW;S{uf@9W51|y!gOZHc&Jfqi!=qyd=hM3lt;*$6Zj&BW<%Yeo zbe|btDqWm7yzzu#e<Cw4%6eZ6fpigpCP{b(8DB;pJL;9`9i z>42N{i#M-ghNIl9hG@NHR%j>Midmn^MYLM)Sn JSq`ZT`VGvMO%MP8 delta 1983 zcmb`IUu;uV9LLYOZEt(~XN;}gKQ?EoelqKlq}UZlyD4jhd~JQDs5ozr zZTr^Tsz-?8g6*Ok8pIY_!c_p~#TGa$I-u4Xi;l0ZC;J*!QmsT?O-NQ!%9Qe9sf4r$ zt%4?aiNa%H$#?snrfJe?@{%<5yW^~kFNV*oXW)5T7p;R2Z4c2#_{QdgcWq`^C$-Ts z7?l>CQs0qeYC4yh%KGi#w+~xsN<&ZJb$hI2kT*F`mOT|O*}q@qg>UUZH^VcIM#~M6 zxXR#tM+1D|sPu&{Iy;x0rC&J%7o35mtsfn|=-hJ|$4wWUO-nIj3=wCruBIm#mB=~C z9`%{dm5WiY>0C8M?X^UG`n9T~0h{@olipu-W~pZGcOEHEV)T+=I=2 zJ}8(oT2?Js52>1(%P}TmZVyg~v+Bf5N@IN(i9*=pr5i!@RIikq_l(ju_}WuPH^3jB z08Q(S6;)DX8>WX4oro@uA=ZsDjKI`*F~2fxD{H^&d$=X_1%i(t>v>bK2{?Ofbs3a%PL3?m+g-Etd! z0$!HaSFRX>aef*J+lv^1v-0|W^oJcpBoS7GvHvktw|qfmL8K6A#5iIj;t(RkJ!+uq zR<=uZeXjnrKxg&3?iRDdsA3e2EVTB@F3xSZ7|)sp0;3o?q6|PQD;+fRLxAJ#yVufNx%D6i~F$tenbK>iZI$7;i_OsO-|>Y zWs?{+%s7C`aKMJjaOG|-@4D0TJ1$m%sZS$t`vsaTI3_cimdTDMCZ>|?FwPi{$fBA; zFocH4A=;p$cMX*x+$6(~%|UUI|Aw^7u+;mxFht=5vQcI`Z^ILzTjGyr(#!raQCNLzDCKHE0>(g?^i82rJrg$U|#!= z7lBX5;}~Vnp%f6u5Sw6c>&TpOWbVaEHiwJuW~77@|7qkMAK5(5(_9?#-xmz@+F*W5 zw{V{7A8)xt1$M$XO83}7P3Wly(E)zi?yG}8Lt)`Js)swTQ2M_bn2GFt5Dhr)uYqCe z*H8Ceq2dXwGXTe<;cZ({Za@SPFCb1LUO~LZQSc-ZT29Snk_r6UW)9BeQd%Ox1{V5j zo7;H8i==3#mLx6OV${-F+%NFkc3K?4dmk+hS(_|Fw0Mxhj{e$`hq0nKt!Bqmjrn!0 nKSFB;@KN~JXnZOQw$S9%*vv$#i@gQ;So^9TiFOtV(n$IXS>m`L diff --git a/app/services/file_service.py b/app/services/file_service.py index 3750737..f0b0a5c 100644 --- a/app/services/file_service.py +++ b/app/services/file_service.py @@ -21,9 +21,9 @@ class FileService: return "." in filename and filename.rsplit(".", 1)[1].lower() in Config.ALLOWED_EXTENSIONS # def handle_file_upload(self, file, subcontractor_id, file_type): - def handle_file_upload(self, file, subcontractor_id): + def handle_file_upload(self, file, subcontractor_id, RA_Bill_No): - RA_Bill_No = 1 # this RA bill define temp for upload (changes also) + # RA_Bill_No = 1 # this RA bill define temp for upload (changes also) if not subcontractor_id: return False, "Please select subcontractor." @@ -49,18 +49,24 @@ class FileService: df_tr_ex = pd.read_excel(filepath, sheet_name ="Tr.Ex.", header=12) df_mh_ex = pd.read_excel(filepath, sheet_name="MH Ex.", header=12) - # df2 = pd.read_excel(filepath, sheet_name="MH & DC", header=0) + df_mh_dc = pd.read_excel(filepath, sheet_name="MH & DC", header=11) print("\n=== Uploaded File tr ex ===") print(df_tr_ex.head()) print("=============================\n") - print("=== Uploaded File mh ex ===") + print("\n=== Uploaded File mh ex ===") print(df_mh_ex.head()) - print("========================================") + print("========================================\n") + + print("\n=== Uploaded File MH DC ===") + print(df_mh_dc.head()) + print("========================================\n") + self.process_trench_excavation(df_tr_ex, subcontractor_id, RA_Bill_No) - self.process_manhole_excavation(df_mh_ex, subcontractor_id,RA_Bill_No) + self.process_manhole_excavation(df_mh_ex, subcontractor_id, RA_Bill_No) + self.process_manhole_domestic_chamber(df_mh_dc, subcontractor_id, RA_Bill_No) @@ -94,11 +100,9 @@ class FileService: except Exception as e: return False, f"Processing failed: {e}" + - - - - # new new + # new new def process_trench_excavation(self, df, subcontractor_id,RA_Bill_No): print("RA_Bill_No of Tr Ex:",RA_Bill_No) @@ -249,6 +253,81 @@ class FileService: db.session.rollback() return False, f"Manhole Excavation save failed: {e}" + # new new + def process_manhole_domestic_chamber(self, df, subcontractor_id, RA_Bill_No): + + print("RA_Bill_No of Manhole Domestic Chamber:",RA_Bill_No) + + print("=== RAW HEADERS ===") + print(df.columns.tolist()) + print("===================") + + # Clean column names + df.columns = ( + df.columns.astype(str) + .str.strip() + .str.replace(r"[^\w]", "_", regex=True) + .str.replace("__+", "_", regex=True) + .str.strip("_") + ) + + # Remove completely empty rows + df = df.dropna(how="all") + + # Forward fill merged Location + if "Location" in df.columns: + df["Location"] = df["Location"].ffill() + + saved_count = 0 + skipped_count = 0 + + try: + for index, row in df.iterrows(): + record_data = {} + location = row.get("Location") + Node_No = row.get("Node_No") + + if (pd.isna(location) or str(location).strip() == "" or pd.isna(Node_No) or str(Node_No).strip() == ""): + skipped_count += 1 + continue + + # Map only model columns + for col in df.columns: + if hasattr(ManholeDomesticChamber, col): + value = row[col] + + # Normalize empty values + if pd.isna(value) or str(value).strip() in ["", "-", "—", "nan"]: + value = None + + record_data[col] = value + + # If all mapped fields are None → skip + if all(v is None for v in record_data.values()): + skipped_count += 1 + continue + + record = ManholeDomesticChamber( + subcontractor_id=subcontractor_id, RA_Bill_No=RA_Bill_No, + **record_data + ) + + print("Saving Row → Location:", record.Location, " Node_No:", record.Node_No) + + db.session.add(record) + saved_count += 1 + + db.session.commit() + + return True, ( + f"Manhole Domestic Chamber saved successfully. " + f"Inserted: {saved_count}, Skipped: {skipped_count}" + ) + + except Exception as e: + db.session.rollback() + return False, f"Manhole Domestic Chamber save failed: {e}" + # olds @@ -409,50 +488,50 @@ class FileService: # Manhole and Domestic Chamber Construction save method (ManholeDomesticChamber model) - def process_manhole_domestic_chamber(self, df, subcontractor_id): - # Clean column names (strip whitespace) - df.columns = [str(c).strip() for c in df.columns] - # If the sheet has merged cells -> forward fill Location - if "Location" in df.columns: - df["Location"] = df["Location"].ffill() + # def process_manhole_domestic_chamber(self, df, subcontractor_id): + # # Clean column names (strip whitespace) + # df.columns = [str(c).strip() for c in df.columns] + # # If the sheet has merged cells -> forward fill Location + # if "Location" in df.columns: + # df["Location"] = df["Location"].ffill() - # REMOVE empty rows - df = df.dropna(how="all") - # Identify missing location rows before insert - missing_loc = df[df["Location"].isna() | (df["Location"].astype(str).str.strip() == "")] - if not missing_loc.empty: - return False, f"Error: Some rows have empty Location. Rows: {missing_loc.index.tolist()}" + # # REMOVE empty rows + # df = df.dropna(how="all") + # # Identify missing location rows before insert + # missing_loc = df[df["Location"].isna() | (df["Location"].astype(str).str.strip() == "")] + # if not missing_loc.empty: + # return False, f"Error: Some rows have empty Location. Rows: {missing_loc.index.tolist()}" - saved_count = 0 + # saved_count = 0 - try: - for index, row in df.iterrows(): - record_data = {} + # try: + # for index, row in df.iterrows(): + # record_data = {} - # Insert only fields that exist in model - for col in df.columns: - if hasattr(ManholeDomesticChamber, col): - value = row[col] - # Normalize empty values - if pd.isna(value) or str(value).strip() in ["", "-", "—", "nan", "NaN"]: - value = None + # # Insert only fields that exist in model + # for col in df.columns: + # if hasattr(ManholeDomesticChamber, col): + # value = row[col] + # # Normalize empty values + # if pd.isna(value) or str(value).strip() in ["", "-", "—", "nan", "NaN"]: + # value = None - record_data[col] = value + # record_data[col] = value - record = ManholeDomesticChamber( - subcontractor_id=subcontractor_id, - **record_data - ) + # record = ManholeDomesticChamber( + # subcontractor_id=subcontractor_id, + # **record_data + # ) - db.session.add(record) - saved_count += 1 + # db.session.add(record) + # saved_count += 1 - db.session.commit() - return True, f"Manhole Domestic Chamber Construction data saved successfully. Total rows: {saved_count}" + # db.session.commit() + # return True, f"Manhole Domestic Chamber Construction data saved successfully. Total rows: {saved_count}" - except Exception as e: - db.session.rollback() - return False, f"Manhole Domestic Chamber Construction Save Failed: {e}" + # except Exception as e: + # db.session.rollback() + # return False, f"Manhole Domestic Chamber Construction Save Failed: {e}" diff --git a/app/templates/file_import.html b/app/templates/file_import.html index aaa977c..dd9d12f 100644 --- a/app/templates/file_import.html +++ b/app/templates/file_import.html @@ -28,6 +28,9 @@ --> + + +