| | from flask import Flask, render_template, request, redirect, url_for, send_file |
| | import os |
| | import shutil |
| | import pandas as pd |
| | from werkzeug.utils import secure_filename |
| | from joblib import load, dump |
| | import numpy as np |
| | from sklearn.preprocessing import LabelEncoder |
| | from time import time |
| | from huggingface_hub import hf_hub_download |
| | import pickle |
| | import uuid |
| | from pathlib import Path |
| | import numpy as np |
| | import pandas as pd |
| | import seaborn as sns |
| | import matplotlib as mpl |
| | import matplotlib.pyplot as plt |
| | import matplotlib.pylab as pylab |
| | from sklearn.preprocessing import OneHotEncoder, LabelEncoder |
| | from sklearn.model_selection import train_test_split |
| | from sklearn.preprocessing import StandardScaler |
| | from sklearn.decomposition import PCA |
| | from sklearn.pipeline import Pipeline |
| | from sklearn.tree import DecisionTreeRegressor |
| | from sklearn.ensemble import RandomForestRegressor |
| | from sklearn.linear_model import LinearRegression |
| | from xgboost import XGBRegressor |
| | from sklearn.neighbors import KNeighborsRegressor |
| | from sklearn.model_selection import cross_val_score |
| | from sklearn.metrics import mean_squared_error |
| | from sklearn import metrics |
| | from sklearn.model_selection import train_test_split |
| | from sklearn.pipeline import Pipeline |
| | from sklearn.preprocessing import PowerTransformer, StandardScaler |
| | from sklearn.ensemble import RandomForestRegressor |
| | from sklearn.model_selection import train_test_split, cross_val_score, RandomizedSearchCV |
| | import lightgbm as lgb |
| | from catboost import CatBoostRegressor |
| | from sklearn.ensemble import StackingRegressor |
| |
|
| | app = Flask(__name__) |
| |
|
| | |
| | app.secret_key = os.urandom(24) |
| |
|
| | |
| | UPLOAD_FOLDER = "uploads/" |
| | DATA_FOLDER = "data/" |
| | MODEL_FOLDER = "models/" |
| |
|
| | os.makedirs(MODEL_FOLDER, exist_ok=True) |
| |
|
| | |
| | MODEL_DIR = r'./Model' |
| | LABEL_ENCODER_DIR = r'./Label_encoders' |
| |
|
| | |
| | |
| | PRED_OUTPUT_FILE = None |
| | CLASS_OUTPUT_FILE = None |
| |
|
| | ALLOWED_EXTENSIONS = {'csv', 'xlsx'} |
| |
|
| | |
| | app.config['UPLOAD_FOLDER'] = UPLOAD_FOLDER |
| | os.makedirs(app.config['UPLOAD_FOLDER'], exist_ok=True) |
| |
|
| | app.config['DATA_FOLDER'] = DATA_FOLDER |
| | os.makedirs(app.config['DATA_FOLDER'], exist_ok=True) |
| |
|
| | os.makedirs("data", exist_ok=True) |
| |
|
| | app.config['MODEL_FOLDER'] = MODEL_FOLDER |
| | os.makedirs(app.config['MODEL_FOLDER'], exist_ok=True) |
| |
|
| | |
| | |
| | |
| |
|
| | |
| | src_path = hf_hub_download( |
| | repo_id="WebashalarForML/Diamond_model_", |
| | filename="models_list/mkble/DecisionTree_best_pipeline_mkble_0_to_0.99_al.pkl", |
| | cache_dir=MODEL_FOLDER |
| | ) |
| | dst_path = os.path.join(MODEL_FOLDER, "DecisionTree_best_pipeline_mkble_0_to_0.99_al.pkl") |
| | shutil.copy(src_path, dst_path) |
| | makable_model = load(dst_path) |
| |
|
| | ''' |
| | src_path = hf_hub_download( |
| | repo_id="WebashalarForML/Diamond_model_", |
| | filename="models_list/grd/StackingRegressor_best_pipeline_grd_0_to_1.01.pkl", |
| | cache_dir=MODEL_FOLDER |
| | ) |
| | dst_path = os.path.join(MODEL_FOLDER, "StackingRegressor_best_pipeline_grd_0_to_1.01.pkl") |
| | shutil.copy(src_path, dst_path) |
| | grade_model = load(dst_path) |
| | |
| | |
| | src_path = hf_hub_download( |
| | repo_id="WebashalarForML/Diamond_model_", |
| | filename="models_list/bygrad/StackingRegressor_best_pipeline_bygrad_0_to_1.01.pkl", |
| | cache_dir=MODEL_FOLDER |
| | ) |
| | dst_path = os.path.join(MODEL_FOLDER, "StackingRegressor_best_pipeline_bygrad_0_to_1.01.pkl") |
| | shutil.copy(src_path, dst_path) |
| | bygrade_model = load(dst_path) |
| | |
| | |
| | src_path = hf_hub_download( |
| | repo_id="WebashalarForML/Diamond_model_", |
| | filename="models_list/gia/StackingRegressor_best_pipeline_gia_0_to_1.01.pkl", |
| | cache_dir=MODEL_FOLDER |
| | ) |
| | dst_path = os.path.join(MODEL_FOLDER, "StackingRegressor_best_pipeline_gia_0_to_1.01.pkl") |
| | shutil.copy(src_path, dst_path) |
| | gia_model = load(dst_path) |
| | ''' |
| |
|
| | |
| | src_path = hf_hub_download( |
| | repo_id="WebashalarForML/Diamond_model_", |
| | filename="models_list/classification/3_pipeline.pkl", |
| | cache_dir=MODEL_FOLDER |
| | ) |
| | dst_path = os.path.join(MODEL_FOLDER, "3_pipeline.pkl") |
| | shutil.copy(src_path, dst_path) |
| | mkble_amt_class_model = load(dst_path) |
| |
|
| |
|
| |
|
| |
|
| | print("makable_model type:", type(makable_model)) |
| | |
| | |
| | |
| | print("================================") |
| | print("mkble_amt_class_model type:", type(mkble_amt_class_model)) |
| |
|
| | |
| | |
| | |
| | |
| | |
| |
|
| | |
| | ''' |
| | col_model = load(os.path.join(MODEL_DIR, 'classification_LogisticRegression_col.joblib')) |
| | cts_model = load(os.path.join(MODEL_DIR, 'classification_LogisticRegression_cts.joblib')) |
| | cut_model = load(os.path.join(MODEL_DIR, 'classification_LogisticRegression_cut.joblib')) |
| | qua_model = load(os.path.join(MODEL_DIR, 'classification_LogisticRegression_qua.joblib')) |
| | shp_model = load(os.path.join(MODEL_DIR, 'classification_LogisticRegression_shp.joblib')) |
| | |
| | blk_eng_to_mkbl_model = load(os.path.join(MODEL_DIR, 'classification_LogisticRegression_mkbl_blk.joblib')) |
| | wht_eng_to_mkbl_model = load(os.path.join(MODEL_DIR, 'classification_LogisticRegression_mkbl_wht.joblib')) |
| | open_eng_to_mkbl_model = load(os.path.join(MODEL_DIR, 'classification_LogisticRegression_mkbl_open.joblib')) |
| | pav_eng_to_mkbl_model = load(os.path.join(MODEL_DIR, 'classification_LogisticRegression_mkbl_pav.joblib')) |
| | blk_eng_to_grade_model = load(os.path.join(MODEL_DIR, 'classification_LogisticRegression_grade_blk.joblib')) |
| | wht_eng_to_grade_model = load(os.path.join(MODEL_DIR, 'classification_LogisticRegression_grade_wht.joblib')) |
| | open_eng_to_grade_model = load(os.path.join(MODEL_DIR, 'classification_LogisticRegression_grade_open.joblib')) |
| | pav_eng_to_grade_model = load(os.path.join(MODEL_DIR, 'classification_LogisticRegression_grade_pav.joblib')) |
| | blk_eng_to_bygrade_model = load(os.path.join(MODEL_DIR, 'classification_LogisticRegression_bygrade_blk.joblib')) |
| | wht_eng_to_bygrade_model = load(os.path.join(MODEL_DIR, 'classification_LogisticRegression_bygrade_wht.joblib')) |
| | open_eng_to_bygrade_model = load(os.path.join(MODEL_DIR, 'classification_LogisticRegression_bygrade_open.joblib')) |
| | pav_eng_to_bygrade_model = load(os.path.join(MODEL_DIR, 'classification_LogisticRegression_bygrade_pav.joblib')) |
| | blk_eng_to_gia_model = load(os.path.join(MODEL_DIR, 'classification_LogisticRegression_gia_blk.joblib')) |
| | wht_eng_to_gia_model = load(os.path.join(MODEL_DIR, 'classification_LogisticRegression_gia_wht.joblib')) |
| | open_eng_to_gia_model = load(os.path.join(MODEL_DIR, 'classification_LogisticRegression_gia_open.joblib')) |
| | pav_eng_to_gia_model = load(os.path.join(MODEL_DIR, 'classification_LogisticRegression_gia_pav.joblib')) |
| | ''' |
| |
|
| |
|
| | |
| | encoder_list = [ |
| | 'Tag', 'EngShp', 'EngQua', 'EngCol', 'EngCut', 'EngPol', 'EngSym', 'EngFlo', |
| | 'EngNts', 'EngMikly', 'EngLab','EngBlk', 'EngWht', 'EngOpen','EngPav', |
| | 'Change_cts_value', 'Change_shape_value', 'Change_quality_value', 'Change_color_value', |
| | 'Change_cut_value', 'Change_Blk_Eng_to_Mkbl_value', 'Change_Wht_Eng_to_Mkbl_value', |
| | 'Change_Open_Eng_to_Mkbl_value', 'Change_Pav_Eng_to_Mkbl_value', 'Change_Blk_Eng_to_Grd_value', |
| | 'Change_Wht_Eng_to_Grd_value', 'Change_Open_Eng_to_Grd_value', 'Change_Pav_Eng_to_Grd_value', |
| | 'Change_Blk_Eng_to_ByGrd_value', 'Change_Wht_Eng_to_ByGrd_value', 'Change_Open_Eng_to_ByGrd_value', |
| | 'Change_Pav_Eng_to_ByGrd_value', 'Change_Blk_Eng_to_Gia_value', 'Change_Wht_Eng_to_Gia_value', |
| | 'Change_Open_Eng_to_Gia_value', 'Change_Pav_Eng_to_Gia_value' |
| | ] |
| |
|
| | |
| | loaded_label_encoder = {} |
| | enc_path = Path(LABEL_ENCODER_DIR) |
| | for val in encoder_list: |
| | encoder_file = enc_path / f"label_encoder_{val}.joblib" |
| | loaded_label_encoder[val] = load(encoder_file) |
| |
|
| | |
| | |
| | |
| | def allowed_file(filename): |
| | return '.' in filename and filename.rsplit('.', 1)[1].lower() in ALLOWED_EXTENSIONS |
| |
|
| | |
| | |
| | |
| | @app.route('/') |
| | def index(): |
| | return render_template('index.html') |
| |
|
| | @app.route('/predict', methods=['POST']) |
| | def predict(): |
| | if 'file' not in request.files: |
| | print('No file part', 'error') |
| | return redirect(url_for('index')) |
| | |
| | file = request.files['file'] |
| | if file.filename == '': |
| | print('No selected file', 'error') |
| | return redirect(url_for('index')) |
| | |
| | if file and allowed_file(file.filename): |
| | filename = secure_filename(file.filename) |
| | filepath = os.path.join(app.config['UPLOAD_FOLDER'], filename) |
| | file.save(filepath) |
| | |
| | |
| | try: |
| | if filename.endswith('.csv'): |
| | df = pd.read_csv(filepath) |
| | else: |
| | df = pd.read_excel(filepath) |
| | except Exception as e: |
| | print(f'Error reading file: {e}', 'error') |
| | return redirect(url_for('index')) |
| | |
| | |
| | df_pred, dx_class = process_dataframe(df) |
| | if df_pred.empty: |
| | print("Processed prediction DataFrame is empty. Check the input file and processing logic.", "error") |
| | return redirect(url_for('index')) |
| | |
| | |
| | current_date = pd.Timestamp.now().strftime("%Y-%m-%d") |
| | unique_id = uuid.uuid4().hex[:8] |
| | global PRED_OUTPUT_FILE, CLASS_OUTPUT_FILE |
| | PRED_OUTPUT_FILE = f'data/prediction_output_{current_date}_{unique_id}.csv' |
| | CLASS_OUTPUT_FILE = f'data/classification_output_{current_date}_{unique_id}.csv' |
| | df_pred.to_csv(PRED_OUTPUT_FILE, index=False) |
| | dx_class.to_csv(CLASS_OUTPUT_FILE, index=False) |
| | |
| | |
| | return redirect(url_for('report_view', report_type='pred', page=1)) |
| | else: |
| | print('Invalid file type. Only CSV and Excel files are allowed.', 'error') |
| | return redirect(url_for('index')) |
| |
|
| | def process_dataframe(df): |
| | try: |
| | |
| | |
| | |
| | required_columns = ['Tag', 'EngCts', 'EngShp', 'EngQua', 'EngCol', 'EngCut', 'EngPol', |
| | 'EngSym', 'EngFlo', 'EngNts', 'EngMikly', 'EngBlk', 'EngWht', 'EngOpen', |
| | 'EngPav', 'EngAmt'] |
| | required_columns_2 = ['Tag', 'EngCts', 'EngShp', 'EngQua', 'EngCol', 'EngCut', 'EngPol', |
| | 'EngSym', 'EngFlo', 'EngNts', 'EngMikly', 'EngAmt'] |
| | |
| | |
| | df_pred = df[required_columns].copy() |
| | df_pred = df_pred[(df_pred[['EngCts']] > 0.00).all(axis=1) & (df_pred[['EngCts']] <= 0.99).all(axis=1)] |
| | df_pred[['EngBlk', 'EngWht', 'EngOpen', 'EngPav']]=df_pred[['EngBlk', 'EngWht', 'EngOpen', 'EngPav']].fillna("NA") |
| | df_class = df[required_columns_2].fillna("NA").copy() |
| | |
| | |
| | for col in ['Tag', 'EngShp', 'EngQua', 'EngCol', 'EngCut', 'EngPol', 'EngSym', 'EngFlo', 'EngNts', 'EngMikly','EngBlk', 'EngWht', 'EngOpen', 'EngPav']: |
| | try: |
| | df_pred[col] = loaded_label_encoder[col].transform(df_pred[col]) |
| | except ValueError as e: |
| | print(f'Invalid value in column {col}: {e}', 'error') |
| | return pd.DataFrame(), pd.DataFrame() |
| | |
| | |
| | for col in ['Tag', 'EngShp', 'EngQua', 'EngCol', 'EngCut', 'EngPol', 'EngSym', 'EngFlo', 'EngNts', 'EngMikly']: |
| | df_class[col] = df_pred[col] |
| | |
| | |
| | |
| | |
| | |
| | |
| | |
| | |
| | |
| | |
| | df_pred = df_pred.astype(float) |
| | df_class = df_class.astype(float) |
| | |
| | |
| | |
| | |
| | try: |
| | x = df_pred.copy() |
| | |
| | |
| | |
| | |
| | |
| | df_pred = df_pred[['Tag', 'EngCts', 'EngShp', 'EngQua', 'EngCol', 'EngCut', 'EngPol', |
| | 'EngSym', 'EngFlo', 'EngNts', 'EngMikly', 'EngBlk', 'EngWht', 'EngOpen', |
| | 'EngPav', 'EngAmt', |
| | |
| | ]] |
| | df_pred['Makable_Predicted'] = pd.DataFrame(np.expm1(makable_model.predict(df_pred)), columns=["Predicted"]) |
| | print(df_pred.columns) |
| | |
| | |
| | |
| | df_pred['Makable_Diff'] = df_pred['EngAmt'] - df_pred['Makable_Predicted'] |
| | |
| | for col in ['Tag', 'EngShp', 'EngQua', 'EngCol', 'EngCut', 'EngPol', 'EngSym', 'EngFlo', 'EngNts', 'EngMikly','EngBlk', 'EngWht', 'EngOpen', 'EngPav']: |
| | try: |
| | df_pred[col] = loaded_label_encoder[col].inverse_transform(df_pred[col].astype(int)) |
| | except ValueError as e: |
| | print(f'inverse transform fails value in column {col}: {e}', 'error') |
| | |
| | except ValueError as e: |
| | print(f'pred model error----->: {e}', 'error') |
| | |
| | |
| | |
| | |
| | ''' |
| | try: |
| | x2 = df_class.copy() |
| | dx = df_pred.copy() # Start with the prediction data. |
| | dx['col_change'] = col_model.predict(x2) |
| | dx['cts_change'] = cts_model.predict(x2) |
| | dx['cut_change'] = cut_model.predict(x2) |
| | dx['qua_change'] = qua_model.predict(x2) |
| | dx['shp_change'] = shp_model.predict(x2) |
| | except ValueError as e: |
| | print(f'class model error----->: {e}', 'error') |
| | |
| | try: |
| | dx['Change_Blk_Eng_to_Mkbl_value'] = blk_eng_to_mkbl_model.predict(x) |
| | dx['Change_Wht_Eng_to_Mkbl_value'] = wht_eng_to_mkbl_model.predict(x) |
| | dx['Change_Open_Eng_to_Mkbl_value'] = open_eng_to_mkbl_model.predict(x) |
| | dx['Change_Pav_Eng_to_Mkbl_value'] = pav_eng_to_mkbl_model.predict(x) |
| | dx['Change_Blk_Eng_to_Grd_value'] = blk_eng_to_grade_model.predict(x) |
| | dx['Change_Wht_Eng_to_Grd_value'] = wht_eng_to_grade_model.predict(x) |
| | dx['Change_Open_Eng_to_Grd_value'] = open_eng_to_grade_model.predict(x) |
| | dx['Change_Pav_Eng_to_Grd_value'] = pav_eng_to_grade_model.predict(x) |
| | dx['Change_Blk_Eng_to_ByGrd_value'] = blk_eng_to_bygrade_model.predict(x) |
| | dx['Change_Wht_Eng_to_ByGrd_value'] = wht_eng_to_bygrade_model.predict(x) |
| | dx['Change_Open_Eng_to_ByGrd_value'] = open_eng_to_bygrade_model.predict(x) |
| | dx['Change_Pav_Eng_to_ByGrd_value'] = pav_eng_to_bygrade_model.predict(x) |
| | dx['Change_Blk_Eng_to_Gia_value'] = blk_eng_to_gia_model.predict(x) |
| | dx['Change_Wht_Eng_to_Gia_value'] = wht_eng_to_gia_model.predict(x) |
| | dx['Change_Open_Eng_to_Gia_value'] = open_eng_to_gia_model.predict(x) |
| | dx['Change_Pav_Eng_to_Gia_value'] = pav_eng_to_gia_model.predict(x) |
| | except ValueError as e: |
| | print(f'grade_code model error----->: {e}', 'error') |
| | |
| | # Inverse transform classification predictions. |
| | dx['col_change'] = loaded_label_encoder['Change_color_value'].inverse_transform(dx['col_change']) |
| | dx['cts_change'] = loaded_label_encoder['Change_cts_value'].inverse_transform(dx['cts_change']) |
| | dx['cut_change'] = loaded_label_encoder['Change_cut_value'].inverse_transform(dx['cut_change']) |
| | dx['qua_change'] = loaded_label_encoder['Change_quality_value'].inverse_transform(dx['qua_change']) |
| | dx['shp_change'] = loaded_label_encoder['Change_shape_value'].inverse_transform(dx['shp_change']) |
| | dx['Change_Blk_Eng_to_Mkbl_value'] = loaded_label_encoder['Change_Blk_Eng_to_Mkbl_value'].inverse_transform(dx['Change_Blk_Eng_to_Mkbl_value']) |
| | dx['Change_Wht_Eng_to_Mkbl_value'] = loaded_label_encoder['Change_Wht_Eng_to_Mkbl_value'].inverse_transform(dx['Change_Wht_Eng_to_Mkbl_value']) |
| | dx['Change_Open_Eng_to_Mkbl_value'] = loaded_label_encoder['Change_Open_Eng_to_Mkbl_value'].inverse_transform(dx['Change_Open_Eng_to_Mkbl_value']) |
| | dx['Change_Pav_Eng_to_Mkbl_value'] = loaded_label_encoder['Change_Pav_Eng_to_Mkbl_value'].inverse_transform(dx['Change_Pav_Eng_to_Mkbl_value']) |
| | dx['Change_Blk_Eng_to_Grd_value'] = loaded_label_encoder['Change_Blk_Eng_to_Grd_value'].inverse_transform(dx['Change_Blk_Eng_to_Grd_value']) |
| | dx['Change_Wht_Eng_to_Grd_value'] = loaded_label_encoder['Change_Wht_Eng_to_Grd_value'].inverse_transform(dx['Change_Wht_Eng_to_Grd_value']) |
| | dx['Change_Open_Eng_to_Grd_value'] = loaded_label_encoder['Change_Open_Eng_to_Grd_value'].inverse_transform(dx['Change_Open_Eng_to_Grd_value']) |
| | dx['Change_Pav_Eng_to_Grd_value'] = loaded_label_encoder['Change_Pav_Eng_to_Grd_value'].inverse_transform(dx['Change_Pav_Eng_to_Grd_value']) |
| | dx['Change_Blk_Eng_to_ByGrd_value'] = loaded_label_encoder['Change_Blk_Eng_to_ByGrd_value'].inverse_transform(dx['Change_Blk_Eng_to_ByGrd_value']) |
| | dx['Change_Wht_Eng_to_ByGrd_value'] = loaded_label_encoder['Change_Wht_Eng_to_ByGrd_value'].inverse_transform(dx['Change_Wht_Eng_to_ByGrd_value']) |
| | dx['Change_Open_Eng_to_ByGrd_value'] = loaded_label_encoder['Change_Open_Eng_to_ByGrd_value'].inverse_transform(dx['Change_Open_Eng_to_ByGrd_value']) |
| | dx['Change_Pav_Eng_to_ByGrd_value'] = loaded_label_encoder['Change_Pav_Eng_to_ByGrd_value'].inverse_transform(dx['Change_Pav_Eng_to_ByGrd_value']) |
| | dx['Change_Blk_Eng_to_Gia_value'] = loaded_label_encoder['Change_Blk_Eng_to_Gia_value'].inverse_transform(dx['Change_Blk_Eng_to_Gia_value']) |
| | dx['Change_Wht_Eng_to_Gia_value'] = loaded_label_encoder['Change_Wht_Eng_to_Gia_value'].inverse_transform(dx['Change_Wht_Eng_to_Gia_value']) |
| | dx['Change_Open_Eng_to_Gia_value'] = loaded_label_encoder['Change_Open_Eng_to_Gia_value'].inverse_transform(dx['Change_Open_Eng_to_Gia_value']) |
| | dx['Change_Pav_Eng_to_Gia_value'] = loaded_label_encoder['Change_Pav_Eng_to_Gia_value'].inverse_transform(dx['Change_Pav_Eng_to_Gia_value']) |
| | |
| | ''' |
| | |
| | |
| | return df_pred, df_pred |
| | except Exception as e: |
| | print(f'Error processing file: {e}', 'error') |
| | return pd.DataFrame(), pd.DataFrame() |
| |
|
| | |
| | |
| | |
| | @app.route('/report') |
| | def report_view(): |
| | report_type = request.args.get('report_type', 'pred') |
| | try: |
| | page = int(request.args.get('page', 1)) |
| | except ValueError: |
| | page = 1 |
| | per_page = 15 |
| |
|
| | |
| | if report_type == 'pred': |
| | df = pd.read_csv(PRED_OUTPUT_FILE) |
| | else: |
| | df = pd.read_csv(CLASS_OUTPUT_FILE) |
| |
|
| | start_idx = (page - 1) * per_page |
| | end_idx = start_idx + per_page |
| | total_records = len(df) |
| | |
| | df_page = df.iloc[start_idx:end_idx] |
| | table_html = df_page.to_html(classes="data-table", index=False) |
| | |
| | has_prev = page > 1 |
| | has_next = end_idx < total_records |
| | |
| | return render_template('output.html', |
| | table_html=table_html, |
| | report_type=report_type, |
| | page=page, |
| | has_prev=has_prev, |
| | has_next=has_next) |
| |
|
| | |
| | |
| | |
| | @app.route('/download_pred', methods=['GET']) |
| | def download_pred(): |
| | return send_file(PRED_OUTPUT_FILE, as_attachment=True) |
| |
|
| | @app.route('/download_class', methods=['GET']) |
| | def download_class(): |
| | return send_file(CLASS_OUTPUT_FILE, as_attachment=True) |
| |
|
| | if __name__ == "__main__": |
| | app.run(debug=True) |
| |
|