from db.DatabaseEngine import DatabaseEngine import pandas as pd from recommender.queries.SqlQueries import PRODUCT_QUERY, VISITOR_QUERY import os import numpy as np from sklearn.preprocessing import LabelEncoder from utils.DatasetHelper import dataset_helper from lightfm import LightFM from lightfm.evaluation import auc_score from scipy import sparse from flask import current_app import pickle import logging from dotenv import load_dotenv class RelatedProductRecommender: DEBUG = True def __init__(self): load_dotenv() self.pickle_file_location = os.getenv("PICKLE_FILES_LOCATION") self.input_data_location = os.getenv("INPUT_DATA_LOCATION") self.related_products_pkl = os.getenv("RELATED_PRODUCTS_PKL_FILE") self.related_products_helper_pkl = os.getenv("RELATED_PRODUCTS_HELPER_PKL_FILE") self.value = "test" self.db = DatabaseEngine(); model_file = os.path.join(self.pickle_file_location, self.related_products_pkl) dataset_helper_file = os.path.join(self.pickle_file_location, self.related_products_helper_pkl) with open(model_file, 'rb') as file: self.model = pickle.load(file) with open(dataset_helper_file, 'rb') as file: self.dataset_helper = pickle.load(file) self.item_representations = self.model.item_embeddings self.tain_model = LightFM(no_components=24, loss="warp", k=15) self.le = LabelEncoder() self.inv_item_mappings = '' self.user_mappings = '' self.dataset_helper_instance = None self.is_trained = False def findCustomerId(self, x): cid=None if str(x['cid']) == 'nan' or x['cid'] is None or x['cid'] <= 0: cid = x['customer_ip'] else: cid = x['cid'] return str(cid) def trainModel(self): prospectVisitDF = self.getVisitData() prospectVisitDF["visited_page_id"] = prospectVisitDF["visited_page_id"].map(lambda x: x.lower() if isinstance(x,str) else x) prospectVisitDF["customer_id"] = prospectVisitDF.apply(lambda x: self.findCustomerId(x), axis=1) prospectVisitDF = prospectVisitDF.query('isVisited == True or isCategory == True or isProduct == True') prospectVisitDF_Group_Customer_page = prospectVisitDF.groupby(["customer_id", "visited_page_id", "isProduct", "isVisited"]).count().reset_index() data_prerating_setup=prospectVisitDF_Group_Customer_page.groupby(['customer_id', 'visited_page_id', 'isProduct']).size().reset_index(name="count") final_data = data_prerating_setup final_data['rating'] = np.where(final_data['isProduct'] == 0, np.log(final_data['count']), np.log(final_data['count'] * 50)) productDF = self.getProductData() productDF[["cat_name_1", "cat_name_2", "cat_name_3", "cat_name_4", "cat_name_5"]] = productDF['cat_name'].str.split('/', expand=True) self.final_merged = pd.merge(productDF, final_data, right_on='visited_page_id',left_on='product_id', how='outer' ) self.baseDataSetupProduct(self.final_merged) self.tain_model.fit(interactions=self.dataset_helper_instance.interactions, sample_weight=self.dataset_helper_instance.weights, item_features=self.dataset_helper_instance.item_features_list, user_features=self.dataset_helper_instance.user_features_list, verbose=True, epochs=10, num_threads=20) train_auc = auc_score(self.model, self.dataset_helper_instance.interactions, item_features=self.dataset_helper_instance.item_features_list, user_features=self.dataset_helper_instance.user_features_list, num_threads=2).mean() self.user_mappings = self.dataset_helper_instance.dataset._user_id_mapping self.item_mappings = self.dataset_helper_instance.dataset._item_id_mapping self.inv_user_mappings = {v:k for k, v in self.user_mappings.items()} self.inv_item_mappings = {v:k for k, v in self.item_mappings.items()} logging.info('Hybrid training set AUC: %s' % train_auc) self.is_trained = True with open(os.path.join(self.pickle_file_location, 'lofl-related-product-recomendataion-model.pickle'), 'wb') as fle: pickle.dump(self.model, fle, protocol=pickle.HIGHEST_PROTOCOL) with open(os.path.join(self.pickle_file_location, 'related_dataset_helper.pickle'), 'wb') as fle: pickle.dump(self.dataset_helper_instance, fle, protocol=pickle.HIGHEST_PROTOCOL) return train_auc; def getRecommendation(self, search_term, cid, count): keys = [] try: count = count+1 data = self.getRelatedProducts(search_term, self.item_representations, count) result = [] for item in data: result.append(int(item[0])) keys = [int(k) for k, v in self.dataset_helper.get_item_id_mapping().items() if v in result and int(k) != int(search_term)] except Exception as e: logging.error("Error getting related recommendations", e) return keys def getProductData(self): file_path = os.path.join(self.input_data_location, 'trash_data/Product_data.csv') return pd.read_csv(file_path, sep="~") def getVisitData(self): file_path = os.path.join(self.input_data_location, 'trash_data/customer_visit.csv') return pd.read_csv(file_path, sep="~") def getCustomerData(self): df = pd.read_sql_query("Select * from customer", con=self.db.getEngine()) return df def getRelatedProducts(self, target_item_id, item_representations, N=10): # 1. Get the representation of the target item id = self.dataset_helper.get_item_id_mapping().get(int(target_item_id)) target_representation = item_representations[id] # 2. Compute cosine similarity scores scores = item_representations.dot(target_representation) # 3. Calculate magnitudes for normalization norms = np.linalg.norm(item_representations, axis=1) # Prevent division by zero norms[norms == 0] = 1e-9 # Normalize the scores cosine_similarities = scores / (norms * np.linalg.norm(target_representation)) # 4. Find the top N most similar items # Use argpartition to quickly get top N indices, then sort them top_indices = np.argpartition(cosine_similarities, -N)[-N:] top_indices = top_indices[np.argsort(-cosine_similarities[top_indices])] return [(idx, cosine_similarities[idx]) for idx in top_indices] def baseDataSetupPageVisit(self, df): data_prerating_setup=df.query("isProduct == 1").groupby(['visited_page_id', 'isProduct']).size().reset_index(name="count") final_data = data_prerating_setup final_data_sorted = final_data.sort_values(by=['isProduct', 'count'], ascending=[True, False]) #final_data_sorted['adjusted_count'] = np.where(final_data_sorted['isProduct'] == 0, final_data_sorted['count'], final_data_sorted['count'] * 100) #final_data_sorted['rating'] = final_data.reset_index(drop=True).index + 1 #final_data_sorted['rating'] = final_data_sorted['adjusted_count'].rank(ascending=False, method='max') #np.where(final_data_sorted['isProduct'] == 0, np.log(final_data_sorted['count']), np.log(final_data_sorted['count'] * 500)) final_data_sorted['rating'] = pd.qcut(final_data_sorted['count'], q=5, labels=[1,2,3,4,5]) customer_product_rating = pd.merge(df.query("isProduct == 1"), final_data_sorted, on='visited_page_id', how='left').sort_index() items_column = "visited_page_id" user_column = "customer_id" ratings_column = "rating" items_feature_columns = ["visited_page_id", "isCategory", "featured", "new", "price", "special_price", "isProduct_y", "url_key", "brand", "packaging_type","alcohol_percentage", "volume", "flash_sale_price", "overall"] user_features_columns = ["browser", "device"] self.dataset_helper_instance = dataset_helper( users_dataframe=customer_product_rating, items_dataframe=customer_product_rating, interactions_dataframe=customer_product_rating, item_id_column=items_column, items_feature_columns=items_feature_columns, user_id_column=user_column, user_features_columns=user_features_columns, interaction_column=ratings_column, clean_unknown_interactions=True, fix_columns_names=False, ) self.dataset_helper_instance.routine()