evaluated-algorithm.ipynb

{"cells":[{"source":"<a href=\"https://www.kaggle.com/code/gpreda/evaluated-algorithm?scriptVersionId=128762016\" target=\"_blank\"><img align=\"left\" alt=\"Kaggle\" title=\"Open in Kaggle\" src=\"https://kaggle.com/static/images/open-in-kaggle.svg\"></a>","metadata":{},"cell_type":"markdown"},{"cell_type":"code","execution_count":1,"id":"cc9a8ae6","metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","execution":{"iopub.execute_input":"2023-05-08T12:42:17.008073Z","iopub.status.busy":"2023-05-08T12:42:17.007545Z","iopub.status.idle":"2023-05-08T12:42:17.144278Z","shell.execute_reply":"2023-05-08T12:42:17.143263Z"},"papermill":{"duration":0.144531,"end_time":"2023-05-08T12:42:17.147277","exception":false,"start_time":"2023-05-08T12:42:17.002746","status":"completed"},"tags":[]},"outputs":[],"source":["##-----------------------------------------------------------------------------------##\n","#\n","# Source: adapted for Kaggle from \n","#         Building Recommender Systems with Machine Learning and AI, Sundog Education\n","#\n","##----------------------------------------------------------------------------------##\n","\n","from recommender_metrics import RecommenderMetrics\n","from evaluation_data import EvaluationData\n","\n","class EvaluatedAlgorithm:\n","    \n","    def __init__(self, algorithm, name):\n","        self.algorithm = algorithm\n","        self.name = name\n","        \n","    def evaluate(self, evaluation_data, do_top_n, n=10, verbose=True):\n","        metrics = {}\n","        # Compute accuracy\n","        if (verbose):\n","            print(\"Evaluating accuracy...\")\n","        self.algorithm.fit(evaluation_data.get_train_set())\n","        predictions = self.algorithm.test(evaluation_data.get_test_set())\n","        metrics[\"RMSE\"] = RecommenderMetrics.rmse(predictions)\n","        metrics[\"MAE\"] = RecommenderMetrics.mae(predictions)\n","        metrics[\"FCP\"] = RecommenderMetrics.fcp(predictions)\n","        \n","        if (do_top_n):\n","            # Evaluate top-10 with Leave One Out testing\n","            if (verbose):\n","                print(\"Evaluating top-N with leave-one-out...\")\n","            self.algorithm.fit(evaluation_data.get_loocv_train_set())\n","            left_out_predictions = self.algorithm.test(evaluation_data.get_loocv_test_set())        \n","            # Build predictions for all ratings not in the training set\n","            all_predictions = self.algorithm.test(evaluation_data.get_loocv_anti_test_set())\n","            # Compute top 10 recs for each user\n","            top_n_predicted = RecommenderMetrics.get_top_n(all_predictions, n)\n","            if (verbose):\n","                print(\"Computing hit-rate and rank metrics...\")\n","            # See how often we recommended a movie the user actually rated\n","            metrics[\"HR\"] = RecommenderMetrics.hit_rate(top_n_predicted, left_out_predictions)   \n","            # See how often we recommended a movie the user actually liked\n","            metrics[\"cHR\"] = RecommenderMetrics.cummulative_hit_rate(top_n_predicted, left_out_predictions)\n","            # Compute ARHR\n","            metrics[\"ARHR\"] = RecommenderMetrics.average_reciprocal_hit_rank(top_n_predicted, left_out_predictions)\n","        \n","            #Evaluate properties of recommendations on full training set\n","            if (verbose):\n","                print(\"Computing recommendations with full data set...\")\n","            self.algorithm.fit(evaluation_data.get_full_train_set())\n","            all_predictions = self.algorithm.test(evaluation_data.get_full_anti_test_set())\n","            top_n_predicted = RecommenderMetrics.get_top_n(all_predictions, n)\n","            if (verbose):\n","                print(\"Analyzing coverage, diversity, and novelty...\")\n","            # Print user coverage with a minimum predicted rating of 4.0:\n","            metrics[\"Coverage\"] = RecommenderMetrics.user_coverage(top_n_predicted, \n","                                                                   evaluation_data.get_full_train_set().n_users, \n","                                                                   rating_threshold=4.0)\n","            # Measure diversity of recommendations:\n","            metrics[\"Diversity\"] = RecommenderMetrics.diversity(top_n_predicted, evaluation_data.get_similarities())\n","            \n","            # Measure novelty (average popularity rank of recommendations):\n","            metrics[\"Novelty\"] = RecommenderMetrics.novelty(top_n_predicted, \n","                                                            evaluation_data.get_popularity_rankings())\n","        \n","        if (verbose):\n","            print(\"Analysis complete.\")\n","    \n","        return metrics\n","    \n","    def get_name(self):\n","        return self.name\n","    \n","    def get_algorithm(self):\n","        return self.algorithm\n","    "]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.7.12"},"papermill":{"default_parameters":{},"duration":11.079493,"end_time":"2023-05-08T12:42:17.771023","environment_variables":{},"exception":null,"input_path":"__notebook__.ipynb","output_path":"__notebook__.ipynb","parameters":{},"start_time":"2023-05-08T12:42:06.69153","version":"2.4.0"}},"nbformat":4,"nbformat_minor":5}