Master/Project_Sathish

src/dependencies/cleaning/cleaner.py

@@ -0,0 +1,22 @@
+import datetime
+import pandas as pd
+
+class Cleaner():
+
+    def __init__(self):
+
+        # Use The Path of Raw Data
+        self.df = pd.read_csv(r'C:\Users\ox\PycharmProjects\Prism_v01\Taiyo\Sats_Kanna\taiyo.csv')
+
+    def clean_date(self, value):
+        date_obj = datetime.datetime.strptime(value, '%m-%d-%Y')
+        date = datetime.datetime.strftime(date_obj, '%Y-%m-%d')
+        return date
+
+    def cleaner(self):
+        print('Cleaner Triggered')
+        self.df['date'] = self.df['time'].apply(lambda d: self.clean_date(d))
+        self.df.drop(columns=['time'], inplace=True)
+        # Execute The Required Cleaner Function Here
+
+        self.df.to_csv('cleaner.csv', index=False)

src/dependencies/geocoding/geocoding.py

@@ -0,0 +1,36 @@
+from geopy.geocoders import Nominatim
+import re
+import pandas as pd
+
+
+class CSVConverter:
+    def __init__(self, csv_file, address_column):
+        self.csv_file = csv_file
+        self.address_column = address_column
+        self.geolocator = Nominatim(user_agent="my_geocoder")
+
+    def get_location(self, address):
+        address = re.sub(r'([A-Z][A-z]+)([A-Z][A-z]+)', r'\2, \1', address)
+        print(address)
+
+        # Geocode the address
+        location = self.geolocator.geocode(address)
+
+        if location:
+            latitude = location.latitude
+            longitude = location.longitude
+            return (latitude, longitude)
+        else:
+            print(f"Unable to retrieve coordinates for the address: {address}")
+            return None
+
+    def get_coordinates(self):
+        df = pd.read_csv(self.csv_file)
+        df['geo_fields'] = df[self.address_column].apply(lambda c: self.get_location(c))
+        print(df['geo_fields'])
+        df['latitude'] = df['geo_fields'].apply(lambda gf: gf[0] if gf else '')
+        df['longitude'] = df['geo_fields'].apply(lambda gf: gf[1] if gf else '')
+        df.drop(columns=['geo_fields'], inplace=True)
+        df.to_csv('geocoder.csv', index=False)
+
+

src/dependencies/main.py

@@ -0,0 +1,12 @@
+from scraper.scrape import Scraper
+from cleaner.cleaner import Cleaner
+from geocoding.geocoding import CSVConverter
+
+if __name__ == '__main__':
+    scraper_object = Scraper()
+    cleaner_object = Cleaner()
+    # scraper_object.scrape_data()
+    cleaner_object.cleaner()
+    '''Give the DF and Column Name to get Geocoding'''
+    geocode_object = CSVConverter(r'C:\Users\ox\PycharmProjects\Prism_v01\Taiyo\Sats_Kanna\cleaner.csv', 'Region')
+    geocode_object.get_coordinates()

src/dependencies/requirements.txt

@@ -0,0 +1,4 @@
+pandas==1.4.2
+bs4==0.0.1
+geopy==2.4.0
+matplotlib==3.7.1

src/dependencies/scraping/scrape.py

@@ -0,0 +1,85 @@
+from bs4 import BeautifulSoup
+import pandas as pd
+import re
+import requests
+import json
+import logging
+
+class Scraper():
+
+    def __init__(self):
+        self.dict_ = []
+        self.main_link = 'http://en.chinabidding.mofcom.gov.cn/zbwcms/front/en/bidding/bulletinList'
+        self.header = {
+            'Accept': 'text/html, */*; q=0.01',
+            'Accept-Encoding': 'gzip, deflate',
+            'Accept-Language': 'en-US,en;q=0.9,bn;q=0.8',
+            'Connection': 'keep-alive',
+            'Content-Length': '83',
+            'Content-Type': 'application/x-www-form-urlencoded; charset=UTF-8',
+            'Cookie': 'insert_cookie=67183482',
+            'Host': 'en.chinabidding.mofcom.gov.cn',
+            'Origin': 'http://en.chinabidding.mofcom.gov.cn',
+            'Referer': 'http://en.chinabidding.mofcom.gov.cn/channel/EnSearchList.shtml?tenders=1',
+            'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/118.0.0.0 Safari/537.36',
+            'X-Requested-With': 'XMLHttpRequest',
+        }
+
+
+    @staticmethod
+    def get_dictionary(s_no=None, title=None, body=None, industry=None, region=None, source=None, time=None):
+        dictionary = {
+            's_no': s_no,
+            'title': title,
+            'body': body,
+            'industry': industry,
+            'region': region,
+            'source': source,
+            'time': time,
+        }
+        return dictionary
+
+    def scrape_data(self):
+        print('Scraper Triggered')
+        page_payload = {
+            'pageNumber': 1,
+            'pageSize': '10',
+            'type': '1',
+            'industry': '',
+            'provinceCode': '',
+            'keyword': '',
+            'capitalSourceCode': '',
+        }
+        page_req = requests.post(self.main_link, data=page_payload)
+        page_data = json.loads(page_req.text)
+        # page_count = page_data['maxPageNum']
+        '''page count for sample data'''
+        page_count = 10
+        print(f'{page_count} Pages To Scrape')
+        for i in range(1, page_count):
+            payload = {
+            'pageNumber': f'{i}',
+            'pageSize': '100',
+            'type': '1',
+            'industry': '',
+            'provinceCode': '',
+            'keyword': '',
+            'capitalSourceCode': '',
+            }
+            r = requests.post(self.main_link, data=payload)
+            data = json.loads(r.text)
+            rows = data['rows']
+
+            for rows_ in rows:
+                industry = rows_['industryName']
+                region = rows_['areaName']
+                source = rows_['capitalSourceName']
+                title = rows_['name']
+                body = rows_['digest']
+                time = rows_['publishTime']
+
+                dictionary = self.get_dictionary(s_no=1, title=title, body=body, industry=industry, region=region, source=source, time=time)
+                self.dict_.append(dictionary)
+        df = pd.DataFrame(self.dict_)
+        df.to_csv(f'taiyo.csv', index=False)
+

src/dependencies/standardization/standardizer.py

@@ -0,0 +1,59 @@
+import os
+
+import matplotlib.pyplot as plt
+import pandas as pd
+
+
+class ChartGenerator:
+    def __init__(self, data):
+        self.data = data
+        self.save_path = os.getcwd()
+
+    def bar_chart(self, x_col, y_col, title, xlabel, ylabel):
+        plt.figure(figsize=(10, 6))
+        plt.bar(self.data[x_col], self.data[y_col])
+        plt.title(title)
+        plt.xlabel(xlabel)
+        plt.ylabel(ylabel)
+        plt.xticks(rotation=45)
+        plt.tight_layout()
+        file_name = os.path.join(self.save_path,'bar_chart')
+
+        plt.savefig(file_name, bbox_inches='tight')
+
+    def scatter_plot(self, x_col, y_col, title, xlabel, ylabel):
+        plt.figure(figsize=(10, 6))
+        plt.scatter(self.data[x_col], self.data[y_col])
+        plt.title(title)
+        plt.xlabel(xlabel)
+        plt.ylabel(ylabel)
+        plt.tight_layout()
+        file_name = os.path.join(self.save_path,'scatter_plot')
+        plt.savefig(file_name, bbox_inches='tight')
+
+    def line_chart(self, x_col, y_col, title, xlabel, ylabel):
+        plt.figure(figsize=(10, 6))
+        plt.plot(self.data[x_col], self.data[y_col])
+        plt.title(title)
+        plt.xlabel(xlabel)
+        plt.ylabel(ylabel)
+        plt.grid(True)
+        plt.tight_layout()
+        plt.show()
+
+
+if __name__ == "__main__":
+    # Read your CSV data into a DataFrame
+    data = pd.read_csv('geocoder.csv')
+
+    # Create an instance of the ChartGenerator class
+    chart_generator = ChartGenerator(data)
+
+    # Generate a bar chart
+    chart_generator.bar_chart('Region', 's_no', 'Bar Chart Example', 'Region', 'Serial Number')
+
+    # Generate a scatter plot
+    chart_generator.scatter_plot('latitude', 'longitude', 'Scatter Plot Example', 'Latitude', 'Longitude')
+
+    # Generate a line chart
+    # chart_generator.line_chart('date', 'body', 'Line Chart Example', 'Date', 'Body')