From f8f8a21593c77d7695ff857fb9d74e7935122595 Mon Sep 17 00:00:00 2001 From: "pixeebot[bot]" <104101892+pixeebot[bot]@users.noreply.github.com> Date: Fri, 1 Aug 2025 03:39:29 +0000 Subject: [PATCH] Sandbox URL Creation --- Air pollution prediction/CodeAP.py | 5 +- All_GitHub_Repos/all_github_repos.py | 122 +++--- Amazon Best Sellers Scraper/script.py | 4 +- Amazon Price Tracker/script.py | 4 +- Amazon Wishlist Notifier/script.py | 4 +- Amazon-Price-Tracker/amazonprice.py | 4 +- Amazon_product_scraper/products.py | 10 +- AskUbuntu-Scraper/questions.py | 4 +- Automated_scraper.py/script.py | 3 +- BITCOIN-price-tracker/tracker.py | 4 +- Book_Scraper/book.py | 6 +- CNN Scraper/cnn.py | 6 +- Covid-19_Real-time_Notification/Covid.py | 4 +- Crypocurrency-Converter-GUI/main.py | 3 +- Currency-Exchange-Rates/exchange_rates.py | 3 +- Current_City_Weather/Current_City_Weather.py | 4 +- DataScraping/Data_Scraping.py | 6 +- Desktop News Notifier/script.py | 4 +- Discord News Bot With AI/main.py | 4 +- Doh-Dig/dog-dig.py | 4 +- Domain_Name_Availability/Domain_name.py | 132 +++---- EazyDiner-Scraper/restaurants.py | 4 +- Facebook-DP-Downloader/fb_dp_downloader.py | 4 +- Facebook_Video_Downloader/script.py | 7 +- .../Fetch_all_links_from_webpage.py | 40 +- Fetch_Contributions/fetch_contributions.py | 360 +++++++++--------- File Downloader Script/file_downloader.py | 4 +- File Downloader/file_downloader.py | 6 +- FileDownload Automationi/script.py | 4 +- Flipkart-price-alert/track.py | 5 +- FlipkartScraper/genricHtmlib.py | 4 +- Flipkart_webscraping/Scrap.py | 4 +- Github-Size-Contributor-List/script.py | 6 +- Github-User-Scraper/scraper.py | 4 +- Github_User_Info/main.py | 4 +- Google-Image-Scrapper/scrapper.py | 4 +- Google-News-Scraapper/app.py | 4 +- GoogleNews-Scraper/googleNews.py | 4 +- IMDB-Scraper/scraper.py | 6 +- IPL Statistics GUI/ipl.py | 4 +- Icc_ranking_scraper/rankings.py | 6 +- Image Downloader GUI/main.py | 142 +++---- Image-Scraper/scrape_images.py | 4 +- Image-scrape-download/image.py | 4 +- Instagram downloader/downloader.py | 6 +- .../Instagram Profile Pic Downloader.py | 28 +- Internshala-Scraper/program.py | 4 +- Job Scraper/JobScraper.py | 224 +++++------ Link Scrapper/script.py | 4 +- Link-Preview/linkPreview.py | 4 +- LinkedIn_Profile_Info/google_API.py | 3 +- LinkedIn_Profile_Info/profile.py | 3 +- .../Linkedin_Posts_Scrapper.py | 48 +-- Linkedin_Video_Downloader/script.py | 6 +- Live Currency Converter GUI/app.py | 172 ++++----- Lyrics_Finder/script.py | 4 +- Movie recommendation system/app.py | 4 +- Movie-Info-Telegram-Bot/bot.py | 8 +- Movie-Recommendation/script.py | 4 +- News_Scrapper/scrapper.py | 6 +- Pagespeed-API/pagespeed.py | 4 +- Pixabay Scraper/main.py | 10 +- .../price_comparison.py | 8 +- Pwned_or_not_GUI/pwned_passwords_GUI.py | 4 +- Rain Alert Mail Sender/main.py | 6 +- .../real_estate_webscrapper.py | 4 +- Reddit_Scraper_without_API/fetch_reddit.py | 6 +- .../search-scripts.py | 4 +- Search-Username/search_user.py | 4 +- .../slideshare_downloader.py | 4 +- Social-Media-Links-From-Website/script.py | 6 +- Space News AI Model/Space_News.py | 16 +- Spacex-Calender/notifly.py | 4 +- Stack-overflow-scraper/scraper.py | 4 +- StackOverflow-Scraper/questions.py | 4 +- Stackoverflow-Tool/script.py | 4 +- Swiggy_Scraper/main.py | 4 +- TOI_Scrapper/TOI_Scrapper.py | 62 +-- TechCrunch-Scraper/techCrunch.py | 6 +- Telegram-Weather-Bot/main.py | 4 +- Telegram_Book_Bot/book.py | 6 +- Temporary-Sign-Up-Tool/temp_sign_up_tool.py | 6 +- Text-to-Image/dalle.py | 5 +- TradingView/setup.py | 2 +- Twitter_Video_Downloader/script.py | 4 +- Weather-App/weatherapp.py | 4 +- .../Web_Application_Mapper.py | 3 +- Wiki_Infobox_Scraper/main.py | 4 +- Wiki_Scrapper/wiki_Scraper.py | 4 +- YouTube-Scraper/video.py | 4 +- Youtube_Scraper/main.py | 4 +- .../emails-from-website.py | 4 +- 92 files changed, 867 insertions(+), 853 deletions(-) diff --git a/Air pollution prediction/CodeAP.py b/Air pollution prediction/CodeAP.py index f8633707ff..340be9a625 100644 --- a/Air pollution prediction/CodeAP.py +++ b/Air pollution prediction/CodeAP.py @@ -1,11 +1,12 @@ -import requests import matplotlib.pyplot as plt +from security import safe_requests + city = input("Enter you city : ") url = 'http://api.waqi.info/feed/' + city + '/?token=' api_key = input("Enter your API key: ") main_url = url + api_key -r = requests.get(main_url) +r = safe_requests.get(main_url) data = r.json()['data'] aqi = data['aqi'] iaqi = data['iaqi'] diff --git a/All_GitHub_Repos/all_github_repos.py b/All_GitHub_Repos/all_github_repos.py index 70fe6c0746..8707d61a91 100644 --- a/All_GitHub_Repos/all_github_repos.py +++ b/All_GitHub_Repos/all_github_repos.py @@ -1,61 +1,61 @@ -#!/usr/bin/env python3 - -# to convert it into a script by running sudo chmod +x all_github_repos.py - -import requests -import sys -from github import Github - -# imports -# pip3/pip install PyGithub is installed to work with the contents of the Github repositories - -username = sys.argv[1] -# reading the username as a commandline argument - -url = f"https://api.github.com/users/{username}" - -user_data = requests.get(url).json() -# to retrieve data contained in the url in json format - - -def repository_names(user): - repo_names = [] - for repo in user.get_repos(): - repo_names.append(repo) - return repo_names - - -# fetching the names of all the repositories - - -def repository_details(user): - all_repo_details = [] - repo_names = repository_names(user) - for repo in repo_names: - repo_details = {} - repo_details["Name"] = repo.full_name.split("/")[1] - repo_details["Description"] = repo.description - repo_details["Created on"] = repo.created_at - repo_details["Programming language"] = repo.language - repo_details["Forked"] = str(repo.forks) + " time(s)" - all_repo_details.append(repo_details) - return all_repo_details - - -# fetching the details of all the repositories - - -user = Github().get_user(username) - -RD = repository_details(user) -# fetching the details of all repositories -# stored as a list of dictionaries - -if __name__ == "__main__": - for content in RD: - # pprint.pprint(content) - for title, description in content.items(): - print(title, ":", description) - print( - "\n-------------------------------------------------------------------------------------------------------------------\n" - ) +#!/usr/bin/env python3 + +# to convert it into a script by running sudo chmod +x all_github_repos.py + +import sys +from github import Github +from security import safe_requests + +# imports +# pip3/pip install PyGithub is installed to work with the contents of the Github repositories + +username = sys.argv[1] +# reading the username as a commandline argument + +url = f"https://api.github.com/users/{username}" + +user_data = safe_requests.get(url).json() +# to retrieve data contained in the url in json format + + +def repository_names(user): + repo_names = [] + for repo in user.get_repos(): + repo_names.append(repo) + return repo_names + + +# fetching the names of all the repositories + + +def repository_details(user): + all_repo_details = [] + repo_names = repository_names(user) + for repo in repo_names: + repo_details = {} + repo_details["Name"] = repo.full_name.split("/")[1] + repo_details["Description"] = repo.description + repo_details["Created on"] = repo.created_at + repo_details["Programming language"] = repo.language + repo_details["Forked"] = str(repo.forks) + " time(s)" + all_repo_details.append(repo_details) + return all_repo_details + + +# fetching the details of all the repositories + + +user = Github().get_user(username) + +RD = repository_details(user) +# fetching the details of all repositories +# stored as a list of dictionaries + +if __name__ == "__main__": + for content in RD: + # pprint.pprint(content) + for title, description in content.items(): + print(title, ":", description) + print( + "\n-------------------------------------------------------------------------------------------------------------------\n" + ) diff --git a/Amazon Best Sellers Scraper/script.py b/Amazon Best Sellers Scraper/script.py index 781e1af4a9..84c18cc17e 100644 --- a/Amazon Best Sellers Scraper/script.py +++ b/Amazon Best Sellers Scraper/script.py @@ -1,11 +1,11 @@ -import requests from bs4 import BeautifulSoup +from security import safe_requests def scrape_amazon_bestsellers(category_url): headers = { "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.3"} - response = requests.get(category_url, headers=headers) + response = safe_requests.get(category_url, headers=headers) if response.status_code == 200: soup = BeautifulSoup(response.content, 'html.parser') diff --git a/Amazon Price Tracker/script.py b/Amazon Price Tracker/script.py index 00796fedf0..8ce348fc68 100644 --- a/Amazon Price Tracker/script.py +++ b/Amazon Price Tracker/script.py @@ -1,15 +1,15 @@ -import requests from bs4 import BeautifulSoup import time import smtplib from email.mime.text import MIMEText +from security import safe_requests def get_amazon_product_price(product_url): headers = { "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.3" } - response = requests.get(product_url, headers=headers) + response = safe_requests.get(product_url, headers=headers) if response.status_code == 200: soup = BeautifulSoup(response.content, 'html.parser') diff --git a/Amazon Wishlist Notifier/script.py b/Amazon Wishlist Notifier/script.py index 1896482140..04fa6ed044 100644 --- a/Amazon Wishlist Notifier/script.py +++ b/Amazon Wishlist Notifier/script.py @@ -1,8 +1,8 @@ -import requests from bs4 import BeautifulSoup import time import smtplib from email.mime.text import MIMEText +from security import safe_requests # Replace with your own email and password SENDER_EMAIL = 'your_sender_email@gmail.com' @@ -17,7 +17,7 @@ def get_wishlist_items(): headers = { "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.3" } - response = requests.get(WISHLIST_URL, headers=headers) + response = safe_requests.get(WISHLIST_URL, headers=headers) if response.status_code == 200: soup = BeautifulSoup(response.content, 'html.parser') diff --git a/Amazon-Price-Tracker/amazonprice.py b/Amazon-Price-Tracker/amazonprice.py index 8d5ea76214..1cc319ac6b 100644 --- a/Amazon-Price-Tracker/amazonprice.py +++ b/Amazon-Price-Tracker/amazonprice.py @@ -1,9 +1,9 @@ -import requests from bs4 import BeautifulSoup import time import smtplib import pywhatkit import datetime +from security import safe_requests # header = { # " @@ -20,7 +20,7 @@ headers = {"User-Agent": f'{user_agent}'} Url = input("Drop the Url of product you wish to buy...!\n") -page = requests.get(Url, headers=headers) +page = safe_requests.get(Url, headers=headers) soup = BeautifulSoup(page.content, "html.parser") # print(soup) diff --git a/Amazon_product_scraper/products.py b/Amazon_product_scraper/products.py index 456e40f9d1..b2494f7c6a 100644 --- a/Amazon_product_scraper/products.py +++ b/Amazon_product_scraper/products.py @@ -1,5 +1,5 @@ -import requests from bs4 import BeautifulSoup +from security import safe_requests class Product: @@ -15,7 +15,7 @@ def get_product(self): "User-Agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 \ (KHTML, like Gecko) Chrome/88.0.4324.96 Safari/537.36" } - r = requests.get(url, headers=headers) + r = safe_requests.get(url, headers=headers) soup = BeautifulSoup(r.content, "html.parser") product = soup.find("div", {"class": "s-product-image-container"}) product_link = product.find( @@ -55,7 +55,7 @@ def get_product_details(self): "User-Agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 \ (KHTML, like Gecko) Chrome/88.0.4324.96 Safari/537.36" } - r = requests.get(product_link, headers=headers) + r = safe_requests.get(product_link, headers=headers) soup = BeautifulSoup(r.content, "html.parser") product_name = soup.find( "span", {"id": "productTitle"}).text.strip() @@ -104,7 +104,7 @@ def get_product_image(self): "User-Agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 \ (KHTML, like Gecko) Chrome/88.0.4324.96 Safari/537.36" } - r = requests.get(product_link, headers=headers) + r = safe_requests.get(product_link, headers=headers) soup = BeautifulSoup(r.content, "html.parser") product_image = soup.find( "img", {"class": "a-dynamic-image a-stretch-horizontal"} @@ -144,7 +144,7 @@ def customer_review(self): "User-Agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 \ (KHTML, like Gecko) Chrome/88.0.4324.96 Safari/537.36" } - r = requests.get(product_link, headers=headers) + r = safe_requests.get(product_link, headers=headers) soup = BeautifulSoup(r.content, "html.parser") review_elements = soup.find_all("div", {"data-hook": "review"}) diff --git a/AskUbuntu-Scraper/questions.py b/AskUbuntu-Scraper/questions.py index 91c14ed787..0db78b2404 100644 --- a/AskUbuntu-Scraper/questions.py +++ b/AskUbuntu-Scraper/questions.py @@ -1,6 +1,6 @@ from bs4 import BeautifulSoup -import requests import json +from security import safe_requests class AskUbuntu: @@ -34,7 +34,7 @@ def getQuestions(self): """ url = "https://askubuntu.com/questions/tagged/" + self.topic try: - res = requests.get(url) + res = safe_requests.get(url) soup = BeautifulSoup(res.text, "html.parser") questions_data = {"questions": []} diff --git a/Automated_scraper.py/script.py b/Automated_scraper.py/script.py index abde80a7dc..e6784a5b04 100644 --- a/Automated_scraper.py/script.py +++ b/Automated_scraper.py/script.py @@ -2,12 +2,13 @@ import requests from bs4 import BeautifulSoup import time +from security import safe_requests def display_content(url, selector): try: # Send a GET request to the URL - response = requests.get(url) + response = safe_requests.get(url) # Check if the request was successful if response.status_code == 200: diff --git a/BITCOIN-price-tracker/tracker.py b/BITCOIN-price-tracker/tracker.py index bff613e488..d8f6e60bf8 100644 --- a/BITCOIN-price-tracker/tracker.py +++ b/BITCOIN-price-tracker/tracker.py @@ -1,6 +1,6 @@ -import requests from bs4 import BeautifulSoup import time +from security import safe_requests # create a function to get price of cryptocurrency @@ -8,7 +8,7 @@ def get_latest_crypto_price(coin): url = 'https://www.google.com/search?q=' + (coin) + 'price' # make a request to the website - HTML = requests.get(url) + HTML = safe_requests.get(url) # Parsse the HTML soup = BeautifulSoup(HTML.text, 'html.parser') # find the current price diff --git a/Book_Scraper/book.py b/Book_Scraper/book.py index aa09ff5801..90c904e284 100644 --- a/Book_Scraper/book.py +++ b/Book_Scraper/book.py @@ -1,6 +1,5 @@ # for scraping books from bs4 import BeautifulSoup as bs -import requests # to identify emoji unicode characters import emoji import pyfiglet @@ -8,6 +7,7 @@ import threading import time import sys +from security import safe_requests def is_emoji(text): @@ -17,7 +17,7 @@ def is_emoji(text): def link_to_get(link): """This function will get the url of the image & book download direct link using the given link for book download""" - response = requests.get(link) + response = safe_requests.get(link) th_html = bs(response.text, "html.parser") td_all = th_html.find_all("td", id="info") td_all = td_all[0] @@ -60,7 +60,7 @@ def book_get(name, mainres=100, results=5): # getting request and response url = f"http://libgen.is/search.php?req={name}&lg_topic=libgen&open=0&view=simple&res={mainres}&phrase=1&column=def" # print(url) - response = requests.get(url) + response = safe_requests.get(url) bs_html = bs(response.text, "html.parser") if "Search string must contain minimum 3 characters.." in bs_html.body: diff --git a/CNN Scraper/cnn.py b/CNN Scraper/cnn.py index bca3b7185f..0ab1216cf7 100644 --- a/CNN Scraper/cnn.py +++ b/CNN Scraper/cnn.py @@ -1,5 +1,5 @@ from bs4 import BeautifulSoup -import requests +from security import safe_requests class NewsCNN: @@ -36,7 +36,7 @@ def news_by_location(self, country: str): obj_keys = ["news", "link"] location = country.lower() URL = f"https://edition.cnn.com/world/{location}" - page = requests.get(URL) + page = safe_requests.get(URL) parse = BeautifulSoup(page.content, "html.parser") heads = parse.find_all("span", attrs={"data-editable": "headline"}) links1 = parse.find_all( @@ -95,7 +95,7 @@ def news_by_category(self, type: str): sol = [] type = type.lower() url = f"https://edition.cnn.com/{type}" - page = requests.get(url, headers=self.headers) + page = safe_requests.get(url, headers=self.headers) parse = BeautifulSoup(page.content, "html.parser") articles = parse.find_all( "a", {"class": "container__link container_lead-plus-headlines__link"} diff --git a/Covid-19_Real-time_Notification/Covid.py b/Covid-19_Real-time_Notification/Covid.py index c2dbff6b51..29b53abe5e 100644 --- a/Covid-19_Real-time_Notification/Covid.py +++ b/Covid-19_Real-time_Notification/Covid.py @@ -1,8 +1,8 @@ from plyer import notification -import requests from bs4 import BeautifulSoup import time from englisttohindi.englisttohindi import EngtoHindi +from security import safe_requests def notify_user(title, message): @@ -14,7 +14,7 @@ def notify_user(title, message): def getInfo(url): - r = requests.get(url) + r = safe_requests.get(url) return r.text diff --git a/Crypocurrency-Converter-GUI/main.py b/Crypocurrency-Converter-GUI/main.py index bc0d9f16ed..21347a312c 100644 --- a/Crypocurrency-Converter-GUI/main.py +++ b/Crypocurrency-Converter-GUI/main.py @@ -4,6 +4,7 @@ from guiVersion import Ui_guiVersion import json import requests +from security import safe_requests class guiVersion(QMainWindow, Ui_guiVersion): @@ -56,7 +57,7 @@ def currencies2(self, item2): def api(self, cur1, cur2): api_link = "https://min-api.cryptocompare.com/data/pricemulti?fsyms={}&tsyms={}".format( cur1, cur2) - resp = requests.get(api_link) + resp = safe_requests.get(api_link) # print(r.status_code) data = json.loads(resp.content) # print(data) diff --git a/Currency-Exchange-Rates/exchange_rates.py b/Currency-Exchange-Rates/exchange_rates.py index cfb8f1bb46..fba389c4c7 100644 --- a/Currency-Exchange-Rates/exchange_rates.py +++ b/Currency-Exchange-Rates/exchange_rates.py @@ -1,5 +1,6 @@ from bs4 import BeautifulSoup import requests as req +from security import safe_requests currencies = [] @@ -26,7 +27,7 @@ currencies_table_url = 'https://www.x-rates.com/table/?from={}&amount={}'.format( currency['short'], amount) -currencies_table_page = req.get(currencies_table_url).text +currencies_table_page = safe_requests.get(currencies_table_url).text soup = BeautifulSoup(currencies_table_page, 'html.parser') diff --git a/Current_City_Weather/Current_City_Weather.py b/Current_City_Weather/Current_City_Weather.py index 29d0743745..b1312b3e8e 100644 --- a/Current_City_Weather/Current_City_Weather.py +++ b/Current_City_Weather/Current_City_Weather.py @@ -1,4 +1,4 @@ -import requests +from security import safe_requests def get_temperature(json_data): @@ -30,7 +30,7 @@ def main(): city = input("City Name : ") units_format = "&units=metric" final_url = api_address + city + units_format - json_data = requests.get(final_url).json() + json_data = safe_requests.get(final_url).json() weather_details = get_weather_data(json_data, city) # print formatted data print(weather_details) diff --git a/DataScraping/Data_Scraping.py b/DataScraping/Data_Scraping.py index 4c5ced1ac7..94fffd6718 100644 --- a/DataScraping/Data_Scraping.py +++ b/DataScraping/Data_Scraping.py @@ -1,5 +1,5 @@ import tweepy -import requests +from security import safe_requests # Function to get tweets from Twitter API @@ -23,7 +23,7 @@ def get_tweets(api_key, api_secret_key, access_token, access_token_secret, usern def get_weather(api_key, city): url = f"http://api.openweathermap.org/data/2.5/weather?q={city}&appid={api_key}" - response = requests.get(url) + response = safe_requests.get(url) data = response.json() if response.status_code == 200: @@ -37,7 +37,7 @@ def get_weather(api_key, city): def get_stock_data(api_key, symbol): url = f"https://www.alphavantage.co/query?function=TIME_SERIES_DAILY&symbol={symbol}&apikey={api_key}" - response = requests.get(url) + response = safe_requests.get(url) data = response.json() if "Time Series (Daily)" in data: diff --git a/Desktop News Notifier/script.py b/Desktop News Notifier/script.py index a485b4d081..4377d35312 100644 --- a/Desktop News Notifier/script.py +++ b/Desktop News Notifier/script.py @@ -1,12 +1,12 @@ from plyer import notification -import requests import json +from security import safe_requests country_code = input("Enter the country code for the news: ") api_key = input("Enter the api key: ") -news = requests.get( +news = safe_requests.get( f'https://newsapi.org/v2/top-headlines?country={country_code}&apiKey={api_key}') data = json.loads(news.content) diff --git a/Discord News Bot With AI/main.py b/Discord News Bot With AI/main.py index e14785b572..96648bc1c6 100644 --- a/Discord News Bot With AI/main.py +++ b/Discord News Bot With AI/main.py @@ -3,8 +3,8 @@ import discord import schedule import asyncio -import requests import openai +from security import safe_requests openai.api_key = 'YOUR_OPENAI_API_KEY' # Get a key at https://platform.openai.com/docs/api-reference @@ -34,7 +34,7 @@ def get_latest_article(): global latest_article - response = requests.get(url, params=params) + response = safe_requests.get(url, params=params) articles = response.json()['articles'] latest_article = articles[0] return latest_article diff --git a/Doh-Dig/dog-dig.py b/Doh-Dig/dog-dig.py index 3619a50230..b9c072c217 100644 --- a/Doh-Dig/dog-dig.py +++ b/Doh-Dig/dog-dig.py @@ -1,5 +1,5 @@ import os -import requests +from security import safe_requests DOH_SERVER = os.environ.get( "DOH_SERVER", "https://cloudflare-dns.com/dns-query") @@ -11,7 +11,7 @@ def doh_dns_lookup(domain, record_type): "Accept": "application/dns-json" } - response = requests.get(url, headers=headers) + response = safe_requests.get(url, headers=headers) if response.status_code == 200: return response.json() diff --git a/Domain_Name_Availability/Domain_name.py b/Domain_Name_Availability/Domain_name.py index 6e9c6bb18c..8e08644216 100644 --- a/Domain_Name_Availability/Domain_name.py +++ b/Domain_Name_Availability/Domain_name.py @@ -1,66 +1,66 @@ -import requests -import decouple - -req_headers = {} - - -def godaddy_credentials(): - """ - This functions reads your credentials from .env file - """ - global req_headers - - print("\n\t::: Reading Godaddy Credentials :::") - - api_key = decouple.config("API_KEY") - api_secret = decouple.config("API_SECRET") - # api_key = input("\nEnter your API Key: ") - # api_secret = input("Enter your SECRET Key: ") - - req_headers = { - "Authorization": f"sso-key {api_key}:{api_secret}", - "accept": "application/json", - } - - -def get_url(domain_name): - """ - creating the required URL from domain name - :param domain_name: (string) domain name - :return: URL - """ - return f"https://api.ote-godaddy.com/v1/domains/suggest?query={domain_name}&country=IN&waitMs=1000" - - -def available_domain_names(): - """ - This function takes godaddy credentials from the user and generates all the - available domain names in a text file. - """ - godaddy_credentials() - domain_name = input("\nEnter required DOMAIN Name: ") - url = get_url(domain_name) - - print("\nSearching for the available domains") - res = requests.get(url, headers=req_headers) - - if res.status_code == 200: - # Output is in JSON file so reading it. - response = res.json() - - # Saving our available domains in a text file. - print( - f"\nSaving all the available domain names in {domain_name}.txt file") - f = open(f"{domain_name}.txt", "a") - for i in response: - f.writelines(i["domain"] + "\n") - f.close() - print( - f"\nFile {domain_name}.txt saved successfully in your current directory") - - else: - print("Error Status Code") - - -if __name__ == "__main__": - available_domain_names() +import decouple +from security import safe_requests + +req_headers = {} + + +def godaddy_credentials(): + """ + This functions reads your credentials from .env file + """ + global req_headers + + print("\n\t::: Reading Godaddy Credentials :::") + + api_key = decouple.config("API_KEY") + api_secret = decouple.config("API_SECRET") + # api_key = input("\nEnter your API Key: ") + # api_secret = input("Enter your SECRET Key: ") + + req_headers = { + "Authorization": f"sso-key {api_key}:{api_secret}", + "accept": "application/json", + } + + +def get_url(domain_name): + """ + creating the required URL from domain name + :param domain_name: (string) domain name + :return: URL + """ + return f"https://api.ote-godaddy.com/v1/domains/suggest?query={domain_name}&country=IN&waitMs=1000" + + +def available_domain_names(): + """ + This function takes godaddy credentials from the user and generates all the + available domain names in a text file. + """ + godaddy_credentials() + domain_name = input("\nEnter required DOMAIN Name: ") + url = get_url(domain_name) + + print("\nSearching for the available domains") + res = safe_requests.get(url, headers=req_headers) + + if res.status_code == 200: + # Output is in JSON file so reading it. + response = res.json() + + # Saving our available domains in a text file. + print( + f"\nSaving all the available domain names in {domain_name}.txt file") + f = open(f"{domain_name}.txt", "a") + for i in response: + f.writelines(i["domain"] + "\n") + f.close() + print( + f"\nFile {domain_name}.txt saved successfully in your current directory") + + else: + print("Error Status Code") + + +if __name__ == "__main__": + available_domain_names() diff --git a/EazyDiner-Scraper/restaurants.py b/EazyDiner-Scraper/restaurants.py index 2d609d5cf1..15408218d2 100644 --- a/EazyDiner-Scraper/restaurants.py +++ b/EazyDiner-Scraper/restaurants.py @@ -1,6 +1,6 @@ import json -import requests from bs4 import BeautifulSoup +from security import safe_requests class EazyDiner: @@ -39,7 +39,7 @@ def getRestaurants(self): + self.location.replace(" ", "-").replace(",", "").lower() ) try: - res = requests.get(url) + res = safe_requests.get(url) soup = BeautifulSoup(res.text, "html.parser") restaurant_data = {"restaurants": []} diff --git a/Facebook-DP-Downloader/fb_dp_downloader.py b/Facebook-DP-Downloader/fb_dp_downloader.py index 046f46d3f9..0bfb3fb653 100644 --- a/Facebook-DP-Downloader/fb_dp_downloader.py +++ b/Facebook-DP-Downloader/fb_dp_downloader.py @@ -6,7 +6,7 @@ """ import os -import requests +from security import safe_requests url = "https://graph.facebook.com/{}/picture?type=large" @@ -31,7 +31,7 @@ # the number should be a valid Facebook user id try: - result = requests.get(url.format(fbid)) + result = safe_requests.get(url.format(fbid)) with open("fb_dps/{}_img.jpg".format(fbid), "wb") as file: file.write(result.content) diff --git a/Facebook_Video_Downloader/script.py b/Facebook_Video_Downloader/script.py index 1b889947c2..a6b919acbe 100644 --- a/Facebook_Video_Downloader/script.py +++ b/Facebook_Video_Downloader/script.py @@ -2,12 +2,13 @@ import time from tkinter.ttk import * import tkinter as tk -from requests import get, HTTPError, ConnectionError +from requests import HTTPError, ConnectionError from re import findall from urllib.parse import unquote from threading import Thread import queue from queue import Empty +from security import safe_requests def Invalid_Url(): @@ -20,7 +21,7 @@ def get_downloadlink(url): url = url.replace("www", "mbasic") try: - r = get(url, timeout=5, allow_redirects=True) + r = safe_requests.get(url, timeout=5, allow_redirects=True) if r.status_code != 200: raise HTTPError a = findall("/video_redirect/", r.text) @@ -81,7 +82,7 @@ def run(self): # save the picture to a file block_size = 1024 # 1kB - r = get(self.url, stream=True) + r = safe_requests.get(self.url, stream=True) total_size = int(r.headers.get("content-length")) with open('video.mp4', 'wb') as file: diff --git a/Fetch All links in given webpage/Fetch_all_links_from_webpage.py b/Fetch All links in given webpage/Fetch_all_links_from_webpage.py index 432dfa26b4..33e566ba60 100644 --- a/Fetch All links in given webpage/Fetch_all_links_from_webpage.py +++ b/Fetch All links in given webpage/Fetch_all_links_from_webpage.py @@ -1,20 +1,20 @@ -# import requests module first -import requests as rq -from bs4 import BeautifulSoup - -url = input("Enter Link: ") # input link by user - -if ("https" or "http") in url: - data = rq.get(url) -else: - data = rq.get("https://" + url) - -soup = BeautifulSoup(data.text, "html.parser") - -links = [] - -for link in soup.find_all("a"): - links.append(link.get("href")) - -with open("myLinks.txt", 'w') as saved: - print(links[:20], file=saved) +# import requests module first +from bs4 import BeautifulSoup +from security import safe_requests + +url = input("Enter Link: ") # input link by user + +if ("https" or "http") in url: + data = safe_requests.get(url) +else: + data = safe_requests.get("https://" + url) + +soup = BeautifulSoup(data.text, "html.parser") + +links = [] + +for link in soup.find_all("a"): + links.append(link.get("href")) + +with open("myLinks.txt", 'w') as saved: + print(links[:20], file=saved) diff --git a/Fetch_Contributions/fetch_contributions.py b/Fetch_Contributions/fetch_contributions.py index 43f2e70635..d7ccc9a910 100644 --- a/Fetch_Contributions/fetch_contributions.py +++ b/Fetch_Contributions/fetch_contributions.py @@ -1,180 +1,180 @@ -import argparse - -import pandas as pd -import requests -from bs4 import BeautifulSoup -from lxml import html -from tabulate import tabulate - - -class Fetch_PullRequests: - """ - Fetches the pull requests of a user in a organization. - """ - - def __init__(self, username, organization, filename): - """ - :param username: github user - :param organization: Organisation name - :param filename: filename, it's optional - """ - self.ORG_URL = f"https://github.com/orgs/{organization}/repositories" - self.URL = f"https://github.com/{organization}" - self.organization = organization - self.username = username - self.filename = filename - - def _list_of_repositories(self): - """ - Function lists the repositories of the organisation. - - Returns - ------- - list - lists the repositories - - """ - page = requests.get(self.ORG_URL) - tree = html.fromstring(page.content) - number_of_pages = tree.xpath( - '//*[@id="org-repositories"]/div/div/div[2]/div/em/@data-total-pages') - Repositories = [] - if len(number_of_pages) == 0: - Repositories.extend(tree.xpath( - '//*[contains(concat( " ", @class, " " ), concat( " ", "wb-break-all", " " ))]//*[contains(concat( " ", @class, " " ), concat( " ", "d-inline-block", " " ))]/text()')) - else: - for number in range(1, int(number_of_pages[0]) + 1): - page_ = requests.get(self.ORG_URL + f"?page={number}") - tree = html.fromstring(page_.content) - Repositories.extend(tree.xpath( - '//*[contains(concat( " ", @class, " " ), concat( " ", "wb-break-all", " " ))]//*[contains(concat( " ", @class, " " ), concat( " ", "d-inline-block", " " ))]/text()')) - - return list(pd.Series(list(set(Repositories))).str.strip().values) - - def _extract_pullrequests(self, repo): - """ - Function fetches the pull request of a repo. - - Parameters - ---------- - repo: str - repository name - - Returns - ------- - pandas dataframe - dataframe consists of columns - "Title to PR", "Link of PR", "Status(Merged/Closed/Open)" - - """ - # initializing the lists to store the title, link and status of the pull request - Title = [] - Link = [] - Status = [] - URL = self.URL + f"/{repo}/pulls?q=is%3Apr+author%3A{self.username}" - page = requests.get(URL) - tree = html.fromstring(page.content) - # to determine the number of pages - number_of_pages = tree.xpath( - '//*[@id="repo-content-pjax-container"]/div/div[6]/div/em/@data-total-pages') - - if len(number_of_pages) == 0: - # Title.extend(tree.xpath('//*[contains(concat( " ", @class, " " ), concat( " ", "markdown-title", " " ))]/text()')) - soup = BeautifulSoup(page.text, 'html.parser') - # "Title may contain text in tags. So,to handle it we use beautiful soup. - for tag in soup.find_all('a', attrs={'class': 'markdown-title'}): - Title.append(tag.text.strip()) - Link.extend( - tree.xpath('//*[contains(concat( " ", @class, " " ), concat( " ", "markdown-title", " " ))]/@href')) - Status.extend(tree.xpath( - '//*[contains(concat( " ", @class, " " ), concat( " ", "pl-3", " " ))]/span/@aria-label')) - - else: - for number in range(1, int(number_of_pages[0]) + 1): - URL = self.URL + \ - f"/{repo}/pulls?page={number}&q=is%3Apr+author%3A{self.username}" - page = requests.get(URL) - tree = html.fromstring(page.content) - - # Title.extend(tree.xpath('//*[contains(concat( " ", @class, " " ), concat( " ", "markdown-title", " " ))]/text()')) - soup = BeautifulSoup(page.text, 'html.parser') - # Names = tree.xpath( - # '//*[contains(concat( " ", @class, " " ), concat( " ", "opened-by", " " ))]//*[contains(concat( " ", @class, " " ), concat( " ", "Link--muted", " " ))]/text()') - - for tag in soup.find_all('a', attrs={'class': 'markdown-title'}): - Title.append(tag.text.strip()) - Link.extend(tree.xpath( - '//*[contains(concat( " ", @class, " " ), concat( " ", "markdown-title", " " ))]/@href')) - Status.extend(tree.xpath( - '//*[contains(concat( " ", @class, " " ), concat( " ", "pl-3", " " ))]/span/@aria-label')) - - Data = { - "Title to PR": Title, - "Link of PR": Link, - "Status(Merged/Closed/Open)": Status - } - - # creating a dataframe with the above dictionary - dataframe = pd.DataFrame.from_dict(Data) - # dataframe.head() - - # make necessary changes to the columns of dataframe before returning it - dataframe['Status(Merged/Closed/Open)'] = dataframe['Status(Merged/Closed/Open)'].astype(str).str.replace( - " pull request", - "", regex=False) - if dataframe['Link of PR'].dtype != 'O': - dataframe['Link of PR'] = dataframe['Link of PR'].astype(str) - dataframe['Link of PR'] = 'https://github.com' + \ - dataframe['Link of PR'] - - return dataframe - - def get_pullrequests(self): - """ - Function pass the repo parameter to the "_extract_pullrequests" to fetch the pull requests of the particular repo. - - Returns - ------- - str - return str saying that the file is stored if markdown is not empty. - - """ - dataframe = pd.DataFrame() - for repo in self._list_of_repositories(): - dataframe = dataframe.append( - self._extract_pullrequests(repo), ignore_index=True) - - markdown = dataframe.to_markdown() - - if len(markdown) > 0: - # creating a markdown file - # markdown_file = open(f"{self.filename}.md", "w") - with open(f"{self.filename}.md", "w") as markdown_file: - markdown_file.write(markdown) - - return "Markdown File is successfully stored" - - return "No pull requests found !!" - - -if __name__ == "__main__": - parser = argparse.ArgumentParser() - parser.add_argument("-u", "--username", action="store_true") - parser.add_argument( - "user", type=str, help="The name of the user to get the pull requests") - parser.add_argument("-o", "--organization", action="store_true") - parser.add_argument("organization_name", type=str, - help="the organisation where user made the pull requests") - parser.add_argument("-f", "--file", nargs="?") - parser.add_argument("filename", type=str, nargs="?", - help="filename to store the markdown table") - args = parser.parse_args() - if args.filename: - file_name = args.filename - else: - file_name = "Markdown_file" - if args.username and args.organization: - response = Fetch_PullRequests( - args.user, args.organization_name, file_name) - print(response.get_pullrequests()) - else: - print("Please pass atleast two arguments: '--username', '--organisation'") +import argparse + +import pandas as pd +from bs4 import BeautifulSoup +from lxml import html +from tabulate import tabulate +from security import safe_requests + + +class Fetch_PullRequests: + """ + Fetches the pull requests of a user in a organization. + """ + + def __init__(self, username, organization, filename): + """ + :param username: github user + :param organization: Organisation name + :param filename: filename, it's optional + """ + self.ORG_URL = f"https://github.com/orgs/{organization}/repositories" + self.URL = f"https://github.com/{organization}" + self.organization = organization + self.username = username + self.filename = filename + + def _list_of_repositories(self): + """ + Function lists the repositories of the organisation. + + Returns + ------- + list + lists the repositories + + """ + page = safe_requests.get(self.ORG_URL) + tree = html.fromstring(page.content) + number_of_pages = tree.xpath( + '//*[@id="org-repositories"]/div/div/div[2]/div/em/@data-total-pages') + Repositories = [] + if len(number_of_pages) == 0: + Repositories.extend(tree.xpath( + '//*[contains(concat( " ", @class, " " ), concat( " ", "wb-break-all", " " ))]//*[contains(concat( " ", @class, " " ), concat( " ", "d-inline-block", " " ))]/text()')) + else: + for number in range(1, int(number_of_pages[0]) + 1): + page_ = safe_requests.get(self.ORG_URL + f"?page={number}") + tree = html.fromstring(page_.content) + Repositories.extend(tree.xpath( + '//*[contains(concat( " ", @class, " " ), concat( " ", "wb-break-all", " " ))]//*[contains(concat( " ", @class, " " ), concat( " ", "d-inline-block", " " ))]/text()')) + + return list(pd.Series(list(set(Repositories))).str.strip().values) + + def _extract_pullrequests(self, repo): + """ + Function fetches the pull request of a repo. + + Parameters + ---------- + repo: str + repository name + + Returns + ------- + pandas dataframe + dataframe consists of columns - "Title to PR", "Link of PR", "Status(Merged/Closed/Open)" + + """ + # initializing the lists to store the title, link and status of the pull request + Title = [] + Link = [] + Status = [] + URL = self.URL + f"/{repo}/pulls?q=is%3Apr+author%3A{self.username}" + page = safe_requests.get(URL) + tree = html.fromstring(page.content) + # to determine the number of pages + number_of_pages = tree.xpath( + '//*[@id="repo-content-pjax-container"]/div/div[6]/div/em/@data-total-pages') + + if len(number_of_pages) == 0: + # Title.extend(tree.xpath('//*[contains(concat( " ", @class, " " ), concat( " ", "markdown-title", " " ))]/text()')) + soup = BeautifulSoup(page.text, 'html.parser') + # "Title may contain text in tags. So,to handle it we use beautiful soup. + for tag in soup.find_all('a', attrs={'class': 'markdown-title'}): + Title.append(tag.text.strip()) + Link.extend( + tree.xpath('//*[contains(concat( " ", @class, " " ), concat( " ", "markdown-title", " " ))]/@href')) + Status.extend(tree.xpath( + '//*[contains(concat( " ", @class, " " ), concat( " ", "pl-3", " " ))]/span/@aria-label')) + + else: + for number in range(1, int(number_of_pages[0]) + 1): + URL = self.URL + \ + f"/{repo}/pulls?page={number}&q=is%3Apr+author%3A{self.username}" + page = safe_requests.get(URL) + tree = html.fromstring(page.content) + + # Title.extend(tree.xpath('//*[contains(concat( " ", @class, " " ), concat( " ", "markdown-title", " " ))]/text()')) + soup = BeautifulSoup(page.text, 'html.parser') + # Names = tree.xpath( + # '//*[contains(concat( " ", @class, " " ), concat( " ", "opened-by", " " ))]//*[contains(concat( " ", @class, " " ), concat( " ", "Link--muted", " " ))]/text()') + + for tag in soup.find_all('a', attrs={'class': 'markdown-title'}): + Title.append(tag.text.strip()) + Link.extend(tree.xpath( + '//*[contains(concat( " ", @class, " " ), concat( " ", "markdown-title", " " ))]/@href')) + Status.extend(tree.xpath( + '//*[contains(concat( " ", @class, " " ), concat( " ", "pl-3", " " ))]/span/@aria-label')) + + Data = { + "Title to PR": Title, + "Link of PR": Link, + "Status(Merged/Closed/Open)": Status + } + + # creating a dataframe with the above dictionary + dataframe = pd.DataFrame.from_dict(Data) + # dataframe.head() + + # make necessary changes to the columns of dataframe before returning it + dataframe['Status(Merged/Closed/Open)'] = dataframe['Status(Merged/Closed/Open)'].astype(str).str.replace( + " pull request", + "", regex=False) + if dataframe['Link of PR'].dtype != 'O': + dataframe['Link of PR'] = dataframe['Link of PR'].astype(str) + dataframe['Link of PR'] = 'https://github.com' + \ + dataframe['Link of PR'] + + return dataframe + + def get_pullrequests(self): + """ + Function pass the repo parameter to the "_extract_pullrequests" to fetch the pull requests of the particular repo. + + Returns + ------- + str + return str saying that the file is stored if markdown is not empty. + + """ + dataframe = pd.DataFrame() + for repo in self._list_of_repositories(): + dataframe = dataframe.append( + self._extract_pullrequests(repo), ignore_index=True) + + markdown = dataframe.to_markdown() + + if len(markdown) > 0: + # creating a markdown file + # markdown_file = open(f"{self.filename}.md", "w") + with open(f"{self.filename}.md", "w") as markdown_file: + markdown_file.write(markdown) + + return "Markdown File is successfully stored" + + return "No pull requests found !!" + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("-u", "--username", action="store_true") + parser.add_argument( + "user", type=str, help="The name of the user to get the pull requests") + parser.add_argument("-o", "--organization", action="store_true") + parser.add_argument("organization_name", type=str, + help="the organisation where user made the pull requests") + parser.add_argument("-f", "--file", nargs="?") + parser.add_argument("filename", type=str, nargs="?", + help="filename to store the markdown table") + args = parser.parse_args() + if args.filename: + file_name = args.filename + else: + file_name = "Markdown_file" + if args.username and args.organization: + response = Fetch_PullRequests( + args.user, args.organization_name, file_name) + print(response.get_pullrequests()) + else: + print("Please pass atleast two arguments: '--username', '--organisation'") diff --git a/File Downloader Script/file_downloader.py b/File Downloader Script/file_downloader.py index 64bc3c3b24..80c5ec8ffd 100644 --- a/File Downloader Script/file_downloader.py +++ b/File Downloader Script/file_downloader.py @@ -1,8 +1,8 @@ -import requests +from security import safe_requests def download_file(url, destination): - response = requests.get(url, stream=True) + response = safe_requests.get(url, stream=True) if response.status_code == 200: with open(destination, 'wb') as file: for chunk in response.iter_content(chunk_size=1024): diff --git a/File Downloader/file_downloader.py b/File Downloader/file_downloader.py index 07c12b2bf8..f74072361d 100644 --- a/File Downloader/file_downloader.py +++ b/File Downloader/file_downloader.py @@ -5,14 +5,14 @@ You just need to have the url and you are good to go! """ import os -import requests from tqdm import tqdm import math import time +from security import safe_requests url = input("Enter the url of the file you want to download: ") -r = requests.get(url) +r = safe_requests.get(url) # receives data from the url file_size = int(r.headers['Content-Length']) @@ -22,7 +22,7 @@ number of bytes downloaded at a time """ -r = requests.get(url, stream=True) +r = safe_requests.get(url, stream=True) """streams=True ensures that will not get data at once, but will get data one by one diff --git a/FileDownload Automationi/script.py b/FileDownload Automationi/script.py index 5d34e481d3..75bc728026 100644 --- a/FileDownload Automationi/script.py +++ b/FileDownload Automationi/script.py @@ -1,4 +1,4 @@ -import requests +from security import safe_requests def download_file(url, save_path, file_name): @@ -11,7 +11,7 @@ def download_file(url, save_path, file_name): - save_path (str): The directory path where the file will be saved. - file_name (str): The name to be used for the downloaded file. """ - response = requests.get(url) + response = safe_requests.get(url) file_path = save_path + "\\" + file_name with open(file_path, 'wb') as file: file.write(response.content) diff --git a/Flipkart-price-alert/track.py b/Flipkart-price-alert/track.py index d6207db4c1..6adb188798 100644 --- a/Flipkart-price-alert/track.py +++ b/Flipkart-price-alert/track.py @@ -1,7 +1,8 @@ -import requests from bs4 import BeautifulSoup as bs import smtplib import time +from security import safe_requests + headers = {'User-Agent': 'Mozilla/5.0 Chrome/86.0.4240.75'} @@ -24,7 +25,7 @@ def sendMail(title): def priceCheck(): '''Price checking function''' - page = requests.get(PRODUCT_URL, headers=headers) + page = safe_requests.get(PRODUCT_URL, headers=headers) soup = bs(page.content, 'html.parser') # title from 'B_NuCI' class title = soup.find("span", {"class": "B_NuCI"}).get_text()[0:8] + '..' diff --git a/FlipkartScraper/genricHtmlib.py b/FlipkartScraper/genricHtmlib.py index a8cd153e3c..afd26488b3 100644 --- a/FlipkartScraper/genricHtmlib.py +++ b/FlipkartScraper/genricHtmlib.py @@ -8,6 +8,8 @@ from selenium.webdriver.chrome.options import Options import warnings import requests +from security import safe_requests + warnings.filterwarnings("ignore") @@ -46,7 +48,7 @@ def fetch_request_normal(self, url, params=None): if response.status_code == 301: # retry with redirect - response = requests.get(response.headers['Location']) + response = safe_requests.get(response.headers['Location']) response.raise_for_status() if response.status_code == 200: return response.text diff --git a/Flipkart_webscraping/Scrap.py b/Flipkart_webscraping/Scrap.py index b313da6da2..e136b8b9bd 100644 --- a/Flipkart_webscraping/Scrap.py +++ b/Flipkart_webscraping/Scrap.py @@ -1,6 +1,6 @@ import pandas as pd -import requests from bs4 import BeautifulSoup +from security import safe_requests Product_name = [] Prices = [] @@ -12,7 +12,7 @@ url = "https://www.flipkart.com/search?q=MOBILE+PHONE+UNDER+50000&otracker=search&otracker1=search&marketplace=FLIPKART&as-show=on&as=off&page=" + \ str(2) - r = requests.get(url) + r = safe_requests.get(url) soup = BeautifulSoup(r.text, "lxml") box = soup.find("div", class_="_1YokD2 _3Mn1Gg") diff --git a/Github-Size-Contributor-List/script.py b/Github-Size-Contributor-List/script.py index de06cc0fad..be6053e531 100644 --- a/Github-Size-Contributor-List/script.py +++ b/Github-Size-Contributor-List/script.py @@ -1,11 +1,11 @@ -import requests import argparse +from security import safe_requests def main(args): GITHUB_URL = f" https://api.github.com/repos/{args.o}/{args.n}" - res = requests.get(GITHUB_URL).json() - response_collaborators = requests.get(f"{GITHUB_URL}/contributors").json() + res = safe_requests.get(GITHUB_URL).json() + response_collaborators = safe_requests.get(f"{GITHUB_URL}/contributors").json() response_text = f"Size of repository is {res.get('size')}" diff --git a/Github-User-Scraper/scraper.py b/Github-User-Scraper/scraper.py index bbf9b213de..ae55449496 100644 --- a/Github-User-Scraper/scraper.py +++ b/Github-User-Scraper/scraper.py @@ -1,4 +1,3 @@ -import requests from bs4 import BeautifulSoup import tkinter as tk from tkinter import ttk @@ -8,6 +7,7 @@ from sqlite3 import Error import time import datetime +from security import safe_requests # Dictionary for date values dates = {'Today': 'daily', 'This week': 'weekly', 'This month': 'monthly'} @@ -101,7 +101,7 @@ def scrape_users(): url_lang = language.get() date_range = date_helper() url = get_URL() - page = requests.get(url) + page = safe_requests.get(url) # Start scraping resultant html data soup = BeautifulSoup(page.content, 'html.parser') diff --git a/Github_User_Info/main.py b/Github_User_Info/main.py index e842e1f95c..f66c5bf3f7 100644 --- a/Github_User_Info/main.py +++ b/Github_User_Info/main.py @@ -1,10 +1,10 @@ -import requests import argparse +from security import safe_requests def main(args): URL = f"https://api.github.com/users/{args.u}" - res = requests.get(URL).json() + res = safe_requests.get(URL).json() output = "" output += "Username: "+res['login'] if (res['name']): diff --git a/Google-Image-Scrapper/scrapper.py b/Google-Image-Scrapper/scrapper.py index ee96a40d5e..899ccdeed1 100644 --- a/Google-Image-Scrapper/scrapper.py +++ b/Google-Image-Scrapper/scrapper.py @@ -1,9 +1,9 @@ import os import time import urllib -import requests from urllib.parse import quote import array as arr +from security import safe_requests class simple_image_download: @@ -92,7 +92,7 @@ def download(self, keywords, limit): keyword_to_search[i]) + "_" + str(j + 1) + ".jpg" try: - r = requests.get(object_raw, allow_redirects=True) + r = safe_requests.get(object_raw, allow_redirects=True) open(os.path.join(path, filename), 'wb').write(r.content) except Exception as e: print(e) diff --git a/Google-News-Scraapper/app.py b/Google-News-Scraapper/app.py index cf6b8d6c5b..148bbed11a 100644 --- a/Google-News-Scraapper/app.py +++ b/Google-News-Scraapper/app.py @@ -1,12 +1,12 @@ -import requests from xml.dom.minidom import parseString import pandas as pd +from security import safe_requests def get_google_news_result(term, count): results = [] obj = parseString( - requests.get('http://news.google.com/news?q=%s&output=rss' % + safe_requests.get('http://news.google.com/news?q=%s&output=rss' % term).text) items = obj.getElementsByTagName('item') # Storing the Titles and Links diff --git a/GoogleNews-Scraper/googleNews.py b/GoogleNews-Scraper/googleNews.py index 74c9f700da..9539a89c9e 100644 --- a/GoogleNews-Scraper/googleNews.py +++ b/GoogleNews-Scraper/googleNews.py @@ -1,5 +1,5 @@ -import requests from bs4 import BeautifulSoup +from security import safe_requests class GoogleNews: @@ -35,7 +35,7 @@ def getArticles(self): """ url = "https://www.google.com/search?q=" + self.topic + "&tbm=nws" try: - res = requests.get(url) + res = safe_requests.get(url) soup = BeautifulSoup(res.text, "html.parser") articles_data = {"articles": []} diff --git a/IMDB-Scraper/scraper.py b/IMDB-Scraper/scraper.py index 552feb4e03..306f78db68 100644 --- a/IMDB-Scraper/scraper.py +++ b/IMDB-Scraper/scraper.py @@ -1,6 +1,6 @@ -import requests from bs4 import BeautifulSoup as bs import argparse +from security import safe_requests parser = argparse.ArgumentParser(description='IMDB Scraper') @@ -62,7 +62,7 @@ def get_info(soup): def find_movie(query): url = base+query - resp = requests.get(url) + resp = safe_requests.get(url) # for parsing we have used the lxml parser for optimization purposes, if lxml does not work for you replace 'lxml' with 'html.parser' soup1 = bs(resp.text, 'lxml') # Since for every query imdb gives about 150-200 responses , we choose the top 5 and return the details for them @@ -74,7 +74,7 @@ def find_movie(query): 'td', attrs={"class": "result_text"}).a.attrs["href"][6:] url = base_id+title_id - respo = requests.get(base_id+title_id) + respo = safe_requests.get(base_id+title_id) soup = bs(respo.text, 'lxml') get_info(soup) else: diff --git a/IPL Statistics GUI/ipl.py b/IPL Statistics GUI/ipl.py index cdfbae3501..bc5d7cdff2 100644 --- a/IPL Statistics GUI/ipl.py +++ b/IPL Statistics GUI/ipl.py @@ -1,7 +1,7 @@ -import requests from bs4 import BeautifulSoup import tkinter as tk from tkinter import ttk +from security import safe_requests # Dictionary containing category and corresponding slug value categories = {'Most Runs': 'most-runs', @@ -28,7 +28,7 @@ def generate_url(): def scrape_results(): url = generate_url() - page = requests.get(url) + page = safe_requests.get(url) # Start scraping resultant html data soup = BeautifulSoup(page.content, 'html.parser') diff --git a/Icc_ranking_scraper/rankings.py b/Icc_ranking_scraper/rankings.py index 3953f2bfd4..0370b7b781 100644 --- a/Icc_ranking_scraper/rankings.py +++ b/Icc_ranking_scraper/rankings.py @@ -1,5 +1,5 @@ from bs4 import BeautifulSoup -import requests +from security import safe_requests class ICC: @@ -13,7 +13,7 @@ def team_rankings(self, format): obj_keys = ["rank", "team"] resposne_list = [] url = self.url + "team-rankings/" + format - response = requests.get(url) + response = safe_requests.get(url) soup = BeautifulSoup(response.content, "html.parser") teams = soup.find_all("span", class_="u-hide-phablet") for rank, team in enumerate(teams, 1): @@ -28,7 +28,7 @@ def player_ranking(self, type, format): try: url = self.url + f"/player-rankings/{format}/{type}" - response = requests.get(url) + response = safe_requests.get(url) soup = BeautifulSoup(response.content, "html.parser") top_player = soup.find( "div", class_="rankings-block__banner--name-large" diff --git a/Image Downloader GUI/main.py b/Image Downloader GUI/main.py index e14ad683c8..69d69d45d4 100644 --- a/Image Downloader GUI/main.py +++ b/Image Downloader GUI/main.py @@ -1,71 +1,71 @@ -import tkinter as tk -from tkinter import messagebox -import os -import requests - - -def get_ext(url: str) -> str | None: - exts = [".png", ".jpeg", ".jpg"] - for ext in exts: - if ext in url: - return ext - return None - - -def download_img(): - u = url_ent.get() - n = name_ent.get() - f = folder_ent.get() - - if not u or not n or not f: - messagebox.showerror("Error", "Please fill in all fields.") - return - - ext = get_ext(u) - if not ext: - messagebox.showerror("Error", "Invalid image URL.") - return - - img_path = os.path.join(f, f"{n}{ext}") - - if os.path.isfile(img_path): - messagebox.showerror( - "Error", "A file with the same name already exists.") - return - - try: - img_content = requests.get(u).content - with open(img_path, "wb") as handler: - handler.write(img_content) - messagebox.showinfo("Success", f"Image downloaded to:\n{img_path}") - except Exception as e: - messagebox.showerror("Error", f"An error occurred: {e}") - - -root = tk.Tk() -root.title("Image Downloader") -root.configure(bg="#1e1e1e") - -ul = tk.Label(root, text="Image URL:", bg="#1e1e1e", fg="white") -ul.pack(pady=5) - -url_ent = tk.Entry(root, width=50) -url_ent.pack(pady=5) - -nl = tk.Label(root, text="Image Name:", bg="#1e1e1e", fg="white") -nl.pack(pady=5) - -name_ent = tk.Entry(root, width=50) -name_ent.pack(pady=5) - -fl = tk.Label(root, text="Folder Path:", bg="#1e1e1e", fg="white") -fl.pack(pady=5) - -folder_ent = tk.Entry(root, width=50) -folder_ent.pack(pady=5) - -dl_btn = tk.Button(root, text="Download", - command=download_img, bg="#303030", fg="white") -dl_btn.pack(pady=10) - -root.mainloop() +import tkinter as tk +from tkinter import messagebox +import os +from security import safe_requests + + +def get_ext(url: str) -> str | None: + exts = [".png", ".jpeg", ".jpg"] + for ext in exts: + if ext in url: + return ext + return None + + +def download_img(): + u = url_ent.get() + n = name_ent.get() + f = folder_ent.get() + + if not u or not n or not f: + messagebox.showerror("Error", "Please fill in all fields.") + return + + ext = get_ext(u) + if not ext: + messagebox.showerror("Error", "Invalid image URL.") + return + + img_path = os.path.join(f, f"{n}{ext}") + + if os.path.isfile(img_path): + messagebox.showerror( + "Error", "A file with the same name already exists.") + return + + try: + img_content = safe_requests.get(u).content + with open(img_path, "wb") as handler: + handler.write(img_content) + messagebox.showinfo("Success", f"Image downloaded to:\n{img_path}") + except Exception as e: + messagebox.showerror("Error", f"An error occurred: {e}") + + +root = tk.Tk() +root.title("Image Downloader") +root.configure(bg="#1e1e1e") + +ul = tk.Label(root, text="Image URL:", bg="#1e1e1e", fg="white") +ul.pack(pady=5) + +url_ent = tk.Entry(root, width=50) +url_ent.pack(pady=5) + +nl = tk.Label(root, text="Image Name:", bg="#1e1e1e", fg="white") +nl.pack(pady=5) + +name_ent = tk.Entry(root, width=50) +name_ent.pack(pady=5) + +fl = tk.Label(root, text="Folder Path:", bg="#1e1e1e", fg="white") +fl.pack(pady=5) + +folder_ent = tk.Entry(root, width=50) +folder_ent.pack(pady=5) + +dl_btn = tk.Button(root, text="Download", + command=download_img, bg="#303030", fg="white") +dl_btn.pack(pady=10) + +root.mainloop() diff --git a/Image-Scraper/scrape_images.py b/Image-Scraper/scrape_images.py index 644c3c1a24..8a7c6e8bb2 100644 --- a/Image-Scraper/scrape_images.py +++ b/Image-Scraper/scrape_images.py @@ -1,13 +1,13 @@ # Scrape all HTML tags from a provided URL. from bs4 import BeautifulSoup -import requests import sys +from security import safe_requests if len(sys.argv) != 2: sys.exit("Usage: python scrape_images.py {url}") -response = requests.get( +response = safe_requests.get( sys.argv[1], headers={ "User-Agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.90 Safari/537.36" diff --git a/Image-scrape-download/image.py b/Image-scrape-download/image.py index a678bc3df9..211389da1f 100644 --- a/Image-scrape-download/image.py +++ b/Image-scrape-download/image.py @@ -3,10 +3,10 @@ from tkhtmlview import HTMLLabel from tkinter import ttk from tkinter import font as tkFont -import requests from bs4 import BeautifulSoup import urllib3 import shutil +from security import safe_requests # Function to save image to the users file system @@ -26,7 +26,7 @@ def getImage(): search = search_box.get() url = "https://www.bing.com/images/search?q={}".format( search.replace(' ', '?')) - page = requests.get(url) + page = safe_requests.get(url) # Start scraping resultant html data soup = BeautifulSoup(page.content, 'html.parser') diff --git a/Instagram downloader/downloader.py b/Instagram downloader/downloader.py index 31dbd01f1a..06773c1473 100644 --- a/Instagram downloader/downloader.py +++ b/Instagram downloader/downloader.py @@ -4,8 +4,8 @@ import selenium.common.exceptions import time from bs4 import BeautifulSoup as bs -import requests import os +from security import safe_requests # get instagram account credentials @@ -181,7 +181,7 @@ def save_content(class_name, img_name): else: link = soup.find('img')['src'] - response = requests.get(link) + response = safe_requests.get(link) with open(img_name, 'wb') as f: f.write(response.content) @@ -210,7 +210,7 @@ def save_multiple(img_name, elem, last_img_flag=False): else: link = user_image.find('img')['src'] - response = requests.get(link) + response = safe_requests.get(link) with open(img_name, 'wb') as f: f.write(response.content) diff --git a/Instagram-Profile-Pic-Downloader/Instagram Profile Pic Downloader.py b/Instagram-Profile-Pic-Downloader/Instagram Profile Pic Downloader.py index 86f3f0d242..bfbc3b719a 100644 --- a/Instagram-Profile-Pic-Downloader/Instagram Profile Pic Downloader.py +++ b/Instagram-Profile-Pic-Downloader/Instagram Profile Pic Downloader.py @@ -1,14 +1,14 @@ -# It will download image in hd - -import requests -from bs4 import BeautifulSoup as soup - -if __name__ == '__main__': - uname = str(input("Enter the username : ")) - new = requests.get('https://www.instadp.com/fullsize/{}'.format(uname)) - imgsoup = soup(new.content, 'lxml') - imglink = imgsoup.find('img', {'class': 'picture'})['src'] - imgfind = requests.get(imglink) - photo = open('photo.png', 'wb') - photo.write(imgfind.content) - photo.close() +# It will download image in hd + +from bs4 import BeautifulSoup as soup +from security import safe_requests + +if __name__ == '__main__': + uname = str(input("Enter the username : ")) + new = safe_requests.get('https://www.instadp.com/fullsize/{}'.format(uname)) + imgsoup = soup(new.content, 'lxml') + imglink = imgsoup.find('img', {'class': 'picture'})['src'] + imgfind = safe_requests.get(imglink) + photo = open('photo.png', 'wb') + photo.write(imgfind.content) + photo.close() diff --git a/Internshala-Scraper/program.py b/Internshala-Scraper/program.py index eb6b358f93..836f44ed23 100644 --- a/Internshala-Scraper/program.py +++ b/Internshala-Scraper/program.py @@ -1,6 +1,6 @@ from bs4 import BeautifulSoup -import requests import csv +from security import safe_requests pages = int(input('How many pages do you want to scrape ? : ')) dict0 = { @@ -49,7 +49,7 @@ ]) for i in range(1, pages + 1): print('Page', i) - resp = requests.get(url + "/page-" + str(i)) + resp = safe_requests.get(url + "/page-" + str(i)) data = BeautifulSoup(resp.content, 'lxml') companies = data.findAll("div", {"class": "heading_6 company_name"}) profiles = data.findAll("div", {"class": "heading_4_5 profile"}) diff --git a/Job Scraper/JobScraper.py b/Job Scraper/JobScraper.py index 57f08c4660..bc6379be74 100644 --- a/Job Scraper/JobScraper.py +++ b/Job Scraper/JobScraper.py @@ -1,112 +1,112 @@ -import urllib -import time -import requests -import re -import csv -from bs4 import BeautifulSoup - - -def write_csv(loc, info): - """ - The function writes the job openings collected in a .csv file - """ - headers = ['Title', 'Company Name', 'Location', 'Date', 'Summary', 'Url'] - - # Adding info into the rows of the file - with open('./Job Scraper/' + loc+'_openings.csv', 'a', encoding='utf-8') as csv_f: - csv_p = csv.writer(csv_f, delimiter=',') - csv_p.writerow(headers) - csv_p.writerows(info) - - print(f'\n{loc}_openings.csv has been saved to your directory!\n') - - -def job_scraper(): - """ - The function scrapes the required number of job openings posted for a given job title and location - and stores all the associated information in a .csv file - """ - title = input("\nEnter job title: ").replace(" ", "+") - loc = input("Enter job location: ").replace(" ", "+") - num = int(input("Enter the number of job openings to obtain: ")) - - url = f'https://in.indeed.com/jobs?q={title}&l={loc}' - req_page = requests.get(url) - - job_array = [] - - if req_page.status_code == 200: - soup = BeautifulSoup(req_page.text, "html.parser") - job_table = soup.find("td", id="resultsCol") - count = 0 - - flag = 1 - while flag: - for job_card in job_table.find_all("div", class_="jobsearch-SerpJobCard"): - # Getting the job title - title_elem = job_card.find( - 'a', class_='jobtitle turnstileLink') - title = title_elem.text.strip() - - # Getting the company name - company_details = job_card.find('div', class_='sjcl') - company_name = company_details.find('span', class_='company') - company_name = company_name.text.strip() - - # Getting the company location - company_loc = company_details.find('span', class_='location') - if company_loc != None: - company_loc = company_loc.text.strip() - else: - company_loc = loc - - # Getting the URL of the post - link = job_card.find('a')['href'] - link = 'https://in.indeed.com' + link - - # Getting the date of the post - date_elem = job_card.find('span', class_='date') - date = date_elem.text.strip() - - # Getting the job summary - summary_ele = job_card.findAll( - 'div', attrs={'class': 'summary'}) - for span in summary_ele: - span = span.text.strip() - - count += 1 - - job_array.append( - [title, company_name, company_loc, date, span, link]) - if count == num: - flag = 0 - break - - # To go to the next page - page = soup.find("ul", class_="pagination-list") - found = 0 - for page in page.find_all('a'): - if page.attrs['aria-label'] == 'Next': - found = 1 - break - - if found: - next_page_link = 'https://in.indeed.com' + page.attrs['href'] - - time.sleep(2) - - req_page = requests.get(next_page_link) - soup = BeautifulSoup(req_page.text, "html.parser") - job_table = soup.find("td", id="resultsCol") - - else: - flag = 0 - - write_csv(loc, job_array) - - else: - print('There seems to be a problem fetching the results. Check your inputs, connections and try again') - - -if __name__ == '__main__': - job_scraper() +import urllib +import time +import re +import csv +from bs4 import BeautifulSoup +from security import safe_requests + + +def write_csv(loc, info): + """ + The function writes the job openings collected in a .csv file + """ + headers = ['Title', 'Company Name', 'Location', 'Date', 'Summary', 'Url'] + + # Adding info into the rows of the file + with open('./Job Scraper/' + loc+'_openings.csv', 'a', encoding='utf-8') as csv_f: + csv_p = csv.writer(csv_f, delimiter=',') + csv_p.writerow(headers) + csv_p.writerows(info) + + print(f'\n{loc}_openings.csv has been saved to your directory!\n') + + +def job_scraper(): + """ + The function scrapes the required number of job openings posted for a given job title and location + and stores all the associated information in a .csv file + """ + title = input("\nEnter job title: ").replace(" ", "+") + loc = input("Enter job location: ").replace(" ", "+") + num = int(input("Enter the number of job openings to obtain: ")) + + url = f'https://in.indeed.com/jobs?q={title}&l={loc}' + req_page = safe_requests.get(url) + + job_array = [] + + if req_page.status_code == 200: + soup = BeautifulSoup(req_page.text, "html.parser") + job_table = soup.find("td", id="resultsCol") + count = 0 + + flag = 1 + while flag: + for job_card in job_table.find_all("div", class_="jobsearch-SerpJobCard"): + # Getting the job title + title_elem = job_card.find( + 'a', class_='jobtitle turnstileLink') + title = title_elem.text.strip() + + # Getting the company name + company_details = job_card.find('div', class_='sjcl') + company_name = company_details.find('span', class_='company') + company_name = company_name.text.strip() + + # Getting the company location + company_loc = company_details.find('span', class_='location') + if company_loc != None: + company_loc = company_loc.text.strip() + else: + company_loc = loc + + # Getting the URL of the post + link = job_card.find('a')['href'] + link = 'https://in.indeed.com' + link + + # Getting the date of the post + date_elem = job_card.find('span', class_='date') + date = date_elem.text.strip() + + # Getting the job summary + summary_ele = job_card.findAll( + 'div', attrs={'class': 'summary'}) + for span in summary_ele: + span = span.text.strip() + + count += 1 + + job_array.append( + [title, company_name, company_loc, date, span, link]) + if count == num: + flag = 0 + break + + # To go to the next page + page = soup.find("ul", class_="pagination-list") + found = 0 + for page in page.find_all('a'): + if page.attrs['aria-label'] == 'Next': + found = 1 + break + + if found: + next_page_link = 'https://in.indeed.com' + page.attrs['href'] + + time.sleep(2) + + req_page = safe_requests.get(next_page_link) + soup = BeautifulSoup(req_page.text, "html.parser") + job_table = soup.find("td", id="resultsCol") + + else: + flag = 0 + + write_csv(loc, job_array) + + else: + print('There seems to be a problem fetching the results. Check your inputs, connections and try again') + + +if __name__ == '__main__': + job_scraper() diff --git a/Link Scrapper/script.py b/Link Scrapper/script.py index d32714103e..5adf4baa12 100644 --- a/Link Scrapper/script.py +++ b/Link Scrapper/script.py @@ -1,9 +1,9 @@ -import requests from bs4 import BeautifulSoup +from security import safe_requests def scrape_links(url): - response = requests.get(url) + response = safe_requests.get(url) soup = BeautifulSoup(response.text, 'html.parser') links = soup.find_all('a') for link in links: diff --git a/Link-Preview/linkPreview.py b/Link-Preview/linkPreview.py index 629028e7ff..37cad3176f 100644 --- a/Link-Preview/linkPreview.py +++ b/Link-Preview/linkPreview.py @@ -1,8 +1,8 @@ -import requests import json import os import time from bs4 import BeautifulSoup +from security import safe_requests # to scrape title @@ -122,7 +122,7 @@ def printData(data): # if not in db get via request # getting the html - r = requests.get(url) + r = safe_requests.get(url) soup = BeautifulSoup(r.text, "html.parser") sevenDaysInSec = 7*24*60*60 diff --git a/LinkedIn_Profile_Info/google_API.py b/LinkedIn_Profile_Info/google_API.py index eb5be2eff8..2fb092f522 100644 --- a/LinkedIn_Profile_Info/google_API.py +++ b/LinkedIn_Profile_Info/google_API.py @@ -2,6 +2,7 @@ import requests import logging import time +from security import safe_requests logger = logging.getLogger(__name__) @@ -23,7 +24,7 @@ def _hit_api(self, linkedin_id: str) -> list: params = self._params.copy() params["exactTerms"] = f"/in/{linkedin_id}" while True: - resp = requests.get(self._api_url, params=params) + resp = safe_requests.get(self._api_url, params=params) if resp.status_code == 200: data = resp.json() items = data.get("items", []) diff --git a/LinkedIn_Profile_Info/profile.py b/LinkedIn_Profile_Info/profile.py index 98207ee2fb..3c96417e92 100644 --- a/LinkedIn_Profile_Info/profile.py +++ b/LinkedIn_Profile_Info/profile.py @@ -2,6 +2,7 @@ import requests from urllib.parse import urlparse, unquote import logging +from security import safe_requests logger = logging.getLogger(__name__) @@ -23,7 +24,7 @@ def _hit_api(self, linkedin_id: str) -> list: params = self._params.copy() params["exactTerms"] = f"/in/{linkedin_id}" while True: - resp = requests.get(self._api_url, params=params) + resp = safe_requests.get(self._api_url, params=params) if resp.status_code != 200: logger.warning( f"Google Custom Search API error: {resp.status_code} - {resp.text}") diff --git a/Linkedin_Posts_Scrapper/Linkedin_Posts_Scrapper.py b/Linkedin_Posts_Scrapper/Linkedin_Posts_Scrapper.py index 0f7a56a57b..f12d92939c 100644 --- a/Linkedin_Posts_Scrapper/Linkedin_Posts_Scrapper.py +++ b/Linkedin_Posts_Scrapper/Linkedin_Posts_Scrapper.py @@ -1,24 +1,24 @@ -import requests - - -def get_linkedin_posts(access_token): - url = 'https://api.linkedin.com/v2/shares?q=owners&owners=urn:li:person:YOUR_USER_ID&count=10' - headers = {'Authorization': f'Bearer {access_token}'} - - response = requests.get(url, headers=headers) - data = response.json() - - if response.status_code == 200: - posts = data.get('elements', []) - for post in posts: - post_text = post.get('text', '') - print(post_text) - else: - print( - f"Error: {response.status_code} - {data.get('message', 'Unknown error')}") - - -if __name__ == "__main__": - # Replace YOUR_ACCESS_TOKEN and YOUR_USER_ID with appropriate values - access_token = "YOUR_ACCESS_TOKEN" - get_linkedin_posts(access_token) +from security import safe_requests + + +def get_linkedin_posts(access_token): + url = 'https://api.linkedin.com/v2/shares?q=owners&owners=urn:li:person:YOUR_USER_ID&count=10' + headers = {'Authorization': f'Bearer {access_token}'} + + response = safe_requests.get(url, headers=headers) + data = response.json() + + if response.status_code == 200: + posts = data.get('elements', []) + for post in posts: + post_text = post.get('text', '') + print(post_text) + else: + print( + f"Error: {response.status_code} - {data.get('message', 'Unknown error')}") + + +if __name__ == "__main__": + # Replace YOUR_ACCESS_TOKEN and YOUR_USER_ID with appropriate values + access_token = "YOUR_ACCESS_TOKEN" + get_linkedin_posts(access_token) diff --git a/Linkedin_Video_Downloader/script.py b/Linkedin_Video_Downloader/script.py index 961fa173d5..f7270376b7 100644 --- a/Linkedin_Video_Downloader/script.py +++ b/Linkedin_Video_Downloader/script.py @@ -1,13 +1,13 @@ # ALL Imports import tkinter as tk -import requests as req import html import time from tkinter.ttk import * from threading import Thread import queue from queue import Empty +from security import safe_requests def Invalid_Url(): @@ -31,7 +31,7 @@ def Download_vid(): Invalid_Url() return - response = req.get(url) + response = safe_requests.get(url) if not response.status_code == 200: Invalid_Url() @@ -66,7 +66,7 @@ def run(self): # save the picture to a file block_size = 1024 # 1kB - r = req.get(self.url, stream=True) + r = safe_requests.get(self.url, stream=True) total_size = int(r.headers.get("content-length")) with open('video.mp4', 'wb') as file: diff --git a/Live Currency Converter GUI/app.py b/Live Currency Converter GUI/app.py index 37b47c96c1..3040abcded 100644 --- a/Live Currency Converter GUI/app.py +++ b/Live Currency Converter GUI/app.py @@ -1,86 +1,86 @@ -import tkinter as tk -from tkinter import messagebox -import requests - -BASE_URL = "http://api.exchangeratesapi.io/v1/latest" -API_KEY = "Your API Key" - - -class CurrencyConverterApp: - def __init__(self, root): - self.root = root - self.root.title("Currency Converter") - self.root.configure(bg="black") - - self.amount_label = tk.Label( - root, text="Amount:", fg="white", bg="black") - self.amount_label.pack() - - self.amount_entry = tk.Entry(root) - self.amount_entry.pack() - - self.from_label = tk.Label( - root, text="From Currency:", fg="white", bg="black") - self.from_label.pack() - - self.from_currency_entry = tk.Entry(root) - self.from_currency_entry.pack() - - self.to_label = tk.Label( - root, text="To Currency:", fg="white", bg="black") - self.to_label.pack() - - self.to_currency_entry = tk.Entry(root) - self.to_currency_entry.pack() - - self.convert_button = tk.Button( - root, text="Convert", command=self.convert_currency) - self.convert_button.pack() - - def get_rates(self): - payload = {"access_key": API_KEY} - - response = requests.get(url=BASE_URL, params=payload) - data = response.json() - - return data.get("rates") - - def get_currency(self, currency, rates): - currency = currency.upper() - if currency in rates.keys(): - - return rates.get(currency) - - else: - raise ValueError(f"{currency} is not a valid currency") - - def convert_currency(self): - amount = float(self.amount_entry.get()) - - from_currency = self.from_currency_entry.get() - - to_currency = self.to_currency_entry.get() - - rates = self.get_rates() - - from_rate = self.get_currency(from_currency, rates) - to_rate = self.get_currency(to_currency, rates) - - conversion = round((to_rate / from_rate) * amount, 2) - - messagebox.showinfo( - "Conversion Result", f"{amount:.2f} ({from_currency}) is {conversion:.2f} ({to_currency})") - - -def main(): - app = tk.Tk() - - app.configure(bg="black") - - currency_converter = CurrencyConverterApp(app) - - app.mainloop() - - -if __name__ == "__main__": - main() +import tkinter as tk +from tkinter import messagebox +from security import safe_requests + +BASE_URL = "http://api.exchangeratesapi.io/v1/latest" +API_KEY = "Your API Key" + + +class CurrencyConverterApp: + def __init__(self, root): + self.root = root + self.root.title("Currency Converter") + self.root.configure(bg="black") + + self.amount_label = tk.Label( + root, text="Amount:", fg="white", bg="black") + self.amount_label.pack() + + self.amount_entry = tk.Entry(root) + self.amount_entry.pack() + + self.from_label = tk.Label( + root, text="From Currency:", fg="white", bg="black") + self.from_label.pack() + + self.from_currency_entry = tk.Entry(root) + self.from_currency_entry.pack() + + self.to_label = tk.Label( + root, text="To Currency:", fg="white", bg="black") + self.to_label.pack() + + self.to_currency_entry = tk.Entry(root) + self.to_currency_entry.pack() + + self.convert_button = tk.Button( + root, text="Convert", command=self.convert_currency) + self.convert_button.pack() + + def get_rates(self): + payload = {"access_key": API_KEY} + + response = safe_requests.get(url=BASE_URL, params=payload) + data = response.json() + + return data.get("rates") + + def get_currency(self, currency, rates): + currency = currency.upper() + if currency in rates.keys(): + + return rates.get(currency) + + else: + raise ValueError(f"{currency} is not a valid currency") + + def convert_currency(self): + amount = float(self.amount_entry.get()) + + from_currency = self.from_currency_entry.get() + + to_currency = self.to_currency_entry.get() + + rates = self.get_rates() + + from_rate = self.get_currency(from_currency, rates) + to_rate = self.get_currency(to_currency, rates) + + conversion = round((to_rate / from_rate) * amount, 2) + + messagebox.showinfo( + "Conversion Result", f"{amount:.2f} ({from_currency}) is {conversion:.2f} ({to_currency})") + + +def main(): + app = tk.Tk() + + app.configure(bg="black") + + currency_converter = CurrencyConverterApp(app) + + app.mainloop() + + +if __name__ == "__main__": + main() diff --git a/Lyrics_Finder/script.py b/Lyrics_Finder/script.py index 0fab39483a..186265addf 100644 --- a/Lyrics_Finder/script.py +++ b/Lyrics_Finder/script.py @@ -1,9 +1,9 @@ -import requests +from security import safe_requests def find_lyrics(artist, title): base_url = 'https://api.lyrics.ovh/v1/{}/{}'.format(artist, title) - response = requests.get(base_url) + response = safe_requests.get(base_url) if response.status_code == 200: data = response.json() diff --git a/Movie recommendation system/app.py b/Movie recommendation system/app.py index c972f68be4..4d8f43e8d8 100644 --- a/Movie recommendation system/app.py +++ b/Movie recommendation system/app.py @@ -1,13 +1,13 @@ import pickle import streamlit as st -import requests import pandas as pd +from security import safe_requests def fetch_poster(movie_id): url = "https://api.themoviedb.org/3/movie/{}?api_key=8265bd1679663a7ea12ac168da84d2e8&language=en-US".format( movie_id) - data = requests.get(url) + data = safe_requests.get(url) data = data.json() poster_path = data['poster_path'] full_path = "https://image.tmdb.org/t/p/w500/" + poster_path diff --git a/Movie-Info-Telegram-Bot/bot.py b/Movie-Info-Telegram-Bot/bot.py index c2b41f0a8d..60e18032c9 100644 --- a/Movie-Info-Telegram-Bot/bot.py +++ b/Movie-Info-Telegram-Bot/bot.py @@ -1,5 +1,4 @@ import logging -import requests import re import urllib.request import urllib.parse @@ -9,6 +8,7 @@ import itertools from telegram.ext import Updater, CommandHandler, MessageHandler, Filters import decouple +from security import safe_requests # Enable logging logging.basicConfig(format='%(asctime)s - %(name)s - %(levelname)s - %(message)s', @@ -41,7 +41,7 @@ def genre(update, context): url = 'https://www.imdb.com/search/title/' genre = str(update.message.text)[7:] print(genre) - r = requests.get(url+'?genres='+genre) + r = safe_requests.get(url+'?genres='+genre) soup = BeautifulSoup(r.text, "html.parser") title = soup.find('title') if title.string == 'IMDb: Advanced Title Search - IMDb': @@ -85,7 +85,7 @@ def error(update, context): def get_info(movie): "To scrape IMDb and get genre and rating of the movie " url = 'https://www.imdb.com/find?q=' - r = requests.get(url+movie+'&ref_=nv_sr_sm') + r = safe_requests.get(url+movie+'&ref_=nv_sr_sm') soup = BeautifulSoup(r.text, "html.parser") title = soup.find('title') tags = soup('a') @@ -102,7 +102,7 @@ def get_info(movie): link = re.search('/title/(.*?)/', str(m)) new_url = 'https://www.imdb.com'+str(link.group(0)) if new_url != pre_url: - html = requests.get(new_url) + html = safe_requests.get(new_url) soup2 = BeautifulSoup(html.text, "html.parser") movietitle = soup2.find('title').string.replace('- IMDb', ' ') a = soup2('a') diff --git a/Movie-Recommendation/script.py b/Movie-Recommendation/script.py index 1506ad5cc0..702c2487f0 100644 --- a/Movie-Recommendation/script.py +++ b/Movie-Recommendation/script.py @@ -4,7 +4,7 @@ # scrapping from bs4 import BeautifulSoup as SOUP import re -import requests as HTTP +from security import safe_requests # Main Function for scraping @@ -53,7 +53,7 @@ def main(emotion): # HTTP request to get the data of # the whole page - response = HTTP.get(urlhere) + response = safe_requests.get(urlhere) data = response.text # Parsing the data using diff --git a/News_Scrapper/scrapper.py b/News_Scrapper/scrapper.py index 9ef5c7fba7..8eff0f7365 100644 --- a/News_Scrapper/scrapper.py +++ b/News_Scrapper/scrapper.py @@ -1,6 +1,6 @@ from bs4 import BeautifulSoup -import requests import csv +from security import safe_requests URL = "https://www.indiatoday.in/" @@ -19,7 +19,7 @@ def getTopTenFromDivTag(category): count = 0 category_url = URL + category - page = requests.get(category_url) + page = safe_requests.get(category_url) soup = BeautifulSoup(page.text, "html.parser") all_div_tags = soup.find_all(class_="detail") @@ -41,7 +41,7 @@ def getTopTenFromLiTag(category): count = 0 category_url = URL + category - page = requests.get(category_url) + page = safe_requests.get(category_url) soup = BeautifulSoup(page.text, "html.parser") ul_tag = soup.find_all(class_="itg-listing") diff --git a/Pagespeed-API/pagespeed.py b/Pagespeed-API/pagespeed.py index acf24daf3e..76e9fe2196 100644 --- a/Pagespeed-API/pagespeed.py +++ b/Pagespeed-API/pagespeed.py @@ -1,6 +1,6 @@ -import requests import json from responses import PageSpeedResponse +from security import safe_requests class PageSpeed(object): @@ -44,7 +44,7 @@ def analyse(self, url, strategy='desktop', category='performance'): raise ValueError('invalid strategy: {0}'.format(strategy)) # Returns raw data - raw = requests.get(self.endpoint, params=params) + raw = safe_requests.get(self.endpoint, params=params) response = PageSpeedResponse(raw) diff --git a/Pixabay Scraper/main.py b/Pixabay Scraper/main.py index 5b405ee2bb..60a6dded7a 100644 --- a/Pixabay Scraper/main.py +++ b/Pixabay Scraper/main.py @@ -1,4 +1,4 @@ -import requests +from security import safe_requests class Pixabay(): @@ -61,7 +61,7 @@ def get_video(self, query, num=10, params={}): BASE_URL = 'https://pixabay.com/api/videos/' _params = self.__get_params_video(query, num, params) - response = requests.get(BASE_URL, params=_params) + response = safe_requests.get(BASE_URL, params=_params) data = response.json() hits = data['hits'] @@ -69,7 +69,7 @@ def get_video(self, query, num=10, params={}): if self.verbose: print(f" Downloading Pixabay videos {i+1}/{num}") video_url = hit['videos']['large']['url'] - response = requests.get(video_url) + response = safe_requests.get(video_url) if response is not None: with open(f'video_pixabay_{i+1:02d}.mp4', 'wb') as f: f.write(response.content) @@ -119,7 +119,7 @@ def get_photo(self, query, num=10, params={}): BASE_URL = 'https://pixabay.com/api/' _params = self.__get_params_photo(query, num, params) - response = requests.get(BASE_URL, params=_params) + response = safe_requests.get(BASE_URL, params=_params) data = response.json() hits = data['hits'] @@ -127,7 +127,7 @@ def get_photo(self, query, num=10, params={}): if self.verbose: print(f" Downloading Pixabay photos {i+1}/{num}") image_url = hit['largeImageURL'] - response = requests.get(image_url) + response = safe_requests.get(image_url) if response is not None: with open(f'photo_pixabay_{i+1:02d}.jpg', 'wb') as f: f.write(response.content) diff --git a/Price Comparison and Checker/price_comparison.py b/Price Comparison and Checker/price_comparison.py index e22983f561..9973be4489 100644 --- a/Price Comparison and Checker/price_comparison.py +++ b/Price Comparison and Checker/price_comparison.py @@ -1,6 +1,6 @@ -import requests from bs4 import BeautifulSoup from tabulate import tabulate +from security import safe_requests headers = { "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.141 Safari/537.36"} @@ -10,7 +10,7 @@ def amazon(item): URL = "https://www.amazon.in/s?k=" + item.replace(" ", "+") - page = requests.get(URL, headers=headers) + page = safe_requests.get(URL, headers=headers) soup = BeautifulSoup(page.content, "html.parser") name = "Amazon" # For access to product links un-comment these: @@ -35,7 +35,7 @@ def amazon(item): def flipkart(item): URL = "https://www.flipkart.com/search?q=" + item - page = requests.get(URL, headers=headers) + page = safe_requests.get(URL, headers=headers) soup = BeautifulSoup(page.content, "html.parser") name = "Flipkart" # For access to product links un-comment these: @@ -58,7 +58,7 @@ def flipkart(item): def snapdeal(item): URL = "https://www.snapdeal.com/search?keyword=" + item.replace(" ", "+") - page = requests.get(URL, headers=headers) + page = safe_requests.get(URL, headers=headers) soup = BeautifulSoup(page.content, "html.parser") name = "Snapdeal" diff --git a/Pwned_or_not_GUI/pwned_passwords_GUI.py b/Pwned_or_not_GUI/pwned_passwords_GUI.py index 038a04713d..a88cf39e22 100644 --- a/Pwned_or_not_GUI/pwned_passwords_GUI.py +++ b/Pwned_or_not_GUI/pwned_passwords_GUI.py @@ -1,6 +1,6 @@ import tkinter as tk -import requests import hashlib +from security import safe_requests def send_request_to_API(start_char): @@ -10,7 +10,7 @@ def send_request_to_API(start_char): # first 5 characters of hashed password added to the URL url = 'https://api.pwnedpasswords.com/range/' + start_char try: - res = requests.get(url) + res = safe_requests.get(url) # Only status code of 200 returns relevant data if res.status_code != 200: diff --git a/Rain Alert Mail Sender/main.py b/Rain Alert Mail Sender/main.py index b399a9d3c7..326ffe19ea 100644 --- a/Rain Alert Mail Sender/main.py +++ b/Rain Alert Mail Sender/main.py @@ -1,7 +1,7 @@ -import requests from smtplib import SMTP import os from dotenv import load_dotenv +from security import safe_requests load_dotenv() @@ -25,7 +25,7 @@ def function(): 'city': CITY, 'format': 'json', } -response_location = requests.get(url=API_END_POINT, params=PARAMETER_LOCATION) +response_location = safe_requests.get(url=API_END_POINT, params=PARAMETER_LOCATION) data_location = response_location .json() LAT = data_location[0]['lat'] LONG = data_location[0]['lon'] @@ -35,7 +35,7 @@ def function(): "appid": API_KEY, "exclude": "current,minutely,daily", } -api = requests.get( +api = safe_requests.get( url="http://api.openweathermap.org/data/2.5/onecall", params=PARAMETER) data = api.json() diff --git a/Real Estate Webscrapper/real_estate_webscrapper.py b/Real Estate Webscrapper/real_estate_webscrapper.py index 854381b0af..3e17181984 100644 --- a/Real Estate Webscrapper/real_estate_webscrapper.py +++ b/Real Estate Webscrapper/real_estate_webscrapper.py @@ -1,13 +1,13 @@ -import requests from bs4 import BeautifulSoup import pandas +from security import safe_requests headers = { 'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:55.0) Gecko/20100101 Firefox/55.0', } -r = requests.get( +r = safe_requests.get( "https://www.magicbricks.com/ready-to-move-flats-in-new-delhi-pppfs", headers=headers) c = r.content soup = BeautifulSoup(c, "html.parser") diff --git a/Reddit_Scraper_without_API/fetch_reddit.py b/Reddit_Scraper_without_API/fetch_reddit.py index 66fc032a8f..47f91c3f5e 100644 --- a/Reddit_Scraper_without_API/fetch_reddit.py +++ b/Reddit_Scraper_without_API/fetch_reddit.py @@ -1,8 +1,8 @@ -import requests import csv import time import sqlite3 from bs4 import BeautifulSoup +from security import safe_requests def sql_connection(): @@ -80,7 +80,7 @@ def scraper(): # Using a user-agent to mimic browser activity headers = {'User-Agent': 'Mozilla/5.0'} - req = requests.get(url, headers=headers) + req = safe_requests.get(url, headers=headers) if req.status_code == 200: soup = BeautifulSoup(req.text, 'html.parser') @@ -140,7 +140,7 @@ def scraper(): time.sleep(2) - req = requests.get(next_page_link, headers=headers) + req = safe_requests.get(next_page_link, headers=headers) soup = BeautifulSoup(req.text, 'html.parser') except: break diff --git a/Search-Amazing-Python-Scripts/search-scripts.py b/Search-Amazing-Python-Scripts/search-scripts.py index 314a37c744..e2185b7612 100644 --- a/Search-Amazing-Python-Scripts/search-scripts.py +++ b/Search-Amazing-Python-Scripts/search-scripts.py @@ -1,9 +1,9 @@ -import requests +from security import safe_requests def get_repository_contents(repo_owner, repo_name): url = f"https://api.github.com/repos/{repo_owner}/{repo_name}/contents" - response = requests.get(url) + response = safe_requests.get(url) if response.status_code == 200: return response.json() diff --git a/Search-Username/search_user.py b/Search-Username/search_user.py index 8d7bfdc6b9..e02c0ee82b 100644 --- a/Search-Username/search_user.py +++ b/Search-Username/search_user.py @@ -1,7 +1,7 @@ # IMPORTS -import requests import argparse import sys +from security import safe_requests def parser_input(): @@ -69,7 +69,7 @@ def search_web(username, target_website): :param username: Username to be searches across the platforms :param target_website: The targetted website """ - r = requests.get(target_website) + r = safe_requests.get(target_website) if r.status_code == 200: print('Got it ' + username + ' in ' + target_website) elif r.status_code == 400: diff --git a/Slideshare downloader/slideshare_downloader.py b/Slideshare downloader/slideshare_downloader.py index 0de5984d4a..fa8cff0af6 100644 --- a/Slideshare downloader/slideshare_downloader.py +++ b/Slideshare downloader/slideshare_downloader.py @@ -1,6 +1,5 @@ # slideshare downloader from bs4 import BeautifulSoup -import requests import itertools import threading import time @@ -8,6 +7,7 @@ import urllib.request import img2pdf import os +from security import safe_requests task = False process = "getting the slides " @@ -26,7 +26,7 @@ def animate(): def get_image_list(url): - code = requests.get(url) + code = safe_requests.get(url) soup = BeautifulSoup(code.text, "html.parser") print(f"Title: {soup.title.get_text()}") imgs = soup.find_all("img") diff --git a/Social-Media-Links-From-Website/script.py b/Social-Media-Links-From-Website/script.py index 56a3e9d13d..cc65a89c61 100644 --- a/Social-Media-Links-From-Website/script.py +++ b/Social-Media-Links-From-Website/script.py @@ -1,6 +1,6 @@ -import requests as req import re import csv +from security import safe_requests # Social Media links to look for media_links = [ @@ -25,7 +25,7 @@ link = input("\nEnter the website url : ") # Makes a get request to the url - r = req.get(link) + r = safe_requests.get(link) # print(r) if r.status_code == 200: @@ -64,7 +64,7 @@ # Iterating through links in csv file for link in csv_file: link = str(link).strip() - r = req.get(link) + r = safe_requests.get(link) # print(r) if r.status_code == 200: diff --git a/Space News AI Model/Space_News.py b/Space News AI Model/Space_News.py index 8aefe7a537..c5a803fd72 100644 --- a/Space News AI Model/Space_News.py +++ b/Space News AI Model/Space_News.py @@ -12,6 +12,8 @@ import requests import time import nasapy +from security import safe_requests + print('🚀🚀🚀🚀🚀🚀🚀🚀🚀🚀🚀🚀🚀🚀🚀🚀🚀🚀🚀🚀🚀🚀🚀🚀🚀🚀🚀🚀🚀🚀🚀🚀🚀🚀🚀🚀🚀🚀🚀🚀🚀🚀🚀🚀🚀🚀🚀🚀🚀🚀🚀🚀🚀🚀🚀🚀🚀🚀\n') # Enter the API key of NASA @@ -33,7 +35,7 @@ def spaceNews(): print('\nPlease wait.....') Url = "https://api.nasa.gov/planetary/apod?api_key=" + str(Api_Key) Params = {'date': str(Date)} - r = requests.get(Url, params=Params) + r = safe_requests.get(Url, params=Params) # Parsing the fetched data for displaying Data = r.json() @@ -44,7 +46,7 @@ def spaceNews(): Image_Url = Data['url'] # Requesting for the image url to fetch and store into an array - Image_r = requests.get(Image_Url) + Image_r = safe_requests.get(Image_Url) FileName = str(Date) + '.jpg' # Saving the images with open(FileName, 'wb') as f: @@ -69,7 +71,7 @@ def MarsImage(): url = f"https://api.nasa.gov/mars-photos/api/v1/rovers/curiosity/photos?earth_date={date}&api_key={Api_Key}" # Fetching data from the internet - r = requests.get(url) + r = safe_requests.get(url) Data = r.json() # Extracting data from the fetched data set @@ -86,7 +88,7 @@ def MarsImage(): img_url = photo['img_src'] # Downloading images from the internet - p = requests.get(img_url) + p = safe_requests.get(img_url) img = f'{index}.jpg' with open(img, 'wb') as file: file.write(p.content) @@ -110,7 +112,7 @@ def Astro(): url = f"https://api.nasa.gov/neo/rest/v1/feed?start_date={start_date}&end_date={end_date}&api_key={Api_Key}" # Fetching data from the internet - r = requests.get(url) + r = safe_requests.get(url) Data = r.json() # Clear the terminal for the next operation @@ -172,7 +174,7 @@ def SolarBodies(): # Fetching data for that body body = Body url_2 = f"https://api.le-systeme-solaire.net/rest/bodies/{body}" - rrr = requests.get(url_2) # Fetchin data from the internet + rrr = safe_requests.get(url_2) # Fetchin data from the internet data = rrr.json() # Listing / parsing the dataset @@ -244,7 +246,7 @@ def solarBodiesPictures(): imageURL = f'https://drive.google.com/uc?export=view&id={imageKey}' # Fetch the image content - response = requests.get(imageURL) + response = safe_requests.get(imageURL) image_content = response.content # Load the image using PIL diff --git a/Spacex-Calender/notifly.py b/Spacex-Calender/notifly.py index 34f2d25fff..5ff45c840f 100644 --- a/Spacex-Calender/notifly.py +++ b/Spacex-Calender/notifly.py @@ -6,8 +6,8 @@ from google_auth_oauthlib.flow import InstalledAppFlow from google.auth.transport.requests import Request from bs4 import BeautifulSoup -import requests from datetime import datetime +from security import safe_requests # Enter your own Google Calendar API json credentials file below (if you renamed it) api_credentials_json = 'credentials.json' @@ -230,7 +230,7 @@ def main(current_month): # GET HTTP request query_page += 1 - response = requests.get( + response = safe_requests.get( 'https://nextspaceflight.com/launches/?page=' + str(query_page)) soup = BeautifulSoup(response.text, 'html.parser') diff --git a/Stack-overflow-scraper/scraper.py b/Stack-overflow-scraper/scraper.py index 7d1614069c..e37c871cf7 100644 --- a/Stack-overflow-scraper/scraper.py +++ b/Stack-overflow-scraper/scraper.py @@ -1,4 +1,3 @@ -import requests from bs4 import BeautifulSoup import tkinter as tk from tkinter import messagebox, simpledialog @@ -7,6 +6,7 @@ import time import sqlite3 from sqlite3 import Error +from security import safe_requests # Function to connect to the SQL Database @@ -72,7 +72,7 @@ def scrape_questions(): url = get_URL() if url: - page = requests.get(url) + page = safe_requests.get(url) else: clear_progress() return diff --git a/StackOverflow-Scraper/questions.py b/StackOverflow-Scraper/questions.py index b3537d99f6..14e2361ff0 100644 --- a/StackOverflow-Scraper/questions.py +++ b/StackOverflow-Scraper/questions.py @@ -1,6 +1,6 @@ from bs4 import BeautifulSoup -import requests import json +from security import safe_requests class StackOverflow: @@ -27,7 +27,7 @@ def getQuestions(self): """ url = "https://stackoverflow.com/questions/tagged/" + self.topic try: - res = requests.get(url) + res = safe_requests.get(url) soup = BeautifulSoup(res.text, "html.parser") questions_data = {"questions": []} diff --git a/Stackoverflow-Tool/script.py b/Stackoverflow-Tool/script.py index 2825e888fa..dde8aff801 100644 --- a/Stackoverflow-Tool/script.py +++ b/Stackoverflow-Tool/script.py @@ -1,6 +1,6 @@ import subprocess -import requests import webbrowser +from security import safe_requests # take the input of the python file file = input( @@ -23,7 +23,7 @@ URL = "https://api.stackexchange.com/2.2/search" PARAMS = {'intitle': errorType, 'tagged': 'python', 'nottagged': errorMessage, 'sort': 'votes', 'site': 'stackoverflow'} -r = requests.get(url=URL, params=PARAMS) +r = safe_requests.get(url=URL, params=PARAMS) data = r.json() links = [] diff --git a/Swiggy_Scraper/main.py b/Swiggy_Scraper/main.py index ae471d935d..3eec95e6d2 100644 --- a/Swiggy_Scraper/main.py +++ b/Swiggy_Scraper/main.py @@ -1,5 +1,5 @@ -import requests from bs4 import BeautifulSoup as bs +from security import safe_requests def get_restraunt_details(restraunt_url): @@ -29,7 +29,7 @@ def get_restraunt_details(restraunt_url): headers = { "User-Agent": "Mozilla/5.0 (Windows NT 6.3; Win 64 ; x64) Apple WeKit /537.36(KHTML , like Gecko) Chrome/80.0.3987.162 Safari/537.36" } - response = requests.get(restraunt_url, headers=headers).text + response = safe_requests.get(restraunt_url, headers=headers).text soup = bs(response, "lxml") restaurant_data = [] name = soup.find( diff --git a/TOI_Scrapper/TOI_Scrapper.py b/TOI_Scrapper/TOI_Scrapper.py index 20e128a000..c2f5ba58c0 100644 --- a/TOI_Scrapper/TOI_Scrapper.py +++ b/TOI_Scrapper/TOI_Scrapper.py @@ -1,31 +1,31 @@ -import pandas as pd -import requests -from bs4 import BeautifulSoup - - -def scrapper(): - headers = { - 'User-Agent': 'Mozilla/5.0 (Windows NT 6.3; Win 64 ; x64) Apple WeKit /537.36(KHTML , like Gecko) Chrome/80.0.3987.162 Safari/537.36'} - # accessing TOI webpage disguised as a browser - webpage = requests.get( - 'https://timesofindia.indiatimes.com/', headers=headers).text - - soup = BeautifulSoup(webpage, 'lxml') - news = [] - link_list = [] - - for i in soup.find_all('div', class_='col_l_6'): - figcaption = i.find('figcaption') - if figcaption is not None: - # finding news headline as well its corresponding link - link_news = i.find('a').get("href") - text_news = figcaption.text.strip() - - news.append(text_news) - link_list.append(link_news) - df = pd.DataFrame({'News_Headline': news, 'News_Link': link_list}) - return df - - -TOI_headline = scrapper() -print(TOI_headline) +import pandas as pd +from bs4 import BeautifulSoup +from security import safe_requests + + +def scrapper(): + headers = { + 'User-Agent': 'Mozilla/5.0 (Windows NT 6.3; Win 64 ; x64) Apple WeKit /537.36(KHTML , like Gecko) Chrome/80.0.3987.162 Safari/537.36'} + # accessing TOI webpage disguised as a browser + webpage = safe_requests.get( + 'https://timesofindia.indiatimes.com/', headers=headers).text + + soup = BeautifulSoup(webpage, 'lxml') + news = [] + link_list = [] + + for i in soup.find_all('div', class_='col_l_6'): + figcaption = i.find('figcaption') + if figcaption is not None: + # finding news headline as well its corresponding link + link_news = i.find('a').get("href") + text_news = figcaption.text.strip() + + news.append(text_news) + link_list.append(link_news) + df = pd.DataFrame({'News_Headline': news, 'News_Link': link_list}) + return df + + +TOI_headline = scrapper() +print(TOI_headline) diff --git a/TechCrunch-Scraper/techCrunch.py b/TechCrunch-Scraper/techCrunch.py index 3372a988fc..a4f08f595e 100644 --- a/TechCrunch-Scraper/techCrunch.py +++ b/TechCrunch-Scraper/techCrunch.py @@ -1,6 +1,6 @@ -import requests from bs4 import BeautifulSoup import json +from security import safe_requests class TechCrunch: @@ -37,7 +37,7 @@ def get_articles(self, category): category.replace(" ", "-").lower() ) try: - res = requests.get(url) + res = safe_requests.get(url) soup = BeautifulSoup(res.text, "html.parser") articles_data = {"articles": []} @@ -120,7 +120,7 @@ def search(self, topic): """ url = "https://search.techcrunch.com/search?p=" + topic + "&fr=techcrunch" try: - res = requests.get(url) + res = safe_requests.get(url) soup = BeautifulSoup(res.text, "html.parser") articles_data = {"articles": []} diff --git a/Telegram-Weather-Bot/main.py b/Telegram-Weather-Bot/main.py index 5bd3e462b8..3b773e051f 100644 --- a/Telegram-Weather-Bot/main.py +++ b/Telegram-Weather-Bot/main.py @@ -1,5 +1,5 @@ -import requests import telebot +from security import safe_requests bot_token = 'YOUR_BOT_TOKEN' bot = telebot.TeleBot(bot_token) @@ -12,7 +12,7 @@ def send_welcome(message): @bot.message_handler(func=lambda m: True) def echo_all(message): - response = requests.get("http://api.weatherapi.com/v1/current.json?key={}&q={}".format( + response = safe_requests.get("http://api.weatherapi.com/v1/current.json?key={}&q={}".format( "2d3f4a2bd175414aa45175205221408", message.text)).json() bot.send_message( message.chat.id, format_response_to_human_readable(response)) diff --git a/Telegram_Book_Bot/book.py b/Telegram_Book_Bot/book.py index 09dbbacc4d..15ce087219 100644 --- a/Telegram_Book_Bot/book.py +++ b/Telegram_Book_Bot/book.py @@ -1,13 +1,13 @@ # for scraping books from bs4 import BeautifulSoup as bs -import requests # to identify emoji unicode characters import demoji +from security import safe_requests def link_to_get(link): """This function will get the url of the image & book download direct link using the given link for book download""" - response = requests.get(link) + response = safe_requests.get(link) th_html = bs(response.text, "html.parser") td_all = th_html.find_all("td", id="info") td_all = td_all[0] @@ -45,7 +45,7 @@ def book_get(name, mainres=25, results=5): name = name.replace(" ", "+") # getting request and response url = f"http://libgen.is/search.php?req={name}&lg_topic=libgen&open=0&view=simple&res={mainres}&phrase=1&column=def" - response = requests.get(url) + response = safe_requests.get(url) bs_html = bs(response.text, "html.parser") if "Search string must contain minimum 3 characters.." in bs_html.body: diff --git a/Temporary-Sign-Up-Tool/temp_sign_up_tool.py b/Temporary-Sign-Up-Tool/temp_sign_up_tool.py index 6f996ac1ee..8542c3702e 100644 --- a/Temporary-Sign-Up-Tool/temp_sign_up_tool.py +++ b/Temporary-Sign-Up-Tool/temp_sign_up_tool.py @@ -1,9 +1,9 @@ from bs4 import BeautifulSoup as soup # HTML data structure -import requests # Web client import webbrowser import string import random import sys +from security import safe_requests def generateNames(): @@ -43,12 +43,12 @@ def generateEmailIdAndLink(): } # using request.get on https://10minutemail.net/ site, and pass header as extra data - response = requests.get('https://10minutemail.net/', headers=headers) + response = safe_requests.get('https://10minutemail.net/', headers=headers) # the response has session Id required to retrieve email and link ses = str(response.cookies).split(",")[2].split(" ")[2] # we then pass this session id to get JSON data for the email - response = requests.get('https://10minutemail.net/address.api.php?' + ses) + response = safe_requests.get('https://10minutemail.net/address.api.php?' + ses) page_soup = soup(response.content, "html.parser") # saving the json # selecting the needed data from the json, cleaning the data diff --git a/Text-to-Image/dalle.py b/Text-to-Image/dalle.py index 54a40aa922..9eb14fa6a5 100644 --- a/Text-to-Image/dalle.py +++ b/Text-to-Image/dalle.py @@ -1,5 +1,6 @@ +from security import safe_requests + def dalle(message): - import requests import openai # Set up the OpenAI API client @@ -16,7 +17,7 @@ def dalle(message): size="1024x1024" ) image_url = response['data'][0]['url'] - response = requests.get(image_url) + response = safe_requests.get(image_url) with open("image.jpg", "wb") as f: f.write(response.content) diff --git a/TradingView/setup.py b/TradingView/setup.py index cb31e34659..bdafed0e76 100644 --- a/TradingView/setup.py +++ b/TradingView/setup.py @@ -48,7 +48,7 @@ author_email="d+fixtool@0x1.org", license="MIT", keywords="fix testing", - install_requires=["simplefix>=1.0.8"], + install_requires=["simplefix>=1.0.8", "security==1.3.1"], package_dir={"": "python"}, packages=["fixtool"], entry_points={ diff --git a/Twitter_Video_Downloader/script.py b/Twitter_Video_Downloader/script.py index 4436921025..298912dbae 100644 --- a/Twitter_Video_Downloader/script.py +++ b/Twitter_Video_Downloader/script.py @@ -1,6 +1,6 @@ from tkinter import * import subprocess -import requests +from security import safe_requests def Invalid_URL(): @@ -22,7 +22,7 @@ def Download_vid(): if not "twitter.com" in url: Invalid_URL() return - response = requests.get(url) + response = safe_requests.get(url) if not response.status_code == 200: Invalid_URL() response.close() diff --git a/Weather-App/weatherapp.py b/Weather-App/weatherapp.py index 8d77ed8fe9..5792ad18fa 100644 --- a/Weather-App/weatherapp.py +++ b/Weather-App/weatherapp.py @@ -1,7 +1,7 @@ # Importing required modules -import requests from bs4 import BeautifulSoup import tkinter as tk +from security import safe_requests # Creating function to get temperature @@ -14,7 +14,7 @@ def get_temp(): # Searching it on google url = f"https://www.google.com/search?&q={search}" # Sending and Receiving Requests - r = requests.get(url) + r = safe_requests.get(url) # Scrape the temperature from the search results s = BeautifulSoup(r.text, "html.parser") # Storing details diff --git a/Web_Application_Mapper/Web_Application_Mapper.py b/Web_Application_Mapper/Web_Application_Mapper.py index e6136afca2..1fe78b7749 100644 --- a/Web_Application_Mapper/Web_Application_Mapper.py +++ b/Web_Application_Mapper/Web_Application_Mapper.py @@ -1,6 +1,7 @@ import os import threading import requests +from security import safe_requests threads = 10 target = "http://www.test.com" @@ -28,7 +29,7 @@ def test_remote(): url = f"{target}{path}" try: - response = requests.get(url) + response = safe_requests.get(url) print(f"[{response.status_code}] => {path}") except requests.HTTPError as error: print(f"Failed {error}") diff --git a/Wiki_Infobox_Scraper/main.py b/Wiki_Infobox_Scraper/main.py index 95d021c330..c38d0decef 100644 --- a/Wiki_Infobox_Scraper/main.py +++ b/Wiki_Infobox_Scraper/main.py @@ -1,6 +1,6 @@ from bs4 import BeautifulSoup -import requests from tkinter import * +from security import safe_requests info_dict = {} @@ -31,7 +31,7 @@ def wikiScraper(): entry = entry_str.get() entry = entry.split() query = '_'.join([i.capitalize() for i in entry]) - req = requests.get('https://en.wikipedia.org/wiki/'+query) + req = safe_requests.get('https://en.wikipedia.org/wiki/'+query) # to check for valid URL if req.status_code == 200: diff --git a/Wiki_Scrapper/wiki_Scraper.py b/Wiki_Scrapper/wiki_Scraper.py index 37422ac724..892368340a 100644 --- a/Wiki_Scrapper/wiki_Scraper.py +++ b/Wiki_Scrapper/wiki_Scraper.py @@ -1,6 +1,6 @@ -import requests from bs4 import BeautifulSoup import re +from security import safe_requests # Taking the URL input and validating using Regex URL = input("Enter a valid Wikipedia URL:\n") @@ -11,7 +11,7 @@ exit() # Requesting the HTML and making the BeautifulSoup object -req = requests.get(mo.group()) +req = safe_requests.get(mo.group()) soup = BeautifulSoup(req.text, "lxml") # Validating if the site has content diff --git a/YouTube-Scraper/video.py b/YouTube-Scraper/video.py index fe13f4551e..1629fdd98b 100644 --- a/YouTube-Scraper/video.py +++ b/YouTube-Scraper/video.py @@ -1,6 +1,6 @@ -import requests from bs4 import BeautifulSoup import json +from security import safe_requests class Video: @@ -41,7 +41,7 @@ def getDetails(self): """ url = self.video_url try: - res = requests.get(url) + res = safe_requests.get(url) soup = BeautifulSoup(res.text, "html.parser") video_data = {"video_data": []} diff --git a/Youtube_Scraper/main.py b/Youtube_Scraper/main.py index 3d9421ddc4..87f6b371c7 100644 --- a/Youtube_Scraper/main.py +++ b/Youtube_Scraper/main.py @@ -1,6 +1,6 @@ -import requests from bs4 import BeautifulSoup import json +from security import safe_requests def getAbout(channel_username): @@ -24,7 +24,7 @@ def getAbout(channel_username): """ url = f"https://www.youtube.com/@{channel_username}/about" try: - res = requests.get(url) + res = safe_requests.get(url) soup = BeautifulSoup(res.text, "html.parser") channel_data = {"channel_data": []} link_data = {"link_data": []} diff --git a/extracting-emails-from-website/emails-from-website.py b/extracting-emails-from-website/emails-from-website.py index 0a981fcee2..5eeb4ee3d6 100644 --- a/extracting-emails-from-website/emails-from-website.py +++ b/extracting-emails-from-website/emails-from-website.py @@ -1,8 +1,8 @@ import re -import requests +from security import safe_requests url = input("Enter Url: ") -text = requests.get(url).text +text = safe_requests.get(url).text # Regex query which search for the particular email format. x = re.findall(r"[a-zA-Z]+@{1}[a-zA-Z]+[.]{1}[a-zA-Z]+", text)