-
Notifications
You must be signed in to change notification settings - Fork 376
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
created scripts/ (will include all the scripts soon) & updated search…
… country filters
- Loading branch information
alvarobartt
committed
May 7, 2021
1 parent
d4f2948
commit 26a61ca
Showing
2 changed files
with
301 additions
and
36 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,263 @@ | ||
| { | ||
| "cells": [ | ||
| { | ||
| "cell_type": "code", | ||
| "execution_count": 103, | ||
| "id": "ec54efbc-7c6b-44a7-90af-dc5a4003ced2", | ||
| "metadata": {}, | ||
| "outputs": [], | ||
| "source": [ | ||
| "from investpy.utils.extra import random_user_agent" | ||
| ] | ||
| }, | ||
| { | ||
| "cell_type": "code", | ||
| "execution_count": 104, | ||
| "id": "ff8a0400-0322-475b-b5d5-a3186aa0831c", | ||
| "metadata": {}, | ||
| "outputs": [], | ||
| "source": [ | ||
| "params = {\n", | ||
| " 'search_text': \"apple\",\n", | ||
| " 'tab': 'quotes',\n", | ||
| " 'isFilter': True,\n", | ||
| " 'limit': 1,\n", | ||
| " 'offset': 0\n", | ||
| "}\n", | ||
| "\n", | ||
| "headers = {\n", | ||
| " \"User-Agent\": random_user_agent(),\n", | ||
| " \"X-Requested-With\": \"XMLHttpRequest\",\n", | ||
| " \"Accept\": \"text/html\",\n", | ||
| " \"Accept-Encoding\": \"gzip, deflate\",\n", | ||
| " \"Connection\": \"keep-alive\",\n", | ||
| "}\n", | ||
| "\n", | ||
| "url = 'https://www.investing.com/search/service/SearchInnerPage'" | ||
| ] | ||
| }, | ||
| { | ||
| "cell_type": "code", | ||
| "execution_count": 105, | ||
| "id": "6978b497-6ad8-44cc-ad88-c878f4120644", | ||
| "metadata": {}, | ||
| "outputs": [], | ||
| "source": [ | ||
| "import requests" | ||
| ] | ||
| }, | ||
| { | ||
| "cell_type": "code", | ||
| "execution_count": 106, | ||
| "id": "eb4618a8-0052-4bf4-8ab0-12271e889910", | ||
| "metadata": {}, | ||
| "outputs": [], | ||
| "source": [ | ||
| "req = requests.post(url, headers=headers, data=params)" | ||
| ] | ||
| }, | ||
| { | ||
| "cell_type": "code", | ||
| "execution_count": 107, | ||
| "id": "34cf663f-f1fe-49d1-ab2b-9fe3602bd2aa", | ||
| "metadata": {}, | ||
| "outputs": [ | ||
| { | ||
| "data": { | ||
| "text/plain": [ | ||
| "200" | ||
| ] | ||
| }, | ||
| "execution_count": 107, | ||
| "metadata": {}, | ||
| "output_type": "execute_result" | ||
| } | ||
| ], | ||
| "source": [ | ||
| "req.status_code" | ||
| ] | ||
| }, | ||
| { | ||
| "cell_type": "code", | ||
| "execution_count": 108, | ||
| "id": "e89e8b74-3c2f-4391-b44e-20e80c13c483", | ||
| "metadata": {}, | ||
| "outputs": [], | ||
| "source": [ | ||
| "import pandas as pd" | ||
| ] | ||
| }, | ||
| { | ||
| "cell_type": "code", | ||
| "execution_count": 116, | ||
| "id": "bc950779-b0b6-4b5a-b090-dc78ccd52cf3", | ||
| "metadata": {}, | ||
| "outputs": [], | ||
| "source": [ | ||
| "replacements = {\n", | ||
| " 'bosnia-herzegovina': 'bosnia',\n", | ||
| " \"cote d'ivoire\": 'ivory coast',\n", | ||
| " 'palestinian territory': 'palestine',\n", | ||
| " 'united arab emirates': 'dubai'\n", | ||
| "}" | ||
| ] | ||
| }, | ||
| { | ||
| "cell_type": "code", | ||
| "execution_count": 110, | ||
| "id": "56d032f5-434e-480b-a19c-8c8beca280c4", | ||
| "metadata": {}, | ||
| "outputs": [], | ||
| "source": [ | ||
| "country2flag = dict()\n", | ||
| "flag2country = dict()\n", | ||
| "\n", | ||
| "for country in req.json()['filters']['country']:\n", | ||
| " if country['country_ID'] == -1: continue\n", | ||
| " country_name = country['country_name_translated'].lower()\n", | ||
| " if country_name in replacements: country_name = replacements[country_name]\n", | ||
| " country2flag[country_name] = country['flag']\n", | ||
| " flag2country[country['flag']] = country_name" | ||
| ] | ||
| }, | ||
| { | ||
| "cell_type": "code", | ||
| "execution_count": 112, | ||
| "id": "3b9bf68d-7c06-4521-9782-761ee9464bfb", | ||
| "metadata": {}, | ||
| "outputs": [], | ||
| "source": [ | ||
| "n_steps = 4" | ||
| ] | ||
| }, | ||
| { | ||
| "cell_type": "code", | ||
| "execution_count": 111, | ||
| "id": "0b6bf7e4-6abe-419b-89b7-9b5095767068", | ||
| "metadata": {}, | ||
| "outputs": [], | ||
| "source": [ | ||
| "country2flag = list(country2flag.items())" | ||
| ] | ||
| }, | ||
| { | ||
| "cell_type": "code", | ||
| "execution_count": 113, | ||
| "id": "4a389f2f-84d0-4ed8-8328-dfe6e6729ffd", | ||
| "metadata": {}, | ||
| "outputs": [ | ||
| { | ||
| "name": "stdout", | ||
| "output_type": "stream", | ||
| "text": [ | ||
| "'andorra': 'Andorra', 'argentina': 'Argentina', 'australia': 'Australia', 'austria': 'Austria', \n", | ||
| "'bahrain': 'Bahrain', 'bangladesh': 'Bangladesh', 'belgium': 'Belgium', 'bermuda': 'Bermuda', \n", | ||
| "'bosnia': 'Bosnia', 'botswana': 'Botswana', 'brazil': 'Brazil', 'bulgaria': 'Bulgaria', \n", | ||
| "'canada': 'Canada', 'cayman islands': 'Cayman_Islands', 'chile': 'Chile', 'china': 'China', \n", | ||
| "'colombia': 'Colombia', 'costa rica': 'Costa_Rica', 'ivory coast': 'Cote_dIvoire', 'croatia': 'Croatia', \n", | ||
| "'cyprus': 'Cyprus', 'czech republic': 'Czech_Republic', 'denmark': 'Denmark', 'ecuador': 'Ecuador', \n", | ||
| "'egypt': 'Egypt', 'estonia': 'Estonia', 'euro zone': 'Europe', 'finland': 'Finland', \n", | ||
| "'france': 'France', 'germany': 'Germany', 'gibraltar': 'Gibraltar', 'greece': 'Greece', \n", | ||
| "'hong kong': 'Hong_Kong', 'hungary': 'Hungary', 'iceland': 'Iceland', 'india': 'India', \n", | ||
| "'indonesia': 'Indonesia', 'iraq': 'Iraq', 'ireland': 'Ireland', 'israel': 'Israel', \n", | ||
| "'italy': 'Italy', 'jamaica': 'Jamaica', 'japan': 'Japan', 'jordan': 'Jordan', \n", | ||
| "'kazakhstan': 'Kazakhstan', 'kenya': 'Kenya', 'kuwait': 'Kuwait', 'latvia': 'Latvia', \n", | ||
| "'lebanon': 'Lebanon', 'liechtenstein': 'Liechtenstein', 'lithuania': 'Lithuania', 'luxembourg': 'Luxembourg', \n", | ||
| "'malawi': 'Malawi', 'malaysia': 'Malaysia', 'malta': 'Malta', 'mauritius': 'Mauritius', \n", | ||
| "'mexico': 'Mexico', 'monaco': 'Monaco', 'mongolia': 'Mongolia', 'montenegro': 'Montenegro', \n", | ||
| "'morocco': 'Morocco', 'namibia': 'Namibia', 'netherlands': 'Netherlands', 'new zealand': 'New_Zealand', \n", | ||
| "'nigeria': 'Nigeria', 'norway': 'Norway', 'oman': 'Oman', 'pakistan': 'Pakistan', \n", | ||
| "'palestine': 'Palestine', 'peru': 'Peru', 'philippines': 'Philippines', 'poland': 'Poland', \n", | ||
| "'portugal': 'Portugal', 'qatar': 'Qatar', 'romania': 'Romania', 'russia': 'Russian_Federation', \n", | ||
| "'rwanda': 'Rwanda', 'saudi arabia': 'Saudi_Arabia', 'serbia': 'Serbia', 'singapore': 'Singapore', \n", | ||
| "'slovakia': 'Slovakia', 'slovenia': 'Slovenia', 'south africa': 'South_Africa', 'south korea': 'South_Korea', \n", | ||
| "'spain': 'Spain', 'sri lanka': 'Sri_Lanka', 'sweden': 'Sweden', 'switzerland': 'Switzerland', \n", | ||
| "'taiwan': 'Taiwan', 'tanzania': 'Tanzania', 'thailand': 'Thailand', 'tunisia': 'Tunisia', \n", | ||
| "'turkey': 'Turkey', 'uganda': 'Uganda', 'ukraine': 'Ukraine', 'united arab emirates': 'Dubai', \n", | ||
| "'united kingdom': 'UK', 'united states': 'USA', 'venezuela': 'Venezuela', 'vietnam': 'Vietnam', \n", | ||
| "'zambia': 'Zambia', 'zimbabwe': 'Zimbabwe', \n" | ||
| ] | ||
| } | ||
| ], | ||
| "source": [ | ||
| "for idx in range(0, len(country2flag), n_steps):\n", | ||
| " print(', '.join([f\"'{k}': '{v}'\" for k, v in country2flag[idx:idx+n_steps]]) + ', ')" | ||
| ] | ||
| }, | ||
| { | ||
| "cell_type": "code", | ||
| "execution_count": 114, | ||
| "id": "1d0bfd67-7b80-4a56-91eb-9fcf97b2c393", | ||
| "metadata": {}, | ||
| "outputs": [], | ||
| "source": [ | ||
| "flag2country = list(flag2country.items())" | ||
| ] | ||
| }, | ||
| { | ||
| "cell_type": "code", | ||
| "execution_count": 115, | ||
| "id": "f90c2d27-3fff-4e61-bee2-fe6eeb5311a9", | ||
| "metadata": {}, | ||
| "outputs": [ | ||
| { | ||
| "name": "stdout", | ||
| "output_type": "stream", | ||
| "text": [ | ||
| "'Andorra': 'andorra', 'Argentina': 'argentina', 'Australia': 'australia', 'Austria': 'austria', \n", | ||
| "'Bahrain': 'bahrain', 'Bangladesh': 'bangladesh', 'Belgium': 'belgium', 'Bermuda': 'bermuda', \n", | ||
| "'Bosnia': 'bosnia', 'Botswana': 'botswana', 'Brazil': 'brazil', 'Bulgaria': 'bulgaria', \n", | ||
| "'Canada': 'canada', 'Cayman_Islands': 'cayman islands', 'Chile': 'chile', 'China': 'china', \n", | ||
| "'Colombia': 'colombia', 'Costa_Rica': 'costa rica', 'Cote_dIvoire': 'ivory coast', 'Croatia': 'croatia', \n", | ||
| "'Cyprus': 'cyprus', 'Czech_Republic': 'czech republic', 'Denmark': 'denmark', 'Ecuador': 'ecuador', \n", | ||
| "'Egypt': 'egypt', 'Estonia': 'estonia', 'Europe': 'euro zone', 'Finland': 'finland', \n", | ||
| "'France': 'france', 'Germany': 'germany', 'Gibraltar': 'gibraltar', 'Greece': 'greece', \n", | ||
| "'Hong_Kong': 'hong kong', 'Hungary': 'hungary', 'Iceland': 'iceland', 'India': 'india', \n", | ||
| "'Indonesia': 'indonesia', 'Iraq': 'iraq', 'Ireland': 'ireland', 'Israel': 'israel', \n", | ||
| "'Italy': 'italy', 'Jamaica': 'jamaica', 'Japan': 'japan', 'Jordan': 'jordan', \n", | ||
| "'Kazakhstan': 'kazakhstan', 'Kenya': 'kenya', 'Kuwait': 'kuwait', 'Latvia': 'latvia', \n", | ||
| "'Lebanon': 'lebanon', 'Liechtenstein': 'liechtenstein', 'Lithuania': 'lithuania', 'Luxembourg': 'luxembourg', \n", | ||
| "'Malawi': 'malawi', 'Malaysia': 'malaysia', 'Malta': 'malta', 'Mauritius': 'mauritius', \n", | ||
| "'Mexico': 'mexico', 'Monaco': 'monaco', 'Mongolia': 'mongolia', 'Montenegro': 'montenegro', \n", | ||
| "'Morocco': 'morocco', 'Namibia': 'namibia', 'Netherlands': 'netherlands', 'New_Zealand': 'new zealand', \n", | ||
| "'Nigeria': 'nigeria', 'Norway': 'norway', 'Oman': 'oman', 'Pakistan': 'pakistan', \n", | ||
| "'Palestine': 'palestine', 'Peru': 'peru', 'Philippines': 'philippines', 'Poland': 'poland', \n", | ||
| "'Portugal': 'portugal', 'Qatar': 'qatar', 'Romania': 'romania', 'Russian_Federation': 'russia', \n", | ||
| "'Rwanda': 'rwanda', 'Saudi_Arabia': 'saudi arabia', 'Serbia': 'serbia', 'Singapore': 'singapore', \n", | ||
| "'Slovakia': 'slovakia', 'Slovenia': 'slovenia', 'South_Africa': 'south africa', 'South_Korea': 'south korea', \n", | ||
| "'Spain': 'spain', 'Sri_Lanka': 'sri lanka', 'Sweden': 'sweden', 'Switzerland': 'switzerland', \n", | ||
| "'Taiwan': 'taiwan', 'Tanzania': 'tanzania', 'Thailand': 'thailand', 'Tunisia': 'tunisia', \n", | ||
| "'Turkey': 'turkey', 'Uganda': 'uganda', 'Ukraine': 'ukraine', 'Dubai': 'united arab emirates', \n", | ||
| "'UK': 'united kingdom', 'USA': 'united states', 'Venezuela': 'venezuela', 'Vietnam': 'vietnam', \n", | ||
| "'Zambia': 'zambia', 'Zimbabwe': 'zimbabwe', \n" | ||
| ] | ||
| } | ||
| ], | ||
| "source": [ | ||
| "for idx in range(0, len(flag2country), n_steps):\n", | ||
| " print(', '.join([f\"'{k}': '{v}'\" for k, v in flag2country[idx:idx+n_steps]]) + ', ')" | ||
| ] | ||
| } | ||
| ], | ||
| "metadata": { | ||
| "kernelspec": { | ||
| "display_name": "Python 3", | ||
| "language": "python", | ||
| "name": "python3" | ||
| }, | ||
| "language_info": { | ||
| "codemirror_mode": { | ||
| "name": "ipython", | ||
| "version": 3 | ||
| }, | ||
| "file_extension": ".py", | ||
| "mimetype": "text/x-python", | ||
| "name": "python", | ||
| "nbconvert_exporter": "python", | ||
| "pygments_lexer": "ipython3", | ||
| "version": "3.6.12" | ||
| } | ||
| }, | ||
| "nbformat": 4, | ||
| "nbformat_minor": 5 | ||
| } |