Skip to content

Commit

Permalink
[github-actions][s]: Set automated workflow
Browse files Browse the repository at this point in the history
  * Switch from Travis CI to GitHub Actions (like other datasets).
  * Fix issue with wrong attribute (`string` -> `text`).
  * Reformat script with Black.
  * Change relative paths to work from root directory.
  • Loading branch information
sglavoie committed May 25, 2020
1 parent ea9690b commit 557fe42
Show file tree
Hide file tree
Showing 3 changed files with 71 additions and 53 deletions.
47 changes: 47 additions & 0 deletions .github/workflows/actions.yml
@@ -0,0 +1,47 @@
on:
schedule:
- cron: '0 1 * * *'
push:
branches:
- master

jobs:
update:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@master
- name: Build the data and create local changes
uses: actions/setup-python@v1
with:
python-version: '3.6.10'
architecture: x64
- name: Install requirements
run: |
pip install -r scripts/requirements.txt
- name: Process Data
run: |
python scripts/constituents.py
- name: Commit files
run: |
git config --local user.email "action@github.com"
git config --local user.name "GitHub Action"
git diff --quiet && git diff --staged --quiet || git commit -a -m "Auto-update of the data packages"
- name: Push changes
uses: ad-m/github-push-action@master
with:
github_token: ${{ secrets.gh }}
deploy:
needs: update
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v1
- uses: actions/setup-node@v1
with:
node-version: '8.x'
- run: npm install -g data-cli
- run: data --version
- run: data push
env:
id: ${{secrets.dhid}}
username: ${{secrets.dhusername}}
token: ${{secrets.dhtoken}}
34 changes: 0 additions & 34 deletions .travis.yml

This file was deleted.

43 changes: 24 additions & 19 deletions scripts/constituents.py
Expand Up @@ -5,59 +5,64 @@
import urllib.request as request


datadir = join('..', 'data')
datadir = "data"
if not exists(datadir):
mkdir(datadir)

if not exists('tmp'):
mkdir('tmp')
if not exists("scripts/tmp"):
mkdir("scripts/tmp")

source = "https://en.wikipedia.org/wiki/List_of_S%26P_500_companies"
cache = join('tmp', 'List_of_S%26P_500_companies.html')
cache = join("scripts/tmp", "List_of_S%26P_500_companies.html")


def retrieve():
request.urlretrieve(source, cache)


def extract():
source_page = open(cache).read()
soup = BeautifulSoup(source_page, 'html.parser')
table = soup.find("table", { "class" : "wikitable sortable" })
soup = BeautifulSoup(source_page, "html.parser")
table = soup.find("table", {"class": "wikitable sortable"})

# Fail now if we haven't found the right table
header = table.findAll('th')
header = table.findAll("th")
if header[0].text.rstrip() != "Symbol" or header[1].string != "Security":
raise Exception("Can't parse wikipedia's table!")
raise Exception("Can't parse Wikipedia's table!")

# Retreive the values in the table
# Retrieve the values in the table
records = []
symbols = []
rows = table.findAll('tr')
rows = table.findAll("tr")
for row in rows:
fields = row.findAll('td')
fields = row.findAll("td")
if fields:
symbol = fields[0].text.rstrip()
# fix as now they have links to the companies on WP
name = fields[1].string.replace(',', '')
sector = fields[3].string.rstrip()
name = fields[1].text.replace(",", "")
sector = fields[3].text.rstrip()
records.append([symbol, name, sector])
symbols.append(symbol + '\n')
symbols.append(symbol + "\n")

header = ['Symbol', 'Name', 'Sector']
writer = csv.writer(open('../data/constituents.csv', 'w'), lineterminator='\n')
header = ["Symbol", "Name", "Sector"]
writer = csv.writer(
open("./data/constituents.csv", "w"), lineterminator="\n"
)
writer.writerow(header)
# Sorting ensure easy tracking of modifications
records.sort(key=lambda s: s[1].lower())
writer.writerows(records)

with open('../data/constituents_symbols.txt', 'w') as f:
with open("./data/constituents_symbols.txt", "w") as f:
# Sorting ensure easy tracking of modifications
symbols.sort(key=lambda s: s[0].lower())
f.writelines(symbols)


def process():
retrieve()
extract()

if __name__ == '__main__':
process()

if __name__ == "__main__":
process()

0 comments on commit 557fe42

Please sign in to comment.