Get latest data #674


	name: Get latest data

	on:
	push:
	workflow_dispatch:
	schedule:
	- cron: '00 18 * * *'

	jobs:
	scheduled:
	runs-on: ubuntu-latest
	steps:
	- name: Check out this repo
	uses: actions/checkout@v2
	with:
	fetch-depth: 0
	- name: Set up Python
	uses: actions/setup-python@v2
	with:
	python-version: 3.8
	- uses: actions/cache@v2
	name: Configure pip caching
	with:
	path: ~/.cache/pip
	key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }}
	restore-keys: \|
	${{ runner.os }}-pip-
	- name: Install dependencies
	run: \|-
	# Pip
	pip install -r requirements.txt
	# csvdiff
	curl -sfL https://raw.githubusercontent.com/aswinkarthik/csvdiff/master/install.sh \| sh -s
	- name: Fetch latest data
	run: \|-
	# Take all the existing state files and make one big file to compare to
	csvstack *.csv > bmf-old.csv
	# Delete all the old files now.
	rm eo_*.csv
	# Get all of the state files
	wget -i url_list.txt --quiet
	# Take all of the new state files and make another big file
	csvstack *.csv > bmf.csv
	# Diff the changes and generate commit message using csv-diff. csvdiff outputs summaries to stderr.
	./bin/csvdiff bmf-old.csv bmf.csv 2> message.txt
	# Put the commit message on one line
	sed -z 's/\n#/ \\|/g' -i message.txt
	# Output the message
	echo "-------"
	cat message.txt
	- name: Commit and push if it changed
	run: \|-
	git config user.name "Automated"
	git config user.email "actions@users.noreply.github.com"
	git add eo*.csv
	timestamp=$(date -u)
	git commit -F message.txt \|\| exit 0
	git push

Provide feedback