-
Notifications
You must be signed in to change notification settings - Fork 7
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
- Loading branch information
Showing
12 changed files
with
368 additions
and
154 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,28 @@ | ||
| #!/usr/bin/env python | ||
|
|
||
| import app_config | ||
| import oauth | ||
| from flask import Flask, make_response, render_template | ||
| from werkzeug.debug import DebuggedApplication | ||
|
|
||
| app = Flask(__name__) | ||
| app.debug = app_config.DEBUG | ||
|
|
||
| @app.route('/') | ||
| @oauth.oauth_required | ||
| def index(): | ||
| return make_response("You're good to go.") | ||
|
|
||
| app.register_blueprint(oauth.oauth) | ||
|
|
||
|
|
||
| # Enable Werkzeug debug pages | ||
| if app_config.DEBUG: | ||
| wsgi_app = DebuggedApplication(app, evalex=False) | ||
| else: | ||
| wsgi_app = app | ||
|
|
||
|
|
||
| # Catch attempts to run the app directly | ||
| if __name__ == '__main__': | ||
| print 'This command has been removed! Please run "fab app" instead!' |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,155 +1,58 @@ | ||
| # Query constructor: https://ga-dev-tools.appspot.com/query-explorer/ | ||
| # | ||
| import app_config | ||
|
|
||
| """ | ||
| Before you begin, you must sigup for a new project in the Google APIs console: | ||
| https://code.google.com/apis/console | ||
| from datetime import datetime | ||
| from oauth import get_credentials | ||
|
|
||
| Then register the project to use OAuth2.0 for installed applications. | ||
| import logging | ||
|
|
||
| Finally you will need to add the client id, client secret, and redirect URL | ||
| into the client_secrets.json file that is in the same directory as this sample. | ||
| logging.basicConfig() | ||
| logger = logging.getLogger(__name__) | ||
| logger.setLevel(logging.INFO) | ||
|
|
||
| Sample Usage: | ||
| class GoogleAnalyticsScraper: | ||
| def __init__(self): | ||
| self.run_time = datetime.utcnow() | ||
|
|
||
| $ python analytics.py | ||
| """ | ||
| from __future__ import print_function | ||
| def scrape_google_analytics(self): | ||
| rows = [] | ||
|
|
||
| import argparse | ||
| import sys | ||
| api_url = 'https://www.googleapis.com/analytics/v3/data/ga' | ||
| credentials = get_credentials() | ||
|
|
||
| from googleapiclient.errors import HttpError | ||
| from googleapiclient import sample_tools | ||
| from oauth2client.client import AccessTokenRefreshError | ||
| metrics = ','.join(['ga:{0}'.format(metric) for metric in app_config.GA_METRICS]) | ||
| dimensions = ','.join(['ga:{0}'.format(dimensions) for dimensions in app_config.GA_DIMENSIONS]) | ||
|
|
||
| class Analytics: | ||
| def __init__(self): | ||
| # Authenticate and construct service. | ||
| argv = [] | ||
| self.service, flags = sample_tools.init( | ||
| argv, 'analytics', 'v3', __doc__, __file__, | ||
| scope='https://www.googleapis.com/auth/analytics.readonly') | ||
| params = { | ||
| 'ids': 'ga:{0}'.format(app_config.GA_ORGANIZATION_ID), | ||
| 'end-date': 'yesterday', | ||
| 'start-date': '30daysAgo', # start_date.strftime('%Y-%m-%d'), | ||
| 'metrics': 'ga:sessions,ga:pageviews', | ||
| 'dimensions': 'ga:pagePath,ga:source,ga:deviceCategory', | ||
| 'max-results': app_config.GA_RESULT_SIZE, | ||
| 'samplingLevel': app_config.GA_SAMPLING_LEVEL, | ||
| 'start-index': 1, | ||
| } | ||
|
|
||
| def autodiscover_stories(self,): | ||
| self.results = self.service.data().ga().get( | ||
| ids='ga:100688391', | ||
| start_date='5daysAgo', | ||
| end_date='today', | ||
| metrics='ga:totalEvents', | ||
| dimensions='ga:eventCategory', | ||
| filters='ga:eventAction==on-screen', | ||
| start_index='1', | ||
| max_results='500').execute() | ||
| while True: | ||
| resp = app_config.authomatic.access(credentials, api_url, params=params) | ||
| data = resp.data | ||
|
|
||
| if self.results.get('rows', []): | ||
| data = [] | ||
| for row in self.results.get('rows'): | ||
| print(row[0]) | ||
| logger.info('Processing rows {0} - {1}'.format(params['start-index'], params['start-index'] + app_config.GA_RESULT_SIZE - 1)) | ||
|
|
||
| def donation_data(self, slug): | ||
| self.results = self.service.data().ga().get( | ||
| ids='ga:100688391', | ||
| start_date='90daysAgo', | ||
| end_date='today', | ||
| metrics='ga:totalEvents', | ||
| # dimensions='ga:date', | ||
| sort='-ga:totalEvents', | ||
| filters='ga:eventCategory==%s;ga:eventLabel==donate' % slug, | ||
| start_index='1', | ||
| max_results='25').execute() | ||
| if not data.get('rows'): | ||
| logger.info('No rows found, done.') | ||
| break | ||
|
|
||
| return self.results | ||
| # ga:eventCategory==carebot | ||
| # ga:eventLabel==10m | ||
| # dimensions: eventCategory, eventLabel, eventAction | ||
| for row in resp.data['rows']: | ||
| analytics_row = GoogleAnalyticsRow(row, app_config.GA_METRICS, app_config.GA_DIMENSIONS, data) | ||
| rows.append(analytics_row.serialize()) | ||
|
|
||
| def get_linger_rate(self, slug): | ||
| self.results = self.service.data().ga().get( | ||
| ids='ga:100688391', #'ga:' + profile_id, | ||
| start_date='90daysAgo', | ||
| end_date='today', | ||
| metrics='ga:totalEvents', | ||
| dimensions='ga:eventLabel', | ||
| sort='-ga:totalEvents', | ||
| filters='ga:eventCategory==%s;ga:eventAction==on-screen;ga:eventLabel==10s,ga:eventLabel==20s,ga:eventLabel==30s,ga:eventLabel==40s,ga:eventLabel==50s,ga:eventLabel==1m,ga:eventLabel==2m,ga:eventLabel==3m,ga:eventLabel==4m,ga:eventLabel==5m,ga:eventLabel==10m' % slug, | ||
| start_index='1', | ||
| max_results='25').execute() | ||
| params['start-index'] += app_config.GA_RESULT_SIZE | ||
|
|
||
| if self.results.get('rows', []): | ||
| data = [] | ||
| #import ipdb; ipdb.set_trace(); | ||
| return rows | ||
|
|
||
| for row in self.results.get('rows'): | ||
| time = row[0] | ||
| seconds = 0 | ||
| if 'm' in time: | ||
| time = time[:-1] # remove 'm' from the end | ||
| seconds = int(time) * 60 | ||
| else: | ||
| time = time[:-1] # remove 's' | ||
| seconds = int(time) | ||
|
|
||
| row[0] = seconds | ||
| row[1] = int(row[1]) | ||
| data.append(row) | ||
|
|
||
| # Calculate the number of visitors in each bucket | ||
| for index, row in enumerate(data): | ||
| if index == len(data) - 1: | ||
| continue | ||
|
|
||
| next_row = data[index + 1] | ||
| row[1] = row[1] - next_row[-1] | ||
|
|
||
| # Exclude everybody in the last bucket | ||
| # (they've been lingering for way too long -- 10+ minutes) | ||
| data = data [:-1] | ||
|
|
||
| # Get the average number of seconds | ||
| total_seconds = 0 | ||
| total_people = 0 | ||
| for row in data: | ||
| total_seconds = total_seconds + (row[0] * row[1]) | ||
| total_people = total_people + row[1] | ||
|
|
||
| average_seconds = total_seconds/total_people | ||
| minutes = average_seconds / 60 | ||
| seconds = average_seconds % 60 | ||
| return (total_people, minutes, seconds) | ||
|
|
||
| def print_results(self): | ||
| print() | ||
| print('Profile Name: %s' % self.results.get('profileInfo').get('profileName')) | ||
| print() | ||
|
|
||
| # Print header. | ||
| output = [] | ||
| for header in self.results.get('columnHeaders'): | ||
| output.append('%30s' % header.get('name')) | ||
| print(''.join(output)) | ||
|
|
||
| # Print data table. | ||
| if self.results.get('rows', []): | ||
| for row in self.results.get('rows'): | ||
| output = [] | ||
| for cell in row: | ||
| output.append('%30s' % cell) | ||
| print(''.join(output)) | ||
|
|
||
| else: | ||
| print('No Rows Found') | ||
|
|
||
| # Testing -- move to a separate folder | ||
| # slugs: elections16 | ||
| a = Analytics() | ||
| a.autodiscover_stories() | ||
|
|
||
| # data = a.get_linger_rate('space-time-stepper-20160208') | ||
| # a.print_results() | ||
|
|
||
| # if data.get('rows', []): | ||
| # row = data.get('rows')[0] | ||
| # cell = row[0] | ||
| # print(cell) | ||
|
|
||
| # a.print_results() | ||
| # def write(self, db, rows): | ||
| # table = db['google_analytics'] | ||
| # table.delete() | ||
| # table.insert_many(rows) |
Oops, something went wrong.