Skip to content

Commit

Permalink
Fixes bug 970406 r=peterbe
Browse files Browse the repository at this point in the history
  • Loading branch information
@selenamarie
selenamarie committed Jun 12, 2014
1 parent b068021 commit c69284c
Show file tree
Hide file tree
Showing 2 changed files with 91 additions and 49 deletions.
42 changes: 23 additions & 19 deletions socorro/cron/jobs/fetch_adi_from_hive.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,9 +5,11 @@
import datetime
import urllib2
import csv
import pyhs2
import getpass
import os
import tempfile

import pyhs2

from configman import Namespace
from crontabber.base import BaseCronApp
Expand Down Expand Up @@ -60,6 +62,7 @@
split(request_url,'/')[3]
"""


@as_backfill_cron_app
@with_postgres_transactions()
@with_single_postgres_transaction()
Expand All @@ -76,65 +79,65 @@ class FetchADIFromHiveCronApp(BaseCronApp):
required_config.add_option(
'query',
default=_QUERY,
doc='Explanation of the option')
doc='Hive query for fetching ADI data')

required_config.add_option(
'hive_host',
default='localhost',
doc='Host to run Hive query on')
doc='Hostname to run Hive query on')

required_config.add_option(
'hive_port',
default=10000,
doc='Host to run Hive query on')
doc='Port to run Hive query on')

required_config.add_option(
'hive_user',
default='socorro',
doc='User to connect to hive with')
doc='User to connect to Hive with')

required_config.add_option(
'hive_password',
default='ignored',
doc='Password to connect to hive with')
doc='Password to connect to Hive with')

required_config.add_option(
'hive_database',
default='default',
doc='Database to connect to hive with')
doc='Database name to connect to Hive with')

required_config.add_option(
'temporary_file_system_storage_path',
doc='a local filesystem path where dumps temporarily',
default='/var/tmp/')
'hive_auth_mechanism',
default='PLAIN',
doc='Auth mechanism for Hive')

def run(self, connection, date):
target_date = (date - datetime.timedelta(days=1)).strftime('%Y-%m-%d')

raw_adi_logs_pathname = os.path.join(
self.config.temporary_file_system_storage_path,
tempfile.gettempdir(),
"%s.raw_adi_logs.TEMPORARY%s" % (
target_date,
'.txt'
)
)
try:
with open(raw_adi_logs_pathname, 'w') as f:
hive = pyhs2.connect(host=self.config.hive_host,
hive = pyhs2.connect(
host=self.config.hive_host,
port=self.config.hive_port,
authMechanism='PLAIN',
authMechanism=self.config.hive_auth_mechanism,
user=self.config.hive_user,
password=self.config.hive_password,
database=self.config.hive_database)
database=self.config.hive_database
)

cur = hive.cursor()
query = self.config.query % target_date
cur.execute(query)
for row in cur:
f.write("\t".join(
['None' if v is None else str(v) for v in row]
)
)
f.write("\t".join(str(v) for v in row))
f.write("\n")

with open(raw_adi_logs_pathname, 'r') as f:
pgcursor = connection.cursor()
Expand All @@ -155,4 +158,5 @@ def run(self, connection, date):
]
)
finally:
os.remove(raw_adi_logs_pathname)
if os.path.isfile(raw_adi_logs_pathname):
os.remove(raw_adi_logs_pathname)
98 changes: 68 additions & 30 deletions socorro/unittest/cron/jobs/test_fetch_adi_from_hive.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,12 +18,10 @@ class TestFetchADIFromHive(IntegrationTestCaseBase):

def setUp(self):
super(TestFetchADIFromHive, self).setUp()
self.conn.commit()

def tearDown(self):
cursor = self.conn.cursor()
cursor.execute("TRUNCATE raw_adi_logs")
self.conn.commit()
super(TestFetchADIFromHive, self).tearDown()

def _setup_config_manager(self):
Expand All @@ -36,7 +34,26 @@ def test_mocked_fetch(self):
config_manager = self._setup_config_manager()

def return_test_data(fake):
test_data = [['2014-01-01', 'WinterWolf', 'Ginko', '2.3.1', '10.0.4', 'nightly-ww3v20', 'nightly', 'a-guid', '1']]
test_data = [
['2014-01-01',
'WinterWolf',
'Ginko',
'2.3.1',
'10.0.4',
'nightly-ww3v20',
'nightly',
'a-guid',
'1'],
['2019-01-01',
'NothingMuch',
'Ginko',
'2.3.2',
'10.0.5a',
None,
'release',
'a-guid',
'2']
]
for item in test_data:
yield item

Expand All @@ -52,36 +69,57 @@ def return_test_data(fake):
assert information['fetch-adi-from-hive']

if information['fetch-adi-from-hive']['last_error']:
raise AssertionError(information['fetch-adi-from-hive']['last_error'])

fake_hive.connect.assert_called_with(database='default', authMechanism='PLAIN', host='localhost', user='socorro', password='ignored', port=10000)
fake_hive.connect.cursor.assert_called()
fake_hive.connect.cursor.execute.assert_called()
fake_hive.connect.cursor.__iter__.assert_called()
raise AssertionError(
information['fetch-adi-from-hive']['last_error']
)

fake_hive.connect.assert_called_with(
database='default',
authMechanism='PLAIN',
host='localhost',
user='socorro',
password='ignored',
port=10000
)

pgcursor = self.conn.cursor()
columns = 'report_date',\
'product_name',\
'product_os_platform',\
'product_os_version',\
'product_version',\
'build',\
'build_channel',\
'product_guid',\
columns = (
'report_date',
'product_name',
'product_os_platform',
'product_os_version',
'product_version',
'build',
'build_channel',
'product_guid',
'count'
pgcursor.execute(""" select %s from raw_adi_logs """
% ','.join(columns))
)
pgcursor.execute(
""" select %s from raw_adi_logs """ % ','.join(columns)
)

adi = [dict(zip(columns, row)) for row in pgcursor.fetchall()]

eq_(adi, [{
'report_date': datetime.date(2014, 1, 1),
'product_name': 'WinterWolf',
'product_os_platform': 'Ginko',
'product_os_version': '2.3.1',
'product_version': '10.0.4',
'build': 'nightly-ww3v20',
'build_channel': 'nightly',
'product_guid': 'a-guid',
'count': 1
}])
eq_(adi, [
{
'report_date': datetime.date(2014, 1, 1),
'product_name': 'WinterWolf',
'product_os_platform': 'Ginko',
'product_os_version': '2.3.1',
'product_version': '10.0.4',
'build': 'nightly-ww3v20',
'build_channel': 'nightly',
'product_guid': 'a-guid',
'count': 1
}, {
'report_date': datetime.date(2019, 1, 1),
'product_name': 'NothingMuch',
'product_os_platform': 'Ginko',
'product_os_version': '2.3.2',
'product_version': '10.0.5a',
'build': None,
'build_channel': 'release',
'product_guid': 'a-guid',
'count': 2
}
])

0 comments on commit c69284c

Please sign in to comment.