Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with
or
.
Download ZIP
Browse files

Merge pull request #7 from Mobiperf/master

Merge from main brach
  • Loading branch information...
commit 3689cee93777a7fad0c79bffc037db4c998f0513 2 parents 0d57979 + 7f4c633
@quietbamboo authored
View
34 server/download_dev_data.sh
@@ -0,0 +1,34 @@
+#!/bin/bash
+#
+# Author: gavaletz@google.com (Eric Gavaletz)
+
+# This script downloads data from the Speedometer service running on AppEngine.
+# For more information on options try "appcfg.py --help" or
+# https://developers.google.com/appengine/docs/python/tools/uploadingdata#Downloading_and_Uploading_All_Data
+
+# WARNING: this will produce a very heavy load on the GAE instance and eat up a
+# lot of the applicaion's quota. Please check to see if there is an existing
+# download that you can use before using this.
+
+. ./script_config.sh
+
+DL_LOG_FILE=bulkloader-log-down
+DL_PROGRESS_FILE=bulkloader-progress-down.sql3
+DL_RESULTS_FILE=bulkloader-results-down.sql3
+
+echo "downloading data from $APP_ID.$APP_DOMAIN into $DOWNLOAD_DATA_PATH"
+echo " --log_file=$DATA_PATH/$DL_LOG_FILE"
+echo " --db_filename=$DATA_PATH/$DL_PROGRESS_FILE"
+echo " --result_db_filename=$DATA_PATH/$DL_RESULTS_FILE"
+
+$PYTHON $APPCFG -e $USER_EMAIL -A s~$APP_ID download_data \
+ --num_threads=10 \
+ --batch_size=10 \
+ --bandwidth_limit=1073741824 \
+ --rps_limit=20 \
+ --http_limit=7.5 \
+ --url=http://$APP_ID.$APP_DOMAIN/_ah/remote_api \
+ --log_file=$DATA_PATH/$DL_LOG_FILE \
+ --db_filename=$DATA_PATH/$DL_PROGRESS_FILE \
+ --result_db_filename=$DATA_PATH/$DL_RESULTS_FILE \
+ --filename=$DOWNLOAD_DATA_PATH
View
94 server/gspeedometer/helpers/test.py
@@ -0,0 +1,94 @@
+# Copyright 2012 Google Inc. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+"""Test utilities."""
+
+__author__ = 'mdw@google.com (Matt Welsh)'
+
+import os
+import unittest2
+from google.appengine.api import users
+from google.appengine.ext import testbed
+from gspeedometer import model
+
+
+class MobiperfTest(unittest2.TestCase):
+ """Test utilities."""
+
+ TEST_USER = 'testuser@google.com'
+ TEST_DEVICE_MANUFACTURER = 'fake_manufacturer'
+ TEST_DEVICE_MODEL = 'fake_model'
+ TEST_DEVICE_OS = 'fake_os'
+ TEST_DEVICE_PROPERTIES_APP_VERSION = 'fake_app_version'
+ TEST_DEVICE_PROPERTIES_OS_VERSION = 'fake_os_version'
+
+ def setUp(self):
+ # First, create an instance of the Testbed class.
+ self.testbed = testbed.Testbed()
+ # Then activate the testbed, which prepares the service stubs for use.
+ self.testbed.activate()
+ # Next, declare which service stubs you want to use.
+ self.testbed.init_datastore_v3_stub()
+ self.testbed.init_memcache_stub()
+ self.testbed.init_user_stub()
+ self._SetCurrentUser(MobiperfTest.TEST_USER)
+
+ def tearDown(self):
+ self._LogoutCurrentUser()
+ self.testbed.deactivate()
+
+ def _SetCurrentUser(self, email, user_id='123', is_admin=False):
+ os.environ['USER_EMAIL'] = email or ''
+ os.environ['USER_ID'] = user_id or ''
+ os.environ['USER_IS_ADMIN'] = '1' if is_admin else '0'
+
+ def _LogoutCurrentUser(self):
+ self._SetCurrentUser(None, None)
+
+ def _CreateFakeDevices(self, n=3, extra_info=None):
+ """Create fake devices.
+
+ Args:
+ n: Number of devices to create.
+ extra_info: optional list of dictionaries, where dictionary x
+ will be written to the device properties of fake device x.
+
+ Returns:
+ A list of the created device objects.
+ """
+ devices = []
+ for dev_num in range(n):
+ dev_name = 'fakedevice%d' % dev_num
+
+ new_device = model.DeviceInfo(key_name=dev_name)
+ new_device.id = dev_name
+ new_device.user = users.User(MobiperfTest.TEST_USER)
+ new_device.manufacturer = MobiperfTest.TEST_DEVICE_MANUFACTURER
+ new_device.model = MobiperfTest.TEST_DEVICE_MODEL
+ new_device.os = MobiperfTest.TEST_DEVICE_OS
+ new_device.put()
+
+ properties = model.DeviceProperties()
+ properties.device_info = new_device
+ properties.app_version = MobiperfTest.TEST_DEVICE_PROPERTIES_APP_VERSION
+ properties.os_version = MobiperfTest.TEST_DEVICE_PROPERTIES_OS_VERSION
+
+ if extra_info and extra_info[dev_num]:
+ for k, v in extra_info[dev_num].iteritems():
+ setattr(properties, k, v)
+ properties.put()
+ devices.append(new_device)
+
+ return devices
View
152 server/gspeedometer/model_test.py
@@ -0,0 +1,152 @@
+# Copyright 2012 Google Inc. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+"""Tests for model.py."""
+
+__author__ = 'mdw@google.com (Matt Welsh)'
+
+import datetime
+from gspeedometer import model
+from gspeedometer.helpers import test
+
+
+class ModelTest(test.MobiperfTest):
+ """Tests for model.py."""
+
+ def testDeviceInfoCreate(self):
+ # Test that a DeviceInfo can be created.
+ device = model.DeviceInfo(key_name='mydevice')
+ device.put()
+ self.assertEqual(1, len(model.DeviceInfo().all().fetch(2)))
+ result = model.DeviceInfo().get_by_key_name('mydevice')
+ self.assertEqual(result.key().name(), 'mydevice')
+
+ def testDevicePropertiesCreate(self):
+ # Test that a DeviceProperties can be created.
+ device = self._CreateFakeDevices(n=1)[0]
+ self.assertEqual(1, len(model.DeviceProperties().all().fetch(2)))
+ result = device.last_update()
+ self.assertEqual(result.device_info.key().name(), 'fakedevice0')
+ self.assertEqual(result.device_info.manufacturer,
+ test.MobiperfTest.TEST_DEVICE_MANUFACTURER)
+ self.assertEqual(result.os_version,
+ test.MobiperfTest.TEST_DEVICE_PROPERTIES_OS_VERSION)
+
+ def testDevicePropertiesUpdate(self):
+ # Test that updates to the device properties are counted correctly.
+ device = self._CreateFakeDevices(n=1)[0]
+ self.assertEqual(device.num_updates(), 1)
+ new_properties = model.DeviceProperties()
+ new_properties.device_info = device
+ new_properties.put()
+ self.assertEqual(device.num_updates(), 2)
+ new_properties = model.DeviceProperties()
+ new_properties.device_info = device
+ new_properties.put()
+ self.assertEqual(device.num_updates(), 3)
+
+ def testDeviceLastUpdateTime(self):
+ # Test that DeviceInfo.LastUpdateTime() works as expected.
+ device = self._CreateFakeDevices(n=1)[0]
+ new_properties = model.DeviceProperties()
+ new_properties.device_info = device
+ now = datetime.datetime.utcnow()
+ new_properties.timestamp = now
+ new_properties.put()
+ self.assertEqual(device.LastUpdateTime(), now)
+
+ def testTaskCreate(self):
+ # Test that a Task can be created.
+ task = model.Task(key_name='mytask')
+ task.tag = 'faketag'
+ task.put()
+ self.assertEqual(1, len(model.Task().all().fetch(2)))
+ result = model.Task().get_by_key_name('mytask')
+ self.assertEqual(result.tag, 'faketag')
+
+ def testTaskExpando(self):
+ # Test that Task.GetParam() and Task.Params() work as expected.
+ task = model.Task(key_name='mytask')
+ task.mparam_foo = 'somevalue1'
+ task.mparam_bar = 'somevalue2'
+ task.put()
+
+ self.assertEqual(task.GetParam('foo'), 'somevalue1')
+ self.assertEqual(task.GetParam('bar'), 'somevalue2')
+ params = task.Params()
+ self.assertEqual(params['foo'], 'somevalue1')
+ self.assertEqual(params['bar'], 'somevalue2')
+ self.assertEqual(len(params), 2)
+
+ def testMeasurementCreate(self):
+ # Test that a Measurement can be created.
+ measurement = model.Measurement(key_name='mymeasurement')
+ measurement.type = 'sometype'
+ measurement.put()
+ self.assertEqual(1, len(model.Measurement().all().fetch(2)))
+ result = model.Measurement().get_by_key_name('mymeasurement')
+ self.assertEqual(result.type, 'sometype')
+ self.assertEqual(result.GetTaskID(), None)
+
+ def testMeasurementCreateWithTask(self):
+ # Test that a Measurement can be created with a corresponding Task.
+ task = model.Task()
+ task.tag = 'faketag'
+ task.put()
+ task_id = task.key().id()
+ measurement = model.Measurement(key_name='mymeasurement')
+ measurement.type = 'sometype'
+ measurement.task = task
+ measurement.put()
+ self.assertEqual(1, len(model.Measurement().all().fetch(2)))
+ result = model.Measurement().get_by_key_name('mymeasurement')
+ self.assertEqual(result.type, 'sometype')
+ self.assertEqual(result.task.tag, 'faketag')
+ self.assertEqual(result.GetTaskID(), task_id)
+
+ def testMeasurementExpando(self):
+ # Test that Measurement Expando functions work as expected.
+ measurement = model.Measurement()
+ measurement.mparam_foo = 'somevalue1'
+ measurement.mparam_bar = 'somevalue2'
+ measurement.mval_baz = 'somevalue3'
+ measurement.put()
+
+ self.assertEqual(measurement.GetParam('foo'), 'somevalue1')
+ self.assertEqual(measurement.GetParam('bar'), 'somevalue2')
+ self.assertEqual(measurement.GetValue('baz'), 'somevalue3')
+ params = measurement.Params()
+ self.assertEqual(params['foo'], 'somevalue1')
+ self.assertEqual(params['bar'], 'somevalue2')
+ self.assertEqual(len(params), 2)
+ params = measurement.Values()
+ self.assertEqual(params['baz'], 'somevalue3')
+ self.assertEqual(len(params), 1)
+
+ def testDeviceTaskCreate(self):
+ # Test that a DeviceTask can be created.
+ device = model.DeviceInfo(key_name='mydevice')
+ device.put()
+ task = model.Task()
+ task.tag = 'mytag'
+ task.put()
+ device_task = model.DeviceTask(key_name='mydevicetask')
+ device_task.device_info = device
+ device_task.task = task
+ device_task.put()
+ self.assertEqual(1, len(model.DeviceTask().all().fetch(2)))
+ result = model.DeviceTask().get_by_key_name('mydevicetask')
+ self.assertEqual(result.device_info.key().name(), 'mydevice')
+ self.assertEqual(result.task.tag, 'mytag')
View
24 server/run_server_locally.sh
@@ -1,23 +1,17 @@
#!/bin/sh
#
-# Author: mdw@google.com (Matt Welsh)
+# Author: mdw@google.com (Matt Welsh), gavaletz@google.com (Eric Gavaletz)
# For more information on options try "dev_appserver.py --help" or
# https://developers.google.com/appengine/docs/python/tools/devserver
-PYTHON=python
-APPSERVER=`which dev_appserver.py`
+. ./script_config.sh
-BLOBSTORE_PATH=dev_data
-DATASTORE_PATH=dev_data/dev_appserver.datastore
-
-CLEAN=""
-#TODO(user) if having trouble with the datastore try uncommenting this
-#CLEAN="-c"
-
-./set_version.sh --notag
-
-$PYTHON $APPSERVER $CLEAN -d .
-#TODO(user) use the following line for testing large data operations
-#$PYTHON $APPSERVER $CLEAN -d --backends --blobstore_path=$BLOBSTORE_PATH \
+$PYTHON $APPSERVER $CLEAN $DEBUG $ADDRESS .
+#TODO(user) use the following line(s) for testing large data operations
+#$PYTHON $APPSERVER $CLEAN $DEBUG $ADDRESS\
+# --high_replication \
+# --backends \
+# --use_sqlite \
+# --blobstore_path=$DATA_PATH \
# --datastore_path=$DATASTORE_PATH .
View
58 server/run_tests.py
@@ -0,0 +1,58 @@
+#!/usr/bin/env python2.5
+
+# Copyright 2012 Google Inc. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""This script runs the Mobiperf server unit tests.
+
+Install the Python unittest2 package before you run this script.
+See: http://pypi.python.org/pypi/unittest2
+"""
+
+__author__ = 'mdw@google.com (Matt Welsh)'
+
+import optparse
+import os
+import sys
+import unittest2
+
+
+def main(sdk_path, test_path):
+ # Get the appserver on the path
+ sys.path.insert(0, sdk_path)
+ import dev_appserver
+ dev_appserver.fix_sys_path()
+
+ # Get correct Django version
+ from google.appengine.dist import use_library
+ use_library('django', '1.2')
+
+ suite = unittest2.loader.TestLoader().discover(test_path,
+ pattern='*_test.py')
+ unittest2.TextTestRunner(verbosity=2).run(suite)
+
+
+if __name__ == '__main__':
+ devapp_server_path = None
+ path = os.environ['PATH'].split(':')
+ for directory in path:
+ dev_appserver_path = os.path.join(directory, 'dev_appserver.py')
+ if os.path.exists(dev_appserver_path):
+ dev_appserver_path = os.path.dirname(os.path.realpath(dev_appserver_path))
+ break
+ if not dev_appserver_path:
+ print >>sys.stderr, 'Can\'t find dev_appserver.py on your PATH.'
+ sys.exit(1)
+ print 'Using appserver path ' + dev_appserver_path
+ main(dev_appserver_path, 'gspeedometer')
View
59 server/script_config.sh
@@ -0,0 +1,59 @@
+#!/bin/bash
+#
+# Author: gavaletz@google.com (Eric Gavaletz)
+
+# Because of the growing number of scripts there was a growing number of
+# dependencies and variables being set in lots of different locations. This
+# script is an attempt to get all of those in one place. To use these
+# in any script simply source it as such.
+#
+# . ./script_config.sh
+#
+# Note that this will execute the contents on this script each time that it is
+# sourced so nothing in this file should produce side effects.
+
+
+# NOTE on python versions:
+# Because a lot of systems have a default python version newer than 2.5 it is
+# necessary to be more precise about which python we want to use for running
+# the dev_appserver. If you do not use 2.5, then it will give you warnings but
+# more importantly it is possible that you could include something that works
+# with the newer version of python that you use locally and it will break your
+# code once it is uploaded to the development GAE instance.
+
+PYTHON=python2.5
+APPCFG=`which appcfg.py`
+APPSERVER=`which dev_appserver.py`
+
+
+APP_ID=`cat app.yaml.tmpl | grep application | cut -d " " -f 2`
+APP_DOMAIN=appspot.com
+VERSION=`./set_version.sh`
+
+
+USER_EMAIL=`git config user.email`
+
+
+# These are used for downloading data from the production environment, and using
+# it with the dev_appserver. See download_dev_data.sh and upload_local_data.sh
+# for more information.
+
+DATA_PATH=dev_data
+DOWNLOAD_DATA_PATH=$DATA_PATH/datastore.sql3
+DATASTORE_PATH=$DATA_PATH/dev_appserver.datastore
+
+
+# These are used for running the local dev_appserver. These will not effect the
+# application as deployed on GAE.
+
+DEBUG=""
+#TODO(user) If having trouble try using this.
+#DEBUG="-d"
+
+CLEAN=""
+#TODO(user) If having trouble with the datastore try uncommenting this.
+#CLEAN="-c"
+
+ADDRESS=""
+#TODO(user) If you need to bind to a local address uncomment and adjust this.
+#ADDRESS="--address=192.168.1.128"
View
9 server/update_dev_server.sh
@@ -6,14 +6,7 @@
# For more information on options try "appcfg.py --help" or
# https://developers.google.com/appengine/docs/python/tools/uploadinganapp
-PYTHON=python
-APPCFG=`which appcfg.py`
-
-VERSION=`./set_version.sh`
-
-APP_ID=openmobiledata
-USER_EMAIL=`git config user.email`
-APP_DOMAIN=appspot.com
+. ./script_config.sh
$PYTHON $APPCFG -e $USER_EMAIL -A $APP_ID update .
View
28 server/upload_local_data.sh
@@ -0,0 +1,28 @@
+#!/bin/bash
+#
+# Author: gavaletz@google.com (Eric Gavaletz)
+
+# This script loads data from the Speedometer service running on AppEngine to
+# dev_appserver. For more information on options try "appcfg.py --help" or
+# https://developers.google.com/appengine/docs/python/tools/uploadingdata#Downloading_and_Uploading_All_Data
+
+# To use this you should either have obtained an sql3 file from someone that
+# has already run download_data.sh or run it yourself. Make sure that the
+# dev_appserver is running with the datastore development options selected see
+# run_server_locally.sh for details. This is not intended to update data on
+# the GAE instance (really you shouldn't be doing that).
+
+. ./script_config.sh
+
+UL_LOG_FILE=bulkloader-log-up
+UL_PROGRESS_FILE=bulkloader-progress-up.sql3
+
+echo "uploading data from $DOWNLOAD_DATA_PATH into the dev_appserver"
+echo " --log_file=$DATA_PATH/$UL_LOG_FILE"
+echo " --db_filename=$DATA_PATH/$UL_PROGRESS_FILE"
+
+$PYTHON $APPCFG -e $USER_EMAIL -A dev~$APP_ID upload_data \
+ --url=http://localhost:8080/_ah/remote_api \
+ --log_file=$DATA_PATH/$UL_LOG_FILE \
+ --db_filename=$DATA_PATH/$UL_PROGRESS_FILE \
+ --filename=$DOWNLOAD_DATA_PATH
Please sign in to comment.
Something went wrong with that request. Please try again.