Merge branch 'master' of github.com:AppScale/appscale into simpler-da…

…tastore-viewer-start

AppController/djinn.rb

@@ -548,7 +548,8 @@ class Djinn
 
 
   # Template used for rsyslog configuration files.
-  RSYSLOG_TEMPLATE_LOCATION = "#{APPSCALE_HOME}/lib/templates/rsyslog-app.conf"
+  RSYSLOG_TEMPLATE_LOCATION = "#{APPSCALE_HOME}/common/appscale/common/" +
+    "templates/rsyslog-app.conf"
 
 
   # Instance variables that we need to restore from the head node.
@@ -4656,7 +4657,8 @@ def initialize_server()
   # This method is called only when the appengine role does not run
   # on the head node.
   def write_app_logrotate()
-    template_dir = File.join(File.dirname(__FILE__), "../lib/templates")
+    template_dir = File.join(File.dirname(__FILE__),
+                             "../common/appscale/common/templates")
     FileUtils.cp("#{template_dir}/#{APPSCALE_APP_LOGROTATE}",
       "#{LOGROTATE_DIR}/appscale-app")
   end

AppController/lib/haproxy.rb

@@ -89,6 +89,10 @@ module HAProxy
   HAPROXY_ERROR_PREFIX = "No such"
 
 
+  # The number of seconds HAProxy should wait for a server response.
+  HAPROXY_SERVER_TIMEOUT = 600
+
+
   def self.start()
     start_cmd = "/usr/sbin/service haproxy start"
     stop_cmd = "/usr/sbin/service haproxy stop"
@@ -314,14 +318,6 @@ def self.initialize_config()
   # Log details about HTTP requests
   #option httplog
 
-  # Abort request if client closes its output channel while waiting for the
-  # request. HAProxy documentation has a long explanation for this option.
-  option abortonclose
-
-  # Check if a "Connection: close" header is already set in each direction,
-  # and will add one if missing.
-  option httpclose
-
   # If sending a request fails, try to send it to another, 3 times
   # before aborting the request
   retries 3
@@ -330,15 +326,14 @@ def self.initialize_config()
   # any Mongrel, not just the one that started the session
   option redispatch
 
-  # Timeout a request if the client did not read any data for 600 seconds
-  timeout client 600000
+  # Time to wait for a connection attempt to a server.
+  timeout connect 5000ms
 
-  # Timeout a request if Mongrel does not accept a connection for 600 seconds
-  timeout connect 600000
+  # The maximum inactivity time allowed for a client.
+  timeout client 50000ms
 
-  # Timeout a request if Mongrel does not accept the data on the connection,
-  # or does not send a response back in 10 minutes.
-  timeout server 600000
+  # The maximum inactivity time allowed for a server.
+  timeout server #{HAPROXY_SERVER_TIMEOUT}s
 
   # Enable the statistics page
   stats enable

AppDB/appscale/datastore/backup/backup_recovery_helper.py

@@ -13,19 +13,18 @@
 
 import gcs_helper
 
+from appscale.common import appscale_info
+from appscale.common.constants import APPSCALE_DATA_DIR
+from appscale.common.unpackaged import APPSCALE_PYTHON_APPSERVER
 from appscale.datastore.backup import backup_exceptions
 from appscale.datastore.backup.br_constants import APP_BACKUP_DIR_LOCATION
 from appscale.datastore.backup.br_constants import APP_DIR_LOCATION
 from appscale.datastore.backup.br_constants import BACKUP_DIR_LOCATION
 from appscale.datastore.backup.br_constants import BACKUP_ROLLBACK_SUFFIX
 from appscale.datastore.backup.br_constants import PADDING_PERCENTAGE
 from appscale.datastore.backup.br_constants import StorageTypes
-from appscale.datastore.unpackaged import APPSCALE_LIB_DIR
-
-sys.path.append(APPSCALE_LIB_DIR)
-import appscale_info
-from constants import APPSCALE_DATA_DIR
 
+sys.path.append(APPSCALE_PYTHON_APPSERVER)
 from google.appengine.api.appcontroller_client import AppControllerClient
 
 # The port that the SOAP server listens to.

AppDB/appscale/datastore/backup/cassandra_backup.py

@@ -7,6 +7,10 @@
 import sys
 import time
 
+from appscale.common import appscale_info
+from appscale.common import monit_interface
+from appscale.common.constants import APPSCALE_DATA_DIR
+from appscale.common.unpackaged import INFRASTRUCTURE_MANAGER_DIR
 from subprocess import CalledProcessError
 from . import backup_recovery_helper
 from .backup_exceptions import BRException
@@ -16,13 +20,6 @@
 from ..cassandra_env import cassandra_interface
 from ..cassandra_env.cassandra_interface import NODE_TOOL
 from ..cassandra_env.cassandra_interface import CASSANDRA_MONIT_WATCH_NAME
-from ..unpackaged import APPSCALE_LIB_DIR
-from ..unpackaged import INFRASTRUCTURE_MANAGER_DIR
-
-sys.path.append(APPSCALE_LIB_DIR)
-import appscale_info
-import monit_interface
-from constants import APPSCALE_DATA_DIR
 
 sys.path.append(INFRASTRUCTURE_MANAGER_DIR)
 from utils import utils

AppDB/appscale/datastore/cassandra_env/cassandra_interface.py

@@ -11,6 +11,9 @@
 import time
 import uuid
 
+from appscale.common import appscale_info
+from appscale.common.constants import SCHEMA_CHANGE_TIMEOUT
+from appscale.common.unpackaged import APPSCALE_PYTHON_APPSERVER
 from cassandra.cluster import Cluster
 from cassandra.concurrent import execute_concurrent
 from cassandra.query import BatchStatement
@@ -27,8 +30,6 @@
 from ..dbconstants import Operations
 from ..dbconstants import TxnActions
 from ..dbinterface import AppDBInterface
-from ..unpackaged import APPSCALE_LIB_DIR
-from ..unpackaged import APPSCALE_PYTHON_APPSERVER
 from ..utils import clean_app_id
 from ..utils import create_key
 from ..utils import encode_index_pb
@@ -41,10 +42,6 @@
 from ..utils import get_write_time
 from ..utils import tx_partition
 
-sys.path.append(APPSCALE_LIB_DIR)
-import appscale_info
-from constants import SCHEMA_CHANGE_TIMEOUT
-
 sys.path.append(APPSCALE_PYTHON_APPSERVER)
 from google.appengine.api.taskqueue import taskqueue_service_pb
 from google.appengine.datastore import entity_pb

AppDB/appscale/datastore/cassandra_env/get_token.py

@@ -1,7 +1,7 @@
 from __future__ import division
 import argparse
-import sys
 
+from appscale.common import appscale_info
 from cassandra.cluster import Cluster
 from random import choice
 from random import shuffle
@@ -14,10 +14,6 @@
 from ..dbconstants import APP_ENTITY_SCHEMA
 from ..dbconstants import KEY_DELIMITER
 from ..dbconstants import KIND_SEPARATOR
-from ..unpackaged import APPSCALE_LIB_DIR
-
-sys.path.append(APPSCALE_LIB_DIR)
-import appscale_info
 
 
 def is_entity(key):

AppDB/appscale/datastore/cassandra_env/py_cassandra.py

@@ -2,9 +2,9 @@
 Cassandra Interface for AppScale
 """
 import cassandra
-import sys
 import time
 
+from appscale.common import appscale_info
 from cassandra.cluster import BatchStatement
 from cassandra.cluster import Cluster
 from cassandra.cluster import SimpleStatement
@@ -19,10 +19,6 @@
 from ..dbconstants import SCHEMA_TABLE
 from ..dbconstants import SCHEMA_TABLE_SCHEMA
 from ..dbinterface import AppDBInterface
-from ..unpackaged import APPSCALE_LIB_DIR
-
-sys.path.append(APPSCALE_LIB_DIR)
-import appscale_info
 
 ERROR_DEFAULT = "DB_ERROR:" # ERROR_CASSANDRA
 

AppDB/appscale/datastore/cassandra_env/rebalance.py

@@ -3,15 +3,12 @@
 import os
 import sys
 
+from appscale.common.constants import LOG_FORMAT
+from appscale.common.unpackaged import INFRASTRUCTURE_MANAGER_DIR
 from subprocess import (CalledProcessError,
                         check_output)
 from ..cassandra_env.cassandra_interface import NODE_TOOL
 from ..cassandra_env.cassandra_interface import KEYSPACE
-from ..unpackaged import APPSCALE_LIB_DIR
-from ..unpackaged import INFRASTRUCTURE_MANAGER_DIR
-
-sys.path.append(APPSCALE_LIB_DIR)
-from constants import LOG_FORMAT
 
 sys.path.append(INFRASTRUCTURE_MANAGER_DIR)
 from utils.utils import KEY_DIRECTORY

AppDB/appscale/datastore/cassandra_env/schema.py

@@ -2,11 +2,12 @@
 
 import cassandra
 import logging
-import sys
 import time
 
 import cassandra_interface
 
+from appscale.common import appscale_info
+from appscale.common.constants import SCHEMA_CHANGE_TIMEOUT
 from appscale.taskqueue.distributed_tq import create_pull_queue_tables
 from cassandra import ConsistencyLevel
 from cassandra.cluster import Cluster
@@ -17,11 +18,6 @@
 from .cassandra_interface import KEYSPACE
 from .cassandra_interface import ThriftColumn
 from .. import dbconstants
-from ..unpackaged import APPSCALE_LIB_DIR
-
-sys.path.append(APPSCALE_LIB_DIR)
-import appscale_info
-from constants import SCHEMA_CHANGE_TIMEOUT
 
 # The data layout version to set after removing the journal table.
 POST_JOURNAL_VERSION = 1.0

AppDB/appscale/datastore/datastore_distributed.py

@@ -11,11 +11,11 @@
 import dbconstants
 import helper_functions
 
+from appscale.common.unpackaged import APPSCALE_PYTHON_APPSERVER
 from .dbconstants import APP_ENTITY_SCHEMA
 from .dbconstants import ID_KEY_LENGTH
 from .dbconstants import MAX_TX_DURATION
 from .cassandra_env import cassandra_interface
-from .unpackaged import APPSCALE_PYTHON_APPSERVER
 from .utils import clean_app_id
 from .utils import encode_entity_table_key
 from .utils import encode_index_pb

AppDB/appscale/datastore/groomer.py

@@ -11,12 +11,13 @@
 import dbconstants
 import utils
 
+from appscale.common import appscale_info
+from appscale.common import constants
+from appscale.common.unpackaged import APPSCALE_PYTHON_APPSERVER
+from appscale.common.unpackaged import DASHBOARD_DIR
 from appscale.taskqueue.distributed_tq import TaskName
 from .cassandra_env import cassandra_interface
 from .datastore_distributed import DatastoreDistributed
-from .unpackaged import APPSCALE_LIB_DIR
-from .unpackaged import APPSCALE_PYTHON_APPSERVER
-from .unpackaged import DASHBOARD_DIR
 from .utils import get_composite_indexes_rows
 from .zkappscale import zktransaction as zk
 
@@ -32,10 +33,6 @@
 from google.appengine.ext.db import metadata
 from google.appengine.api import datastore_errors
 
-sys.path.append(APPSCALE_LIB_DIR)
-import appscale_info
-import constants
-
 sys.path.append(os.path.join(DASHBOARD_DIR, 'lib'))
 from dashboard_logs import RequestLogLine
 

AppDB/appscale/datastore/scripts/backup_data.py

@@ -3,15 +3,11 @@
 """
 import argparse
 import logging
-import sys
 
+from appscale.common import appscale_info
 from ..backup.datastore_backup import DatastoreBackup
-from ..unpackaged import APPSCALE_LIB_DIR
 from ..zkappscale import zktransaction as zk
 
-sys.path.append(APPSCALE_LIB_DIR)
-import appscale_info
-
 # The location to look at in order to verify that an app is deployed.
 _SOURCE_LOCATION = '/opt/appscale/apps/'
 

AppDB/appscale/datastore/scripts/blobstore.py

@@ -26,8 +26,11 @@
 import urllib
 import urllib2
 
-from ..unpackaged import APPSCALE_LIB_DIR
-from ..unpackaged import APPSCALE_PYTHON_APPSERVER
+from appscale.common import appscale_info
+from appscale.common.constants import LOG_FORMAT
+from appscale.common.deployment_config import DeploymentConfig
+from appscale.common.deployment_config import ConfigInaccessible
+from appscale.common.unpackaged import APPSCALE_PYTHON_APPSERVER
 from StringIO import StringIO
 
 sys.path.append(APPSCALE_PYTHON_APPSERVER)
@@ -39,12 +42,6 @@
 from google.appengine.api.blobstore import datastore_blob_storage
 from google.appengine.tools import dev_appserver_upload
 
-sys.path.append(APPSCALE_LIB_DIR)
-import appscale_info
-from constants import LOG_FORMAT
-from deployment_config import DeploymentConfig
-from deployment_config import ConfigInaccessible
-
 # The URL path used for uploading blobs
 UPLOAD_URL_PATH = '_ah/upload/'
 

AppDB/appscale/datastore/scripts/cassandra_backup.py

@@ -5,15 +5,12 @@
 import os
 import sys
 
+from appscale.common.constants import LOG_FORMAT
+from appscale.common.unpackaged import INFRASTRUCTURE_MANAGER_DIR
 from ..backup.backup_exceptions import AmbiguousKeyException
 from ..backup.backup_exceptions import NoKeyException
 from ..backup.cassandra_backup import backup_data
 from ..backup.cassandra_backup import restore_data
-from ..unpackaged import APPSCALE_LIB_DIR
-from ..unpackaged import INFRASTRUCTURE_MANAGER_DIR
-
-sys.path.append(APPSCALE_LIB_DIR)
-from constants import LOG_FORMAT
 
 sys.path.append(INFRASTRUCTURE_MANAGER_DIR)
 from utils.utils import KEY_DIRECTORY

AppDB/appscale/datastore/scripts/datastore.py

@@ -16,19 +16,16 @@
 import tornado.ioloop
 import tornado.web
 
+from appscale.common import appscale_info
+from appscale.common.unpackaged import APPSCALE_PYTHON_APPSERVER
 from .. import dbconstants
 from ..appscale_datastore_batch import DatastoreFactory
 from ..datastore_distributed import DatastoreDistributed
 from ..utils import (clean_app_id,
                      logger,
                      UnprocessedQueryResult)
-from ..unpackaged import APPSCALE_LIB_DIR
-from ..unpackaged import APPSCALE_PYTHON_APPSERVER
 from ..zkappscale import zktransaction
 
-sys.path.append(APPSCALE_LIB_DIR)
-import appscale_info
-
 sys.path.append(APPSCALE_PYTHON_APPSERVER)
 from google.appengine.api import api_base_pb
 from google.appengine.api.taskqueue import taskqueue_service_pb

AppDB/appscale/datastore/scripts/delete_records.py

@@ -3,6 +3,7 @@
 import logging
 import sys
 
+from appscale.common.constants import LOG_FORMAT
 from ..dbconstants import APP_ENTITY_SCHEMA
 from ..dbconstants import APP_ENTITY_TABLE
 from ..dbconstants import APP_KIND_SCHEMA
@@ -14,12 +15,8 @@
 from ..dbconstants import METADATA_SCHEMA
 from ..dbconstants import METADATA_TABLE
 from ..dbconstants import PROPERTY_SCHEMA
-from ..unpackaged import APPSCALE_LIB_DIR
 from ..utils import fetch_and_delete_entities
 
-sys.path.append(APPSCALE_LIB_DIR)
-from constants import LOG_FORMAT
-
 
 def main():
   logging.basicConfig(format=LOG_FORMAT, level=logging.INFO)

AppDB/appscale/datastore/scripts/groomer_service.py

@@ -1,15 +1,11 @@
 """ Provides a service which periodically runs the groomer. """
 import logging
-import sys
 
+from appscale.common import appscale_info
+from appscale.common.constants import LOG_FORMAT
 from .. import groomer
-from ..unpackaged import APPSCALE_LIB_DIR
 from ..zkappscale import zktransaction as zk
 
-sys.path.append(APPSCALE_LIB_DIR)
-import appscale_info
-from constants import LOG_FORMAT
-
 # Location to find the datastore service.
 LOCAL_DATASTORE = "localhost:8888"