Merge branch master into removing-cassandra-pq-backend

AdminServer/appscale/admin/admin_server.py

@@ -27,7 +27,7 @@
   VERSION_PATH_SEPARATOR,
   ZK_PERSISTENT_RECONNECTS
 )
-from appscale.common.monit_interface import MonitOperator
+from appscale.common.service_helper import ServiceOperator
 from appscale.common.appscale_utils import get_md5
 from appscale.common.ua_client import UAClient
 from appscale.common.ua_client import UAException
@@ -75,7 +75,7 @@
 from .resource_validator import validate_resource, ResourceValidationError
 from .routing.routing_manager import RoutingManager
 from .service_manager import ServiceManager, ServiceManagerHandler
-from .summary import get_combined_services
+from .summary import get_services
 
 logger = logging.getLogger(__name__)
 
@@ -1365,7 +1365,7 @@ def main():
 
   args = parser.parse_args()
   if args.command == 'summary':
-    table = sorted(list(get_combined_services().items()))
+    table = sorted(list(get_services().items()))
     print(tabulate(table, headers=['Service', 'State']))
     sys.exit(0)
 
@@ -1395,7 +1395,7 @@ def main():
   zk_client.start()
   version_update_lock = zk_client.Lock(constants.VERSION_UPDATE_LOCK_NODE)
   thread_pool = ThreadPoolExecutor(4)
-  monit_operator = MonitOperator()
+  service_operator = ServiceOperator(thread_pool)
   all_resources = {
     'acc': acc,
     'ua_client': ua_client,
@@ -1406,7 +1406,7 @@ def main():
 
   if options.private_ip in appscale_info.get_taskqueue_nodes():
     logger.info('Starting push worker manager')
-    GlobalPushWorkerManager(zk_client, monit_operator)
+    GlobalPushWorkerManager(zk_client, service_operator)
 
   if options.private_ip in appscale_info.get_load_balancer_ips():
     logger.info('Starting RoutingManager')

AdminServer/appscale/admin/appengine_api.py

@@ -5,14 +5,27 @@
 
 import json
 import logging
+import random
+
+try:
+  import urllib.request
+  urlopen = urllib.request.urlopen
+  from urllib.error import URLError
+except ImportError:
+  import urllib2
+  urlopen = urllib2.urlopen
+  from urllib2 import URLError
+
 import six
 import yaml
 from kazoo.exceptions import NoNodeError
+from tornado.options import options
 from yaml.parser import ParserError
 
 from appscale.appcontroller_client import AppControllerException
-from appscale.common.constants import HTTPCodes, InvalidIndexConfiguration
-from appscale.common.datastore_index import DatastoreIndex, merge_indexes
+from appscale.common.constants import (
+  DB_SERVER_PORT, HTTPCodes, InvalidIndexConfiguration)
+from appscale.common.datastore_index import DatastoreIndex
 from .base_handler import BaseHandler
 from .constants import (
   CustomHTTPError,
@@ -68,7 +81,21 @@ def post(self):
       raise CustomHTTPError(HTTPCodes.BAD_REQUEST,
                             message=six.text_type(error))
 
-    merge_indexes(self.zk_client, project_id, given_indexes)
+    datastore_location = ':'.join(
+      [random.choice(options.load_balancers), str(DB_SERVER_PORT)])
+    url = 'http://{}/index/add?project={}'.format(
+      datastore_location, project_id)
+    payload = json.dumps([index.to_dict() for index in given_indexes])
+    try:
+      response = urlopen(url, payload)
+    except URLError:
+      raise CustomHTTPError(HTTPCodes.INTERNAL_ERROR,
+                            message='Unable to forward request to datastore')
+
+    if response.code != 200:
+      message = 'Unable to add indexes: {}'.format(response.read())
+      raise CustomHTTPError(HTTPCodes.INTERNAL_ERROR, message=message)
+
     logger.info('Updated indexes for {}'.format(project_id))
 
 

AdminServer/appscale/admin/constants.py

@@ -158,7 +158,6 @@ class Types(object):
   + list(HAPROXY_PORTS)
   + [
     2181,     # Zookeeper
-    2812,     # Monit
     3306,     # MySQL
     4341,     # UserAppServer service
     4342,     # UserAppServer server

AdminServer/appscale/admin/instance_manager/constants.py

@@ -35,8 +35,8 @@ def http_response(self, request, response):
 API_SERVER_LOCATION = os.path.join('/', 'opt', 'appscale_venvs', 'api_server',
                                    'bin', 'appscale-api-server')
 
-# The Monit watch prefix for API servers.
-API_SERVER_PREFIX = 'api-server_'
+# Prefix for API server services.
+API_SERVER_PREFIX = 'appscale-api-server@'
 
 # Max application server log size in bytes.
 APP_LOG_SIZE = 250 * 1024 * 1024
@@ -109,8 +109,8 @@ def http_response(self, request, response):
   os.path.join('/', 'usr', 'share', 'appscale', 'ext', '*')
 ]
 
-# A prefix added to instance entries to distinguish them from services.
-MONIT_INSTANCE_PREFIX = 'app___'
+# Common prefix for instance services.
+SERVICE_INSTANCE_PREFIX = 'appscale-instance-run@'
 
 # The script used for starting Python AppServer instances.
 PYTHON_APPSERVER = os.path.join(APPSCALE_HOME, 'AppServer',

AdminServer/appscale/admin/instance_manager/instance.py

@@ -170,15 +170,14 @@ def create_java_start_cmd(app_name, port, load_balancer_port, load_balancer_host
   return ' '.join(cmd)
 
 
-def create_python_api_start_cmd(app_name, login_ip, port, pidfile,
+def create_python_api_start_cmd(app_name, login_ip, port,
                                 api_server_port):
   """ Creates the start command to run the python api server.
 
   Args:
     app_name: The name of the application to run
     login_ip: The public IP of this deployment
     port: The local port the api server will bind to
-    pidfile: A string specifying the pidfile location.
     api_server_port: An integer specifying the port of the external API server.
   Returns:
     A string of the start command.
@@ -194,7 +193,6 @@ def create_python_api_start_cmd(app_name, login_ip, port, pidfile,
       '--uaserver_path', '{}:{}'.format(options.load_balancer_ip,
                                         UA_SERVER_PORT),
       '--datastore_path', '{}:{}'.format(options.db_proxy, DB_SERVER_PORT),
-      '--pidfile', pidfile,
       '--external_api_port', str(api_server_port)
   ]