AppScale should not crash with bad applications.

AppController/djinn.rb

@@ -20,27 +20,26 @@
 
 # Imports for AppController libraries
 $:.unshift File.join(File.dirname(__FILE__), "lib")
-require 'helperfunctions'
+require 'app_controller_client'
+require 'blobstore'
+require 'custom_exceptions'
+require 'ejabberd'
+require 'error_app'
+require 'collectd'
 require 'cron_helper'
+require 'godinterface'
 require 'haproxy'
-require 'collectd'
-require 'nginx'
+require 'helperfunctions'
+require 'infrastructure_manager_client'
+require 'neptune_manager_client'
 require 'pbserver'
-require 'blobstore'
+require 'nginx'
 require 'rabbitmq'
-require 'app_controller_client'
-require 'user_app_client'
-require 'ejabberd'
 require 'repo'
+require 'user_app_client'
 require 'zkinterface'
-require 'godinterface'
-require 'infrastructure_manager_client'
-require 'neptune_manager_client'
 
 
-class AppScaleException < Exception
-end
-
 
 WANT_OUTPUT = true
 
@@ -1543,7 +1542,6 @@ def write_zookeeper_locations
 
 
   def update_api_status()
-    return
     if my_node.is_appengine?
       repo_host = my_node.private_ip
     else
@@ -2129,7 +2127,7 @@ def change_job()
       retval = 0
       while retries > 0
         replication = @creds["replication"]
-        Djinn.log_run("MASTER_IP='localhost' LOCAL_DB_IP='localhost' python2.6 #{prime_script} #{replication}; echo $? > /tmp/retval")
+        Djinn.log_run("APPSCALE_HOME='#{APPSCALE_HOME}' MASTER_IP='localhost' LOCAL_DB_IP='localhost' python2.6 #{prime_script} #{replication}; echo $? > /tmp/retval")
         retval = `cat /tmp/retval`.to_i
         break if retval == 0
         Djinn.log_debug("Fail to create initial table. Retry #{retries} times.")
@@ -2245,7 +2243,7 @@ def start_pbserver
     zoo_connection = get_zk_connection_string(@nodes)
     PbServer.start(db_master_ip, @userappserver_private_ip, my_ip, table, zoo_connection)
     HAProxy.create_pbserver_config(my_node.private_ip, PbServer::PROXY_PORT, table)
-    Nginx.create_pbserver_config(my_ip, PbServer::PROXY_PORT)
+    Nginx.create_pbserver_config(my_node.private_ip, PbServer::PROXY_PORT)
     Nginx.restart()
 
     # TODO check the return value
@@ -2741,6 +2739,26 @@ def stop_shadow()
     Djinn.log_debug("Stopping Shadow role")
   end
 
+  #
+  # Swaps out an application with one that relays an error message to the 
+  # developer. It will take the application that currently exists in the 
+  # application folder, deletes it, and places a templated app that prints out the 
+  # given error message. 
+  #
+  # Args: 
+  #   app_name: Name of application to construct an error application for
+  #   err_msg: A String message that will be displayed as 
+  #            the reason why we couldn't start their application.
+  #
+  # Returns: 
+  #   Returns: Nothing
+  #
+  def place_error_app(app_name, err_msg)
+    Djinn.log_debug("Placing error application for #{app_name} because of: #{err_msg}")
+    ea = ErrorApp.new(app_name, err_msg)
+    ea.generate() 
+  end
+
   def start_appengine()
     @state = "Preparing to run AppEngine apps if needed"
     Djinn.log_debug("Starting appengine - pbserver is at [#{@userappserver_private_ip}]")
@@ -2803,7 +2821,9 @@ def start_appengine()
       app_path = "#{app_dir}/#{app}.tar.gz"
       FileUtils.mkdir_p(app_dir)
 
-      copy_app_to_local(app)
+      if !copy_app_to_local(app)
+        place_error_app(app, "ERROR: Failed to copy app: #{app}")
+      end
       HelperFunctions.setup_app(app)
 
 
@@ -2813,15 +2833,16 @@ def start_appengine()
       end
       app_number = @nginx_port - Nginx::START_PORT
       proxy_port = HAProxy.app_listen_port(app_number)
-      login_ip = get_login.public_ip
+      login_ip = get_login.private_ip
       if my_node.is_login? and !my_node.is_appengine?
         success = Nginx.write_fullproxy_app_config(app, app_number, my_public,
           my_private, proxy_port, login_ip, get_all_appengine_nodes())
         if success
           Nginx.reload
         else
-          Djinn.log_debug("ERROR: Failure to create valid nginx config file for application #{app} full proxy.")
-          next
+          err_msg = "ERROR: Failure to create valid nginx config file" + \
+                    " for application #{app} full proxy."
+          place_error_app(app, err_msg)
         end
         @nginx_port += 1
         @haproxy_port += 1
@@ -2831,14 +2852,22 @@ def start_appengine()
       if my_node.is_appengine?
         app_number = @nginx_port - Nginx::START_PORT
         start_port = HelperFunctions::APP_START_PORT
-        static_handlers = HelperFunctions.parse_static_data(app)
+        begin
+          static_handlers = HelperFunctions.parse_static_data(app)
+        rescue Exception => e
+          # This specific exception may be a json parse error
+          error_msg = "ERROR: Unable to parse app.yaml file for #{app}." + \
+                      " Exception of #{e.class} with message #{e.message}" 
+          place_error_app(app, error_msg)
+        end
         proxy_port = HAProxy.app_listen_port(app_number)
-        login_ip = get_login.public_ip
+        login_ip = get_login.private_ip
         success = Nginx.write_app_config(app, app_number, my_public, my_private,
           proxy_port, static_handlers, login_ip)
-        if not success
-          Djinn.log_debug("ERROR: Failure to create valid nginx config file for application #{app}.")
-          next
+        if !success
+          error_msg = "ERROR: Failure to create valid nginx config file " + \
+                      "for application #{app}."
+          place_error_app(app, error_msg)
         end
         Collectd.write_app_config(app)
 
@@ -2859,14 +2888,14 @@ def start_appengine()
             @userappserver_private_ip, get_load_balancer_ip(), my_private, 
             app_version, app_language, @nginx_port, xmpp_ip)
           if pid == -1
-            Djinn.log_debug("ERROR: Unable to start application #{app}.") 
-            next
+            place_error_app(app, "ERROR: Unable to start application " + \
+                "#{app}. Please check the application logs.") 
           end
 
           pid_file_name = "/etc/appscale/#{app}-#{@appengine_port}.pid"
           HelperFunctions.write_file(pid_file_name, pid)
 
-          location = "http://#{my_public}:#{@appengine_port}#{warmup_url}"
+          location = "http://#{my_private}:#{@appengine_port}#{warmup_url}"
           wget_cmd = "wget #{WGET_OPTIONS} #{location}"
 
           Djinn.log_run(wget_cmd)
@@ -2875,7 +2904,7 @@ def start_appengine()
         }
 
         HAProxy.update_app_config(app, app_number, 
-          @app_info_map[app][:appengine], my_public)
+          @app_info_map[app][:appengine], my_private)
         Nginx.reload
         HAProxy.reload
         Collectd.restart
@@ -2903,7 +2932,7 @@ def start_appengine()
         login_ip = get_login.public_ip
 
         Thread.new {
-          haproxy_location = "http://#{my_public}:#{haproxy}#{warmup_url}"
+          haproxy_location = "http://#{my_private}:#{haproxy}#{warmup_url}"
           nginx_location = "http://#{my_public}:#{nginx}#{warmup_url}"
 
           wget_haproxy = "wget #{WGET_OPTIONS} #{haproxy_location}"
@@ -3209,7 +3238,7 @@ def add_appserver_process(app)
     my_private = my_node.private_ip
     Djinn.log_debug("port apps error contains - #{@app_info_map[app][:appengine]}")
     HAProxy.update_app_config(app, app_number, @app_info_map[app][:appengine],
-      my_public)     
+      my_private)     
 
     Djinn.log_debug("Adding #{app_language} app #{app} on #{HelperFunctions.local_ip}:#{@appengine_port} ")
     xmpp_ip = get_login.public_ip
@@ -3221,7 +3250,7 @@ def add_appserver_process(app)
     pid_file_name = "#{APPSCALE_HOME}/.appscale/#{app}-#{@appengine_port}.pid"
     HelperFunctions.write_file(pid_file_name, pid)
 
-    location = "http://#{my_public}:#{@appengine_port}#{warmup_url}"
+    location = "http://#{my_private}:#{@appengine_port}#{warmup_url}"
     wget_cmd = "wget #{WGET_OPTIONS} #{location}"
 
     Djinn.log_run(wget_cmd)
@@ -3234,10 +3263,8 @@ def add_appserver_process(app)
 
     # add_instance_info = uac.add_instance(app, my_public, @nginx_port)
 
-    login_ip = get_login.public_ip
-
     Thread.new {
-      haproxy_location = "http://#{my_public}:#{haproxy_port}#{warmup_url}"
+      haproxy_location = "http://#{my_private}:#{haproxy_port}#{warmup_url}"
       nginx_location = "http://#{my_public}:#{nginx_port}#{warmup_url}"
 
       wget_haproxy = "wget #{WGET_OPTIONS} #{haproxy_location}"
@@ -3294,7 +3321,7 @@ def remove_appserver_process(app)
     @app_info_map[app][:appengine].delete(port)
 
     HAProxy.update_app_config(app, app_number, @app_info_map[app][:appengine],
-      my_public)
+      my_private)
     HAProxy.reload
   end 
 
@@ -3441,20 +3468,26 @@ def start_sisyphus
 
     my_public = my_node.public_ip
     my_private = my_node.private_ip
-    login_ip = get_login.public_ip
-
-    static_handlers = HelperFunctions.parse_static_data(app)
+    public_login_ip = get_login.public_ip
+    private_login_ip = get_login.private_ip
+
+    begin
+      static_handlers = HelperFunctions.parse_static_data(app)
+    rescue Exception => e
+      error_msg = "ERROR: Unable to parse app.yaml file for #{app}." + \
+                  " Exception of type #{e.class}. Exception message #{e.message}"
+      place_error_app(app, error_msg)
+    end
     proxy_port = HAProxy.app_listen_port(app_number)
-    Nginx.write_app_config(app, app_number, my_public, my_private, 
-                           proxy_port, static_handlers, login_ip)
+    Nginx.write_app_config(app, app_number, my_public, my_private, proxy_port, static_handlers, private_login_ip)
     HAProxy.write_app_config(app, app_number, num_servers, my_private)
     Collectd.write_app_config(app)
 
     [19994, 19995, 19996].each { |port|
       Djinn.log_debug("Starting #{app_language} app #{app} on " +
         "#{HelperFunctions.local_ip}:#{port}")
       pid = HelperFunctions.run_app(app, port, @userappserver_private_ip, 
-        my_public, my_private, app_version, app_language, nginx_port, login_ip)
+        my_public, my_private, app_version, app_language, nginx_port, public_login_ip)
       pid_file_name = "#{APPSCALE_HOME}/.appscale/#{app}-#{port}.pid"
       HelperFunctions.write_file(pid_file_name, pid)
     }

AppController/djinnServer.rb

@@ -70,6 +70,8 @@ def on_init
 `rm -f #{APPSCALE_HOME}/.appscale/status-*`
 `rm -f #{APPSCALE_HOME}/.appscale/database_info`
 `rm -f /tmp/mysql.sock`
+
+
 Nginx.clear_sites_enabled
 Collectd.clear_sites_enabled
 HAProxy.clear_sites_enabled

AppController/lib/custom_exceptions.rb

@@ -0,0 +1,14 @@
+# Programmer: Navraj Chohan
+
+# A class of exceptions that can be thrown if the AppController is put into an
+# unrecoverable state, or a state that we would not normally expect a perfectly
+# working AppScale system to get into.
+class AppScaleException < Exception
+end
+
+# A class of exceptions that can be thrown if the AppController 
+# (or its associated libraries) attempts to execute shell commands which 
+# do not return properly (specifically, not having a return value of zero).
+class FailedShellExec < Exception
+end
+

AppController/lib/error_app.rb

@@ -0,0 +1,87 @@
+#!/usr/bin/ruby -w
+
+require 'fileutils'
+
+$:.unshift File.join(File.dirname(__FILE__))
+require 'custom_exceptions'
+require 'helperfunctions'
+
+$:.unshift File.join(File.dirname(__FILE__), "..")
+require 'djinn'
+
+
+# This class generates a Python Google App Engine application that 
+# relays an error message to the user as to why their app failed to come up.
+class ErrorApp
+
+  #
+  # Constructor 
+  #
+  # Args: 
+  #   app_name: Name of the application to construct an error application for.
+  #   error_msg: A String message that will be displayed as the reason 
+  #              why we couldn't start their application.
+  def initialize(app_name, error_msg)
+    @app_name = app_name
+    @error_msg = error_msg
+    @dir_path = "/var/apps/#{app_name}/app/"
+  end
+
+  #
+  # This function places an updated app.yaml and error.py into the application
+  # and retars the application file.
+  # 
+  # Args: None
+  def generate()
+    app_yaml = <<CONFIG
+application: #{@app_name}
+version: 1
+runtime: python
+api_version: 1
+
+handlers:
+- url: .*
+  script: #{@app_name}.py
+CONFIG
+
+    script = <<SCRIPT
+from google.appengine.ext import webapp
+import cgi
+import datetime
+import wsgiref.handlers
+class MainPage(webapp.RequestHandler):
+  def get(self):
+    self.response.out.write('<html><body>')
+    self.response.out.write("""<p>Your application failed to start</p>""")
+    self.response.out.write("""<p>#{@error_msg}</p>""")
+    self.response.out.write("""<p>If this is an AppScale issue please report it on <a href="https://github.com/AppScale/appscale/issues">http://github.com/AppScale/appscale/issues</a></p>""")
+    self.response.out.write('</body></html>')
+
+application = webapp.WSGIApplication([
+  ('/', MainPage),
+], debug=True)
+
+
+def main():
+  wsgiref.handlers.CGIHandler().run(application)
+
+
+if __name__ == '__main__':
+  main()
+
+SCRIPT
+
+    HelperFunctions.write_file(@dir_path + 'app.yaml', app_yaml)
+    HelperFunctions.write_file(@dir_path + "#{@app_name}.py", script) 
+
+    Djinn.log_run("rm #{@dir_path}/#{@app_name}.tar.gz")
+    Dir.chdir(@dir_path) do
+      Djinn.log_debug("Running: tar zcvf #{@dir_path}/#{@app_name}.tar.gz #{@dir_path}")
+      Djinn.log_run("tar zcvf #{@app_name}.tar.gz app.yaml #{@app_name}.py")
+    end
+
+    return true
+  end
+
+end
+