Test suite: Add busy-loop for making sure the app starts

Note that this also handles the vboxsf bind mount which otherwise I
have been manually typing out in my dev environment.

Makefile

@@ -22,13 +22,17 @@ action-deploy-app: stage-install-service action-update-source
 	if sudo grep -q systemd /proc/1/exe ; then sudo systemctl restart sandcats.service ; fi
 
 action-run-dev:
+	# If we are using VirtualBox file sharing, work around race conditions in vboxsf by playing
+	# games with loopback mounts.
+	mkdir -p /tmp/meteor-local
+	if mount | grep -q vboxsf ; then if ! mount | grep -q /vagrant/sandcats/.meteor/local ; then sudo mount --bind /tmp/meteor-local /vagrant/sandcats/.meteor/local ; fi; fi
 	(cd sandcats ; MAIL_URL=smtp://localhost:2500 meteor run --settings=dev-settings.json )
 
 action-run-tests: /usr/share/doc/python-requests /usr/share/doc/python-dnspython /usr/share/doc/python-netifaces /usr/share/doc/python-twisted
 	cd sandcats && python -u integration_tests.py
 
 action-reset-app-state: /tmp/can-reset-state /usr/share/doc/python-requests /usr/share/doc/python-dnspython /usr/share/doc/python-netifaces /usr/share/doc/python-twisted
-	cd sandcats && echo 'reset_app_state()' | python -i integration_tests.py
+	cd sandcats && python integration_tests.py --reset-app-state
 
 action-run-unit-tests:
 	(cd sandcats ; tail -c 0 --retry -f ./.meteor/local/log/jasmine-server-integration.log & (meteor --test --settings=dev-settings.json 2>&1 || true) | python ../meteor-testing-nonsense/input-filter.py )

sandcats/integration_tests.py

@@ -7,6 +7,7 @@
 import time
 import socket
 import subprocess
+import sys
 
 import logging
 import httplib
@@ -432,6 +433,26 @@ def reset_app_state():
     os.system('killall -INT node')
 
     time.sleep(1)  # Make sure the restart gets a chance to start, to avoid HTTP 502.
+
+    # Busy-loop waiting for / to get a HTTP response. This way, we avoid restarting nginx
+    # until the service is online.
+    RESOLUTION = 0.5
+    WAIT_SECONDS = 120  # Note that it could be at worst 2x this, since requests.get() blocks for
+                        # timeout=RESOLUTION as well.
+    got_success = False
+    sys.stdout.write('waiting ')
+    sys.stdout.flush()
+    for i in range(int(WAIT_SECONDS * 1/RESOLUTION)):
+        try:
+            requests.get('http://localhost:3000', timeout=RESOLUTION)
+            got_success = True
+            break  # stop the loop since we got what we needed
+        except (requests.exceptions.ReadTimeout, requests.exceptions.ConnectionError):
+            sys.stdout.write('.')
+            sys.stdout.flush()
+            continue  # keep looping, maybe it'll work next time!
+    assert got_success, "Bailing out - service failed to come up"
+
     os.system('sudo service nginx restart')
     os.system('sudo service pdns restart')
     # Attempt to get the homepage, which will mean that Meteor is back, waiting at most 10 seconds.
@@ -783,6 +804,10 @@ def test_udp_protocol():
 
 
 if __name__ == '__main__':
+    if '--reset-app-state' in sys.argv:
+        reset_app_state()
+        sys.exit(0)
+
     test_register()
     test_recovery()
     test_update()