diff --git a/scripts/dev/Makefile-dmg b/scripts/dev/Makefile-dmg
index 1ecdb15141e..0f699ea8b50 100644
--- a/scripts/dev/Makefile-dmg
+++ b/scripts/dev/Makefile-dmg
@@ -24,7 +24,7 @@ SOURCE_DIR ?= .
 SOURCE_FILES ?= dist/qutebrowser.app COPYING
 
 TEMPLATE_DMG ?= template.dmg
-TEMPLATE_SIZE ?= 120m
+TEMPLATE_SIZE ?= 300m
 
 ################################################################################
 # DMG building. No editing should be needed beyond this point.
@@ -37,17 +37,12 @@ WC_DIR=wc
 .PHONY: all
 all: $(MASTER_DMG)
 
-$(TEMPLATE_DMG): $(TEMPLATE_DMG).bz2
-	bunzip2 -k $<
-
-$(TEMPLATE_DMG).bz2: 
+$(TEMPLATE_DMG):
 	@echo
 	@echo --------------------- Generating empty template --------------------
 	mkdir template
 	hdiutil create -fs HFSX -layout SPUD -size $(TEMPLATE_SIZE) "$(TEMPLATE_DMG)" -srcfolder template -format UDRW -volname "$(NAME)" -quiet
 	rmdir template
-	bzip2 "$(TEMPLATE_DMG)"
-	@echo
 
 $(WC_DMG): $(TEMPLATE_DMG)
 	cp $< $@
diff --git a/scripts/dev/build_release.py b/scripts/dev/build_release.py
index 31906b0c27e..fc0897d4bf8 100755
--- a/scripts/dev/build_release.py
+++ b/scripts/dev/build_release.py
@@ -28,6 +28,7 @@
 import subprocess
 import argparse
 import tarfile
+import tempfile
 import collections
 
 sys.path.insert(0, os.path.join(os.path.dirname(__file__), os.pardir,
@@ -94,7 +95,7 @@ def build_osx():
     utils.print_title("Updating 3rdparty content")
     update_3rdparty.update_pdfjs()
     utils.print_title("Building .app via pyinstaller")
-    call_tox('pyinstaller')
+    call_tox('pyinstaller', '-r')
     utils.print_title("Building .dmg")
     subprocess.check_call(['make', '-f', 'scripts/dev/Makefile-dmg'])
     utils.print_title("Cleaning up...")
@@ -103,6 +104,17 @@ def build_osx():
     for d in ['dist', 'build']:
         shutil.rmtree(d)
 
+    utils.print_title("Running smoke test")
+    with tempfile.TemporaryDirectory() as tmpdir:
+        subprocess.check_call(['hdiutil', 'attach', 'qutebrowser.dmg',
+                               '-mountpoint', tmpdir])
+        try:
+            binary = os.path.join(tmpdir, 'qutebrowser.app', 'Contents',
+                                  'MacOS', 'qutebrowser')
+            smoke_test(binary)
+        finally:
+            subprocess.check_call(['hdiutil', 'detach', tmpdir])
+
 
 def build_windows():
     """Build windows executables/setups."""
@@ -116,6 +128,8 @@ def build_windows():
     python_x86 = r'C:\Python{}_x32'.format(ver)
     python_x64 = r'C:\Python{}'.format(ver)
 
+    utils.print_title("Rebuilding tox environment")
+    call_tox('cxfreeze-windows', '-r', '--notest')
     utils.print_title("Running 32bit freeze.py build_exe")
     call_tox('cxfreeze-windows', 'build_exe', python=python_x86)
     utils.print_title("Running 32bit freeze.py bdist_msi")