diff --git a/app/models/app.rb b/app/models/app.rb
index 04b3a1ff7..c974d3412 100644
--- a/app/models/app.rb
+++ b/app/models/app.rb
@@ -12,6 +12,7 @@ class App < ActiveRecord::Base
                    }
   validate :custom_tracking_domain_points_to_correct_place
   validate :validate_dkim_settings
+  validate :webhook_url_works
   # Validating booleans so that they can't have nil values.
   # See https://stackoverflow.com/questions/34759092/to-validate-or-not-to-validate-boolean-field
   validates :open_tracking_enabled, inclusion: { in: [true, false] }
@@ -21,7 +22,7 @@ class App < ActiveRecord::Base
   validates :legacy_dkim_selector, inclusion: { in: [true, false] }
 
   before_create :set_smtp_password
-  before_create :set_webhook_key
+  before_validation :set_webhook_key
   after_create :set_smtp_username
 
   def self.cuttlefish
@@ -144,6 +145,21 @@ def custom_tracking_domain_points_to_correct_place
     )
   end
 
+  def webhook_url_works
+    return if webhook_url.nil?
+
+    RestClient.post(
+      webhook_url,
+      { key: webhook_key, test_event: {} }.to_json,
+      { content_type: :json }
+    )
+  rescue RestClient::ExceptionWithResponse => e
+    errors.add(
+      :webhook_url,
+      "returned #{e.response.code} code when doing POST to #{webhook_url}"
+    )
+  end
+
   def set_smtp_password
     # Only set if it hasn't been set already
     # This makes testing a little more straightforward
diff --git a/spec/fixtures/vcr_cassettes/webhook.yml b/spec/fixtures/vcr_cassettes/webhook.yml
new file mode 100644
index 000000000..aa9fe5370
--- /dev/null
+++ b/spec/fixtures/vcr_cassettes/webhook.yml
@@ -0,0 +1,50 @@
+---
+http_interactions:
+- request:
+    method: post
+    uri: https://www.planningalerts.org.au/deliveries
+    body:
+      encoding: UTF-8
+      string: '{"key":"T7rmrThuYw6vqJSVhrfz","test_event":{}}'
+    headers:
+      Accept:
+      - "*/*"
+      User-Agent:
+      - rest-client/2.1.0 (linux-gnu x86_64) ruby/2.5.1p57
+      Content-Type:
+      - application/json
+      Content-Length:
+      - '46'
+      Accept-Encoding:
+      - gzip;q=1.0,deflate;q=0.6,identity;q=0.3
+  response:
+    status:
+      code: 404
+      message: Not Found
+    headers:
+      Content-Type:
+      - text/html; charset=UTF-8
+      Content-Length:
+      - '0'
+      Connection:
+      - keep-alive
+      Status:
+      - 404 Not Found
+      Vary:
+      - Origin
+      X-Request-Id:
+      - 8ff6d34e-3670-4e19-9f43-6b05669af44b
+      X-Runtime:
+      - '0.001405'
+      Date:
+      - Tue, 25 Aug 2020 07:37:34 GMT
+      X-Powered-By:
+      - Phusion Passenger 5.2.0
+      Server:
+      - nginx/1.12.2 + Phusion Passenger 5.2.0
+    body:
+      encoding: UTF-8
+      string: ''
+    http_version: 
+  recorded_at: Tue, 25 Aug 2020 07:37:34 GMT
+recorded_with: VCR 4.0.0
diff --git a/spec/models/app_spec.rb b/spec/models/app_spec.rb
index 0fd2fc022..eafcc1270 100644
--- a/spec/models/app_spec.rb
+++ b/spec/models/app_spec.rb
@@ -247,4 +247,38 @@
   #     ).to be_nil
   #   end
   # end
+
+  describe "#webhook_url" do
+    it "should validate if the url returns 200 code from POST" do
+      url = "https://www.planningalerts.org.au/deliveries"
+      key = "abc123"
+      # Expect a POST to be done succesfully
+      expect(RestClient).to receive(:post).with(
+        url,
+        { key: key, test_event: {} }.to_json,
+        { content_type: :json }
+      )
+      app = build(:app, webhook_url: url, webhook_key: key)
+      expect(app).to be_valid
+    end
+
+    it "should validate with nil and not try to do a POST" do
+      expect(RestClient).to_not receive(:post)
+      app = build(:app, webhook_url: nil)
+      expect(app).to be_valid
+    end
+
+    it "should not validate if the url returns a 404" do
+      VCR.use_cassette("webhook") do
+        app = build(
+          :app,
+          webhook_url: "https://www.planningalerts.org.au/deliveries"
+        )
+        expect(app).to_not be_valid
+        expect(app.errors[:webhook_url]).to eq(
+          ["returned 404 code when doing POST to https://www.planningalerts.org.au/deliveries"]
+        )
+      end
+    end
+  end
 end