diff --git a/CHANGELOG.md b/CHANGELOG.md
index fda5a5044..b243b1fed 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,19 +1,17 @@
-# Release 0.6.0-dev
-
-### New features since last release
-
-### Breaking changes
-
-### Improvements
-
-### Documentation
+# Release 0.5.1
 
 ### Bug fixes
 
+* Fixed a bug where backend keyword arguments, such as `backend_options`
+  and `noise_model`, were being passed to backends that did not support it.
+  [#51](https://github.com/XanaduAI/pennylane-qiskit/pull/51)
+
 ### Contributors
 
 This release contains contributions from (in alphabetical order):
 
+Josh Izaac
+
 ---
 
 # Release 0.5.0
diff --git a/pennylane_qiskit/_version.py b/pennylane_qiskit/_version.py
index 36b932532..6ded37b2b 100644
--- a/pennylane_qiskit/_version.py
+++ b/pennylane_qiskit/_version.py
@@ -16,4 +16,4 @@
    Version number (major.minor.patch[-label])
 """
 
-__version__ = "0.6.0-dev"
+__version__ = "0.5.1"
diff --git a/pennylane_qiskit/aer.py b/pennylane_qiskit/aer.py
index e2c5133a7..7a907a7e8 100644
--- a/pennylane_qiskit/aer.py
+++ b/pennylane_qiskit/aer.py
@@ -54,7 +54,6 @@ class AerDevice(QiskitDevice):
     Args:
         wires (int): The number of qubits of the device
         backend (str): the desired backend
-        noise_model (NoiseModel): NoiseModel Object from ``qiskit.providers.aer.noise``
         shots (int): number of circuit evaluations/random samples used
             to estimate expectation values and variances of observables
 
@@ -62,22 +61,12 @@ class AerDevice(QiskitDevice):
         name (str): The name of the circuit. Default ``'circuit'``.
         compile_backend (BaseBackend): The backend used for compilation. If you wish
             to simulate a device compliant circuit, you can specify a backend here.
+        noise_model (NoiseModel): NoiseModel Object from ``qiskit.providers.aer.noise``
     """
 
     # pylint: disable=too-many-arguments
 
     short_name = "qiskit.aer"
 
-    def __init__(self, wires, shots=1024, backend="qasm_simulator", noise_model=None, **kwargs):
-        super().__init__(wires, qiskit.Aer, backend=backend, shots=shots, **kwargs)
-        self._noise_model = noise_model
-
-    def run(self, qobj):
-        """Run the compiled circuit, and query the result."""
-        self._current_job = self.backend.run(
-            qobj, noise_model=self._noise_model, backend_options=self.kwargs
-        )
-        result = self._current_job.result()
-
-        if self.backend_name in self._state_backends:
-            self._state = self._get_state(result)
+    def __init__(self, wires, shots=1024, backend="qasm_simulator", **kwargs):
+        super().__init__(wires, provider=qiskit.Aer, backend=backend, shots=shots, **kwargs)
diff --git a/pennylane_qiskit/ibmq.py b/pennylane_qiskit/ibmq.py
index ec8792319..2ce41eb09 100644
--- a/pennylane_qiskit/ibmq.py
+++ b/pennylane_qiskit/ibmq.py
@@ -65,6 +65,8 @@ class IBMQDevice(QiskitDevice):
             variable ``IBMQX_TOKEN`` is used.
         ibmqx_url (str): The IBM Q URL. If not provided, the environment
             variable ``IBMQX_URL`` is used, followed by the default URL.
+        noise_model (NoiseModel): NoiseModel Object from ``qiskit.providers.aer.noise``.
+            Only applicable for simulator backends.
     """
 
     short_name = "qiskit.ibmq"
diff --git a/pennylane_qiskit/qiskit_device.py b/pennylane_qiskit/qiskit_device.py
index 34940fb6a..c99b36517 100644
--- a/pennylane_qiskit/qiskit_device.py
+++ b/pennylane_qiskit/qiskit_device.py
@@ -32,6 +32,7 @@
 import abc
 from collections import OrderedDict
 import functools
+import inspect
 import itertools
 
 import numpy as np
@@ -138,9 +139,6 @@ def __init__(self, wires, provider, backend, shots=1024, **kwargs):
 
         self.provider = provider
         self.backend_name = backend
-        self.compile_backend = kwargs.get("compile_backend")
-        self.kwargs = kwargs
-
         self._capabilities["backend"] = [b.name() for b in self.provider.backends()]
 
         # check that backend exists
@@ -173,6 +171,24 @@ def __init__(self, wires, provider, backend, shots=1024, **kwargs):
         # job execution options
         self.memory = False  # do not return samples, just counts
 
+        # determine if backend supports backend options and noise models,
+        # and properly put together backend run arguments
+        s = inspect.signature(b.run)
+        self.run_args = {}
+        self.compile_backend = None
+
+        if "compile_backend" in kwargs:
+            self.compile_backend = kwargs.pop("compile_backend")
+
+        if "noise_model" in kwargs:
+            if "noise_model" in s.parameters:
+                self.run_args["noise_model"] = kwargs.pop("noise_model")
+            else:
+                raise ValueError("Backend {} does not support noisy simulations".format(backend))
+
+        if "backend_options" in s.parameters:
+            self.run_args["backend_options"] = kwargs
+
         self.reset()
 
     @property
@@ -228,7 +244,7 @@ def compile(self):
 
     def run(self, qobj):
         """Run the compiled circuit, and query the result."""
-        self._current_job = self.backend.run(qobj, backend_options=self.kwargs)
+        self._current_job = self.backend.run(qobj, **self.run_args)
         result = self._current_job.result()
 
         if self.backend_name in self._state_backends:
diff --git a/tests/test_integration.py b/tests/test_integration.py
index d72207092..5064d3548 100644
--- a/tests/test_integration.py
+++ b/tests/test_integration.py
@@ -66,3 +66,33 @@ def circuit(x, y, z):
             return qml.expval(qml.PauliZ(0))
 
         assert np.allclose(circuit(a, b, c), np.cos(a) * np.sin(b), **tol)
+
+
+class TestKeywordArguments:
+    """Test keyword argument logic is correct"""
+
+    @pytest.mark.parametrize("d", pldevices)
+    def test_compile_backend(self, d):
+        """Test that the compile backend argument is properly
+        extracted"""
+        dev = qml.device(d[0], wires=2, compile_backend="test value")
+        assert dev.compile_backend == "test value"
+
+    def test_noise_model(self):
+        """Test that the noise model argument is properly
+        extracted if the backend supports it"""
+        dev = qml.device("qiskit.aer", wires=2, noise_model="test value")
+        assert dev.run_args["noise_model"] == "test value"
+
+    def test_invalid_noise_model(self):
+        """Test that the noise model argument causes an exception to be raised
+        if the backend does not support it"""
+        with pytest.raises(ValueError, match="does not support noisy simulations"):
+            dev = qml.device("qiskit.basicaer", wires=2, noise_model="test value")
+
+    @pytest.mark.parametrize("d", pldevices)
+    def test_overflow_backend_options(self, d):
+        """Test all overflow backend options are extracted"""
+        dev = qml.device(d[0], wires=2, k1="v1", k2="v2")
+        assert dev.run_args["backend_options"]["k1"] == "v1"
+        assert dev.run_args["backend_options"]["k2"] == "v2"