feat: Override for project configs

common/src/config.rs

@@ -1,3 +1,4 @@
+use std::collections::HashMap;
 use std::env;
 use std::fmt;
 use std::fs;
@@ -9,8 +10,9 @@ use std::time::Duration;
 
 use failure::{Backtrace, Context, Fail};
 use log;
+use marshal::processor::PiiConfig;
 // Dsn must be imported from sentry and not sentry-types for compatibility with sentry::init!
-use sentry_types::Dsn;
+use sentry_types::{Dsn, ProjectId};
 use serde::de::DeserializeOwned;
 use serde::ser::Serialize;
 use serde_json;
@@ -291,6 +293,42 @@ impl Default for Sentry {
     }
 }
 
+/// These are config values that the user can modify in the UI.
+#[derive(Debug, Clone, Serialize, Deserialize, Default)]
+#[serde(rename_all = "camelCase")]
+pub struct ProjectConfig {
+    /// URLs that are permitted for cross original JavaScript requests.
+    pub allowed_domains: Vec<String>,
+    /// List of relay public keys that are permitted to access this project.
+    pub trusted_relays: Vec<PublicKey>,
+    /// Configuration for PII stripping.
+    pub pii_config: Option<PiiConfig>,
+}
+
+/// Project state that was hardcoded by the user in the local config
+#[derive(Debug, Clone, Serialize, Deserialize)]
+#[serde(rename_all = "camelCase")]
+pub struct StaticProjectState {
+    /// Indicates that the project is disabled.
+    #[serde(default)]
+    pub disabled: bool,
+    /// A container of known public keys in the project.
+    pub public_keys: HashMap<String, bool>,
+    /// The project's slug if configured.
+    #[serde(default)]
+    pub slug: Option<String>,
+    /// The project's current config.
+    pub config: ProjectConfig,
+}
+
+/// Local overrides for project state
+#[derive(Serialize, Deserialize, Debug, Default)]
+#[serde(default)]
+pub struct Projects {
+    /// The overridden projects
+    pub configs: HashMap<ProjectId, StaticProjectState>,
+}
+
 /// Controls interal reporting to Sentry.
 #[derive(Serialize, Deserialize, Debug, Default)]
 #[serde(default)]
@@ -348,6 +386,8 @@ struct ConfigValues {
     metrics: Metrics,
     #[serde(default)]
     sentry: Sentry,
+    #[serde(default)]
+    projects: Projects,
 }
 
 impl ConfigObject for ConfigValues {
@@ -465,18 +505,18 @@ impl Config {
     }
 
     /// Returns the secret key if set.
-    pub fn secret_key(&self) -> &SecretKey {
-        &self.credentials.as_ref().unwrap().secret_key
+    pub fn secret_key(&self) -> Option<&SecretKey> {
+        self.credentials.as_ref().map(|x| &x.secret_key)
     }
 
     /// Returns the public key if set.
-    pub fn public_key(&self) -> &PublicKey {
-        &self.credentials.as_ref().unwrap().public_key
+    pub fn public_key(&self) -> Option<&PublicKey> {
+        self.credentials.as_ref().map(|x| &x.public_key)
     }
 
     /// Returns the relay ID.
-    pub fn relay_id(&self) -> &RelayId {
-        &self.credentials.as_ref().unwrap().id
+    pub fn relay_id(&self) -> Option<&RelayId> {
+        self.credentials.as_ref().map(|x| &x.id)
     }
 
     /// Returns the upstream target as descriptor.
@@ -621,6 +661,11 @@ impl Config {
             None
         }
     }
+
+    /// Return local project state overrides
+    pub fn projects(&self) -> &Projects {
+        &self.values.projects
+    }
 }
 
 enum ConfigFormat {

server/src/actors/events.rs

@@ -140,7 +140,7 @@ impl ProcessEvent {
             event.id.set_value(Some(Some(self.event_id)))
         }
 
-        let processed_event = match self.project_state.config.pii_config {
+        let processed_event = match self.project_state.config().pii_config {
             Some(ref pii_config) => pii_config.processor().process_root_value(event),
             None => event,
         };

server/src/actors/keys.rs

@@ -194,6 +194,14 @@ impl KeyCache {
             }
         }
 
+        if self.config.credentials().is_none() {
+            error!(
+                "No credentials configured. Relay {} cannot send requests to this relay.",
+                relay_id
+            );
+            return Response::ok((relay_id, None));
+        }
+
         debug!("relay {} public key requested", relay_id);
         if !self.backoff.started() {
             self.backoff.reset();

server/src/actors/project.rs

@@ -12,7 +12,7 @@ use futures::{future::Shared, sync::oneshot, Future};
 use url::Url;
 use uuid::Uuid;
 
-use semaphore_common::{processor::PiiConfig, Config, ProjectId, PublicKey, RetryBackoff};
+use semaphore_common::{Config, ProjectConfig, ProjectId, RetryBackoff, StaticProjectState};
 
 use actors::controller::{Controller, Shutdown, Subscribe, TimeoutError};
 use actors::upstream::{SendQuery, UpstreamQuery, UpstreamRelay};
@@ -63,8 +63,22 @@ impl Project {
         &mut self,
         context: &mut Context<Self>,
     ) -> Response<Arc<ProjectState>, ProjectError> {
+        if let Some(state) = self.config.projects().configs.get(&self.id) {
+            return Response::ok(Arc::new(ProjectState::FromLocalConfig(state.clone())));
+        }
+
+        if self.config.credentials().is_none() {
+            error!(
+                "No credentials configured. Configure project {} in your config.yml",
+                self.id
+            );
+            return Response::ok(Arc::new(ProjectState::missing()));
+        }
+
         if let Some(ref state) = self.state {
-            return Response::ok(state.clone());
+            if !state.outdated(&self.config) {
+                return Response::ok(state.clone());
+            }
         }
 
         debug!("project {} state requested", self.id);
@@ -158,58 +172,60 @@ pub enum PublicKeyStatus {
     Enabled,
 }
 
-/// These are config values that the user can modify in the UI.
-#[derive(Debug, Clone, Serialize, Deserialize, Default)]
-#[serde(rename_all = "camelCase")]
-pub struct ProjectConfig {
-    /// URLs that are permitted for cross original JavaScript requests.
-    pub allowed_domains: Vec<String>,
-    /// List of relay public keys that are permitted to access this project.
-    pub trusted_relays: Vec<PublicKey>,
-    /// Configuration for PII stripping.
-    pub pii_config: Option<PiiConfig>,
-}
-
-/// The project state is a cached server state of a project.
+/// Project state that came from upstream
 #[derive(Debug, Clone, Serialize, Deserialize)]
 #[serde(rename_all = "camelCase")]
-pub struct ProjectState {
+pub struct FetchedProjectState {
     /// The timestamp of when the state was received.
     pub last_fetch: DateTime<Utc>,
     /// The timestamp of when the state was last changed.
-    ///
-    /// This might be `None` in some rare cases like where states
-    /// are faked locally.
-    pub last_change: Option<DateTime<Utc>>,
+    pub last_change: DateTime<Utc>,
     /// Indicates that the project is disabled.
     pub disabled: bool,
     /// A container of known public keys in the project.
     pub public_keys: HashMap<String, bool>,
-    /// The project's slug if available.
-    pub slug: Option<String>,
-    /// The project's current config
+    /// The project's slug.
+    pub slug: String,
+    /// The project's current config.
     pub config: ProjectConfig,
     /// The project state's revision id.
     pub rev: Option<Uuid>,
 }
 
+/// The project state is a cached server state of a project.
+#[derive(Debug, Clone, Serialize, Deserialize)]
+#[serde(untagged)]
+#[cfg_attr(feature = "cargo-clippy", allow(large_enum_variant))]
+pub enum ProjectState {
+    /// Project state that came from upstream
+    FromServer(FetchedProjectState),
+    /// Project state that was set through the local config
+    FromLocalConfig(StaticProjectState),
+    /// Missing project state, used when project state was not fetched yet.
+    #[serde(rename_all = "camelCase")]
+    Missing {
+        /// The timestamp of when the state was checked.
+        last_fetch: DateTime<Utc>,
+    },
+}
+
 impl ProjectState {
     /// Project state for a missing project.
     pub fn missing() -> Self {
-        ProjectState {
+        ProjectState::Missing {
             last_fetch: Utc::now(),
-            last_change: None,
-            disabled: true,
-            public_keys: HashMap::new(),
-            slug: None,
-            config: Default::default(),
-            rev: None,
         }
     }
 
     /// Returns the current status of a key.
     pub fn get_public_key_status(&self, public_key: &str) -> PublicKeyStatus {
-        match self.public_keys.get(public_key) {
+        let public_keys = match *self {
+            ProjectState::FromServer(ref x) => &x.public_keys,
+            ProjectState::FromLocalConfig(ref x) => &x.public_keys,
+            ProjectState::Missing { .. } => return PublicKeyStatus::Unknown,
+        };
+
+        match public_keys.get(public_key) {
             Some(&true) => PublicKeyStatus::Enabled,
             Some(&false) => PublicKeyStatus::Disabled,
             None => PublicKeyStatus::Unknown,
@@ -219,27 +235,43 @@ impl ProjectState {
     /// Returns `true` if the entire project should be considered
     /// disabled (blackholed, deleted etc.).
     pub fn disabled(&self) -> bool {
-        self.disabled
+        match *self {
+            ProjectState::Missing { .. } => true,
+            ProjectState::FromServer(ref x) => x.disabled,
+            ProjectState::FromLocalConfig(ref x) => x.disabled,
+        }
     }
 
     /// Returns whether this state is outdated and needs to be refetched.
     pub fn outdated(&self, config: &Config) -> bool {
-        SystemTime::from(self.last_fetch)
-            .elapsed()
-            .map(|e| match self.slug {
-                Some(_) => e > config.project_cache_expiry(),
-                None => e > config.cache_miss_expiry(),
-            })
-            .unwrap_or(false)
+        let elapsed = |dt| SystemTime::from(dt).elapsed().ok();
+
+        match *self {
+            ProjectState::FromLocalConfig(_) => false,
+            ProjectState::FromServer(ref x) => elapsed(x.last_fetch)
+                .map(|lf| lf > config.project_cache_expiry())
+                .unwrap_or(false),
+            ProjectState::Missing { ref last_fetch } => elapsed(*last_fetch)
+                .map(|lf| lf > config.cache_miss_expiry())
+                .unwrap_or(false),
+        }
     }
 
     /// Returns the project config.
     pub fn config(&self) -> &ProjectConfig {
-        &self.config
+        lazy_static! {
+            static ref DUMMY_CONFIG: ProjectConfig = ProjectConfig::default();
+        }
+
+        match *self {
+            ProjectState::FromServer(ref x) => &x.config,
+            ProjectState::FromLocalConfig(ref x) => &x.config,
+            ProjectState::Missing { .. } => &*DUMMY_CONFIG,
+        }
     }
 
     /// Checks if this origin is allowed for this project.
-    fn is_valid_origin(&self, origin: Option<&Url>) -> bool {
+    pub fn is_valid_origin(&self, origin: Option<&Url>) -> bool {
         // Generally accept any event without an origin.
         let origin = match origin {
             Some(origin) => origin,

server/src/actors/upstream.rs

@@ -169,7 +169,10 @@ impl Handler<Authenticate> for UpstreamRelay {
         let credentials = match self.config.credentials() {
             Some(x) => x,
             None => {
-                warn!("no credentials configured, not authenticating");
+                warn!(
+                    "no credentials configured, not authenticating. \
+                     Other relays will not be able to use this relay."
+                );
                 return Box::new(fut::err(()));
             }
         };

server/src/endpoints/project_configs.rs

@@ -23,7 +23,7 @@ fn get_project_configs(
                 let project_state = project_state.ok()?;
                 // If public key is known (even if rate-limited, which is Some(false)), it has
                 // access to the project config
-                if project_state.config.trusted_relays.contains(&public_key) {
+                if project_state.config().trusted_relays.contains(&public_key) {
                     Some((*project_state).clone())
                 } else {
                     None

src/cli.rs

@@ -298,12 +298,6 @@ pub fn process_event<'a>(matches: &ArgMatches<'a>) -> Result<(), Error> {
 }
 
 pub fn run<'a>(config: Config, _matches: &ArgMatches<'a>) -> Result<(), Error> {
-    if !config.has_credentials() {
-        return Err(err_msg(
-            "relay has no stored credentials. Generate some \
-             with \"semaphore credentials generate\" first.",
-        ));
-    }
     setup::dump_spawn_infos(&config);
     setup::init_metrics(&config)?;
     semaphore_server::run(config)?;

src/setup.rs

@@ -19,15 +19,30 @@ pub fn dump_spawn_infos(config: &Config) {
         "launching relay from config folder {}",
         config.path().display()
     );
-    info!("  relay id: {}", config.relay_id());
-    info!("  public key: {}", config.public_key());
+
+    match config.relay_id() {
+        Some(id) => info!("  relay id: {}", id),
+        None => info!("  relay id: -"),
+    };
+    match config.public_key() {
+        Some(key) => info!("  public key: {}", key),
+        None => info!("  public key: -"),
+    };
+
     info!("  log level: {}", config.log_level_filter());
 }
 
 /// Dumps out credential info.
 pub fn dump_credentials(config: &Config) {
-    println!("  relay id: {}", config.relay_id());
-    println!("  public key: {}", config.public_key());
+    match config.relay_id() {
+        Some(id) => println!("  relay id: {}", id),
+        None => println!("  relay id: -"),
+    };
+
+    match config.public_key() {
+        Some(key) => println!("  public key: {}", key),
+        None => println!("  public key: -"),
+    };
 }
 
 /// Initialize the logging system.

tests/integration/conftest.py

@@ -1,3 +1,4 @@
+import datetime
 import os
 import uuid
 import pytest
@@ -106,7 +107,7 @@ def fail(e):
     @request.addfinalizer
     def reraise_test_failures():
         if sentry.test_failures:
-            raise AssertionError(f"Exceptions happened in mini_sentry: {test_failures}")
+            raise AssertionError(f"Exceptions happened in mini_sentry: {sentry.test_failures}")
 
     server = WSGIServer(application=app, threaded=True)
     server.start()
@@ -172,8 +173,8 @@ def basic_project_config(self):
             "publicKeys": {self.dsn_public_key: True},
             "rev": "5ceaea8c919811e8ae7daae9fe877901",
             "disabled": False,
-            "lastFetch": "2018-08-24T17:29:04.426Z",
-            "lastChange": "2018-07-27T12:27:01.481Z",
+            "lastFetch": datetime.datetime.now().isoformat(),
+            "lastChange": datetime.datetime.now().isoformat(),
             "config": {
                 "allowedDomains": ["*"],
                 "trustedRelays": list(self.iter_public_keys()),