feat: Add test(should_fail) attribute for tests that are meant to fail

crates/lsp/src/lib.rs

@@ -199,7 +199,7 @@ fn on_code_lens_request(
         let tests = context
             .get_all_test_functions_in_crate_matching(&crate_id, FunctionNameMatch::Anything);
 
-        for (func_name, func_id) in tests {
+        for (func_name, func_id, _should_fail) in tests {
             let location = context.function_meta(&func_id).name.location;
             let file_id = location.file;
 

crates/nargo_cli/src/cli/test_cmd.rs

@@ -89,12 +89,20 @@
     let writer = StandardStream::stderr(ColorChoice::Always);
     let mut writer = writer.lock();
 
-    for (test_name, test_function) in test_functions {
+    for (test_name, test_function, should_fail) in test_functions {
         write!(writer, "[{}] Testing {test_name}... ", package.name)
             .expect("Failed to write to stdout");
         writer.flush().expect("Failed to flush writer");
 
-        match run_test(backend, &test_name, test_function, &context, show_output, compile_options) {
+        match run_test(
+            backend,
+            &test_name,
+            test_function,
+            should_fail,
+            &context,
+            show_output,
+            compile_options,
+        ) {
             Ok(_) => {
                 writer
                     .set_color(ColorSpec::new().set_fg(Some(Color::Green)))
@@ -124,29 +132,43 @@
     backend: &B,
     test_name: &str,
     main: FuncId,
+    should_fail: bool,
     context: &Context,
     show_output: bool,
     config: &CompileOptions,
 ) -> Result<(), CliError<B>> {
-    let mut program = compile_no_check(context, config, main).map_err(|err| {
-        noirc_errors::reporter::report_all(&context.file_manager, &[err], config.deny_warnings);
-        CliError::Generic(format!("Test '{test_name}' failed to compile"))
-    })?;
-
-    // Note: We could perform this test using the unoptimized ACIR as generated by `compile_no_check`.
-    program.circuit = optimize_circuit(backend, program.circuit).unwrap().0;
-
-    // Run the backend to ensure the PWG evaluates functions like std::hash::pedersen,
-    // otherwise constraints involving these expressions will not error.
-    match execute_circuit(backend, program.circuit, WitnessMap::new(), show_output) {
-        Ok(_) => Ok(()),
-        Err(error) => {
-            let writer = StandardStream::stderr(ColorChoice::Always);
-            let mut writer = writer.lock();
-            writer.set_color(ColorSpec::new().set_fg(Some(Color::Red))).ok();
-            writeln!(writer, "failed").ok();
-            writer.reset().ok();
-            Err(error.into())
+    if !should_fail {
+        let mut program = compile_no_check(context, config, main).map_err(|err| {
+            noirc_errors::reporter::report_all(&context.file_manager, &[err], config.deny_warnings);
+            CliError::Generic(format!("Test '{test_name}' failed to compile"))
+        })?;
+
+        // Note: We could perform this test using the unoptimized ACIR as generated by `compile_no_check`.
+        program.circuit = optimize_circuit(backend, program.circuit).unwrap().0;
+
+        // Run the backend to ensure the PWG evaluates functions like std::hash::pedersen,
+        // otherwise constraints involving these expressions will not error.
+        match execute_circuit(backend, program.circuit, WitnessMap::new(), show_output) {
+            Ok(_) => Ok(()),
+            Err(error) => {
+                let writer = StandardStream::stderr(ColorChoice::Always);
+                let mut writer = writer.lock();
+                writer.set_color(ColorSpec::new().set_fg(Some(Color::Red))).ok();
+                writeln!(writer, "failed").ok();
+                writer.reset().ok();
+                Err(error.into())
+            }
+        }
+    } else {
+        let program = compile_no_check(context, config, main);
+        if program.is_err() {
+            return Ok(());
+        }
+        let mut program = program.unwrap();
+        program.circuit = optimize_circuit(backend, program.circuit).unwrap().0;
+        match execute_circuit(backend, program.circuit, WitnessMap::new(), show_output) {
+            Ok(_) => Err(CliError::Generic(format!("Test '{test_name}' should fail"))),
+            Err(_) => Ok(()),
         }
     }
 }

crates/noirc_frontend/src/ast/function.rs

@@ -83,7 +83,7 @@ impl From<FunctionDefinition> for NoirFunction {
         let kind = match fd.attribute {
             Some(Attribute::Builtin(_)) => FunctionKind::Builtin,
             Some(Attribute::Foreign(_)) => FunctionKind::LowLevel,
-            Some(Attribute::Test) => FunctionKind::Normal,
+            Some(Attribute::Test(_)) => FunctionKind::Normal,
             Some(Attribute::Oracle(_)) => FunctionKind::Oracle,
             Some(Attribute::Deprecated(_)) | None => FunctionKind::Normal,
             Some(Attribute::Custom(_)) => FunctionKind::Normal,

crates/noirc_frontend/src/hir/def_map/mod.rs

@@ -129,12 +129,22 @@ impl CrateDefMap {
     pub fn get_all_test_functions<'a>(
         &'a self,
         interner: &'a NodeInterner,
-    ) -> impl Iterator<Item = FuncId> + 'a {
+    ) -> impl Iterator<Item = (FuncId, bool)> + 'a {
         self.modules.iter().flat_map(|(_, module)| {
             module
                 .value_definitions()
                 .filter_map(|id| id.as_function())
-                .filter(|id| interner.function_meta(id).attributes == Some(Attribute::Test))
+                .filter(|id| {
+                    interner.function_meta(id).attributes == Some(Attribute::Test(true))
+                        || interner.function_meta(id).attributes == Some(Attribute::Test(false))
+                })
+                .map(|id| {
+                    if interner.function_meta(&id).attributes == Some(Attribute::Test(true)) {
+                        (id, true)
+                    } else {
+                        (id, false)
+                    }
+                })
         })
     }
 

crates/noirc_frontend/src/hir/mod.rs

@@ -107,22 +107,21 @@ impl Context {
         &self,
         crate_id: &CrateId,
         pattern: FunctionNameMatch,
-    ) -> Vec<(String, FuncId)> {
+    ) -> Vec<(String, FuncId, bool)> {
         let interner = &self.def_interner;
         let def_map = self.def_map(crate_id).expect("The local crate should be analyzed already");
 
         def_map
             .get_all_test_functions(interner)
-            .filter_map(|id| {
+            .filter_map(|(id, not_fail)| {
                 let fully_qualified_name = self.fully_qualified_function_name(crate_id, &id);
                 match &pattern {
-                    FunctionNameMatch::Anything => Some((fully_qualified_name, id)),
-                    FunctionNameMatch::Exact(pattern) => {
-                        (&fully_qualified_name == pattern).then_some((fully_qualified_name, id))
-                    }
-                    FunctionNameMatch::Contains(pattern) => {
-                        fully_qualified_name.contains(pattern).then_some((fully_qualified_name, id))
-                    }
+                    FunctionNameMatch::Anything => Some((fully_qualified_name, id, not_fail)),
+                    FunctionNameMatch::Exact(pattern) => (&fully_qualified_name == pattern)
+                        .then_some((fully_qualified_name, id, not_fail)),
+                    FunctionNameMatch::Contains(pattern) => fully_qualified_name
+                        .contains(pattern)
+                        .then_some((fully_qualified_name, id, not_fail)),
                 }
             })
             .collect()

crates/noirc_frontend/src/hir/resolution/resolver.rs

@@ -700,7 +700,9 @@
             self.push_err(ResolverError::DistinctNotAllowed { ident: func.name_ident().clone() });
         }
 
-        if attributes == Some(Attribute::Test) && !parameters.is_empty() {
+        if (attributes == Some(Attribute::Test(true)) || attributes == Some(Attribute::Test(false)))
+            && !parameters.is_empty()
+        {
             self.push_err(ResolverError::TestFunctionHasParameters {
                 span: func.name_ident().span(),
             });
@@ -1934,7 +1936,7 @@
                 println(f"I want to print {0}");
 
                 let new_val = 10;
                 println(f"randomstring{new_val}{new_val}");
             }
             fn println<T>(x : T) -> T {
                 x

crates/noirc_frontend/src/lexer/token.rs

@@ -325,7 +325,8 @@
     Builtin(String),
     Oracle(String),
     Deprecated(Option<String>),
-    Test,
+    // Boolean flag that represents if the test should fail
+    Test(bool),
     Custom(String),
 }
 
@@ -335,7 +336,8 @@
             Attribute::Foreign(ref k) => write!(f, "#[foreign({k})]"),
             Attribute::Builtin(ref k) => write!(f, "#[builtin({k})]"),
             Attribute::Oracle(ref k) => write!(f, "#[oracle({k})]"),
-            Attribute::Test => write!(f, "#[test]"),
+            Attribute::Test(false) => write!(f, "#[test]"),
+            Attribute::Test(true) => write!(f, "#[test(should_fail)]"),
             Attribute::Deprecated(None) => write!(f, "#[deprecated]"),
             Attribute::Deprecated(Some(ref note)) => write!(f, r#"#[deprecated("{note}")]"#),
             Attribute::Custom(ref k) => write!(f, "#[{k}]"),
@@ -391,7 +393,17 @@
 
                 Attribute::Deprecated(name.trim_matches('"').to_string().into())
             }
-            ["test"] => Attribute::Test,
+            ["test"] => Attribute::Test(false),
+            ["test", name] => {
+                if name != &"should_fail" {
+                    return Err(LexerErrorKind::MalformedFuncAttribute {
+                        span,
+                        found: word.to_owned(),
+                    });
+                }
+
+                Attribute::Test(true)
+            }
             tokens => {
                 tokens.iter().try_for_each(|token| validate(token))?;
                 Attribute::Custom(word.to_owned())
@@ -431,7 +443,7 @@
             Attribute::Builtin(string) => string,
             Attribute::Oracle(string) => string,
             Attribute::Deprecated(Some(string)) => string,
-            Attribute::Test | Attribute::Deprecated(None) => "",
+            Attribute::Test(_) | Attribute::Deprecated(None) => "",
             Attribute::Custom(string) => string,
         }
     }
@@ -496,7 +508,7 @@
             Keyword::Field => write!(f, "Field"),
             Keyword::Fn => write!(f, "fn"),
             Keyword::For => write!(f, "for"),
             Keyword::FormatString => write!(f, "fmtstr"),
             Keyword::Global => write!(f, "global"),
             Keyword::If => write!(f, "if"),
             Keyword::Impl => write!(f, "impl"),
@@ -538,7 +550,7 @@
             "Field" => Keyword::Field,
             "fn" => Keyword::Fn,
             "for" => Keyword::For,
             "fmtstr" => Keyword::FormatString,
             "global" => Keyword::Global,
             "if" => Keyword::If,
             "impl" => Keyword::Impl,