From aa3dad4d175eefc85ec5ae9e94913d3d049aeee4 Mon Sep 17 00:00:00 2001
From: colin <colin@thinkr.fr>
Date: Wed, 30 Sep 2020 15:38:28 +0200
Subject: [PATCH 1/6] This function will perform a sanity check on `setup` and
 `solution` chunks

---
 DESCRIPTION        |   3 +-
 NAMESPACE          |   1 +
 R/check_exercise.R | 107 +++++++++++++++++++++++++++++++++++++++++++++
 3 files changed, 110 insertions(+), 1 deletion(-)
 create mode 100644 R/check_exercise.R

diff --git a/DESCRIPTION b/DESCRIPTION
index 0eb1a142d..474cfde4c 100644
--- a/DESCRIPTION
+++ b/DESCRIPTION
@@ -39,7 +39,8 @@ Imports:
     renv (>= 0.8.0),
     curl,
     promises,
-    digest
+    digest,
+    cli
 Remotes:
     rstudio/htmltools,
     rstudio/shinytest
diff --git a/NAMESPACE b/NAMESPACE
index e1995ca9c..a1d98505d 100644
--- a/NAMESPACE
+++ b/NAMESPACE
@@ -17,6 +17,7 @@ S3method(question_ui_initialize,default)
 S3method(question_ui_try_again,default)
 export(answer)
 export(available_tutorials)
+export(check_exercise)
 export(correct)
 export(disable_all_tags)
 export(duplicate_env)
diff --git a/R/check_exercise.R b/R/check_exercise.R
new file mode 100644
index 000000000..3f912455b
--- /dev/null
+++ b/R/check_exercise.R
@@ -0,0 +1,107 @@
+#' Check the code from an Exercise
+#'
+#' This function will take all the chunks with a label that matches `setup` or
+#' `-solution`, put them in a separate script and try to run them all.
+#' This allows teachers to check that their setup and solution chunks
+#' contain valid code.
+#'
+#' @param path Path to the Markdown file containing the RMarkdown.
+#' @param verbose Should the test output information on the console?
+#'
+#' @return TRUE or FALSE invisibly.
+#' @export
+#'
+#' @examples
+#' if (interactive()){
+#'   check_exercise("sandbox/sandbox.Rmd")
+#' }
+check_exercise <- function(
+  path,
+  verbose = TRUE
+){
+  # Create a file that will receive the chunks
+  tempr <- tempfile(fileext = ".R")
+  write_there <- function(x){
+    write(
+      x,
+      tempr,
+      append = TRUE
+    )
+  }
+
+  # Getting the old chunk hook, and reset it on exit
+  hook_old <- knitr::knit_hooks$get("chunk")
+  on.exit(
+    knitr::knit_hooks$set(chunk = hook_old)
+  )
+
+  # Setting a hook on every chunk
+  knitr::knit_hooks$set(chunk = function(x, options) {
+    # It the chunk is a setup or solution chunk, we add it to
+    # the temp .R script
+    if(grepl("(\\-*setup|\\-solution)$", options$label)){
+      write_there(
+        sprintf(
+          "# %s ----",
+          options$label
+        )
+      )
+      if (verbose){
+        write_there(
+          sprintf(
+            'cli::cat_rule("Checking chunk %s")',
+            options$label
+          )
+        )
+      }
+      write_there(
+        options$code
+      )
+      if (verbose){
+        write_there(
+          'cli::cat_bullet("Ok", col = "green", bullet = "tick");cli::cat_line(" ")'
+        )
+      }
+    }
+    hook_old(x, options)
+  })
+
+  # Trick knitr into thinking we are in a shiny_prerender context
+  hook_runtime<- knitr::knit_hooks$get("rmarkdown.runtime")
+  on.exit(
+    knitr::knit_hooks$set("rmarkdown.runtime" = hook_runtime)
+  )
+  knitr::opts_knit$set(rmarkdown.runtime = "shiny_prerendered")
+
+  # We don't need the knitted output so we unlink it immediatly
+  unlink(knitr::knit(path, quiet = TRUE))
+
+  # Trying to source the temp R script
+  tc <- try( source(tempr) )
+  unlink(tempr)
+
+  cli::cat_line(" ")
+  cli::cat_rule("Check finished")
+  cli::cat_line(" ")
+
+  if (
+    inherits(tc, "try-error")
+  ){
+    cli::cat_bullet(
+      "Running setup and/or solution chunks failed",
+      col = "red",
+      bullet = "cross"
+    )
+    return(invisible(FALSE))
+  }
+
+  cli::cat_bullet(
+    "Successfully run setup and/or solution chunks",
+    col = "green",
+    bullet = "tick"
+  )
+
+  cli::cat_line(" ")
+
+  return(invisible(TRUE))
+}

From 3780415f483c10cd814f187dc9092f8d210aac7a Mon Sep 17 00:00:00 2001
From: colin <colin@thinkr.fr>
Date: Wed, 30 Sep 2020 15:49:00 +0200
Subject: [PATCH 2/6] added doc

---
 man/check_exercise.Rd | 27 +++++++++++++++++++++++++++
 1 file changed, 27 insertions(+)
 create mode 100644 man/check_exercise.Rd

diff --git a/man/check_exercise.Rd b/man/check_exercise.Rd
new file mode 100644
index 000000000..6cc4218c9
--- /dev/null
+++ b/man/check_exercise.Rd
@@ -0,0 +1,27 @@
+% Generated by roxygen2: do not edit by hand
+% Please edit documentation in R/check_exercise.R
+\name{check_exercise}
+\alias{check_exercise}
+\title{Check the code from an Exercise}
+\usage{
+check_exercise(path, verbose = TRUE)
+}
+\arguments{
+\item{path}{Path to the Markdown file containing the RMarkdown.}
+
+\item{verbose}{Should the test output information on the console?}
+}
+\value{
+TRUE or FALSE invisibly.
+}
+\description{
+This function will take all the chunks with a label that matches \code{setup} or
+\code{-solution}, put them in a separate script and try to run them all.
+This will allow teachers to check that their setup and solution chunks
+contain valid code.
+}
+\examples{
+if (interactive()){
+  check_exercise("sandbox/sandbox.Rmd")
+}
+}

From d7ad3eaa322cdae7462db23003fede1cdcd5cbc2 Mon Sep 17 00:00:00 2001
From: colin <colin@thinkr.fr>
Date: Fri, 23 Oct 2020 16:50:05 +0200
Subject: [PATCH 3/6] Exercise checker based on evaluate_code

---
 R/check_exercise.R       | 439 ++++++++++++++++++++++++++++++++-------
 sandbox/sandbox-exos.Rmd | 124 +++++++++++
 2 files changed, 490 insertions(+), 73 deletions(-)
 create mode 100644 sandbox/sandbox-exos.Rmd

diff --git a/R/check_exercise.R b/R/check_exercise.R
index 3f912455b..58a067b3e 100644
--- a/R/check_exercise.R
+++ b/R/check_exercise.R
@@ -1,3 +1,69 @@
+
+#######
+# Functions extracted from the knitr hook
+#######
+
+# helper to check for an exercise chunk
+is_exercise_chunk <- function(options) {
+  isTRUE(options[["exercise"]])
+}
+
+# helper to find chunks that name a chunk as their setup chunk
+exercise_chunks_for_setup_chunk <- function(label) {
+  label_query <- paste0("knitr::all_labels(exercise.setup == '", label, "')")
+  eval(parse(text = label_query))
+}
+
+# helper to check for an exercise support chunk
+is_exercise_support_chunk <- function(
+  options,
+  type = c(
+    "setup",
+    "hint",
+    "hint-\\d+",
+    "solution",
+    "error-check",
+    "code-check",
+    "check"
+  )
+) {
+  support_regex <- paste0("-(", paste(type, collapse = "|"), ")$")
+  if (grepl(support_regex, options$label)) {
+    exercise_label <- sub(support_regex, "", options$label)
+    label_query <- "knitr::all_labels(exercise == TRUE)"
+    all_exercise_labels <- eval(parse(text = label_query))
+    exercise_label %in% all_exercise_labels
+  }
+  else if ("setup" %in% type) {
+    # look for another chunk which names this as it's setup chunk or if it has `exercise.setup`
+    # this second condition is for support chunks that isn't referenced by an exercise yet
+    # but is part of a chain and should be stored as a setup chunk
+    is_referenced <- length(exercise_chunks_for_setup_chunk(options$label)) > 0
+    if (is_referenced) {
+      find_parent_setup_chunks(options) # only used to check for cycles; the return value is not useful here
+      TRUE
+    } else {
+      # if this looks like a setup chunk, but no one references it, error
+      if (is.null(options$exercise) && !is.null(options$exercise.setup)) {
+        stop(
+          "Chunk '", options$label, "' is not being used by any exercise or exercise setup chunk.\n",
+          "Please remove chunk '", options$label, "' or reference '", options$label, "' with `exercise.setup = '", options$label, "'`",
+          call. = FALSE)
+      }
+      # just a random chunk
+      FALSE
+    }
+  }
+  else {
+    FALSE
+  }
+}
+
+is_exercise_setup_chunk <- function(label) {
+  grepl("-setup$", label) ||
+    (length(exercise_chunks_for_setup_chunk(label)) > 0)
+}
+
 #' Check the code from an Exercise
 #'
 #' This function will take all the chunks with a label that matches `setup` or
@@ -6,7 +72,6 @@
 #' contain valid code.
 #'
 #' @param path Path to the Markdown file containing the RMarkdown.
-#' @param verbose Should the test output information on the console?
 #'
 #' @return TRUE or FALSE invisibly.
 #' @export
@@ -16,92 +81,320 @@
 #'   check_exercise("sandbox/sandbox.Rmd")
 #' }
 check_exercise <- function(
-  path,
-  verbose = TRUE
+  path
 ){
-  # Create a file that will receive the chunks
-  tempr <- tempfile(fileext = ".R")
-  write_there <- function(x){
-    write(
-      x,
-      tempr,
-      append = TRUE
+
+  path <- normalizePath(path)
+
+  # set global tutorial option which we can use as a basis for hooks
+  # (this is so we don't collide with hooks set by the user or
+  # by other packages or Rmd output formats)
+  knitr::opts_chunk$set(tutorial = TRUE)
+
+
+  # The goal of this function is to check that a given
+  # Rmd is correct, in the sense that the solution can be run
+  # using the parameters that has been provided.
+  # In other words, it tries to mimic the evaluation of an exercise
+  # where the student's input == the teacher's solution
+  #
+  # Here is  how it achieves this
+  # - parse a given Rmd
+  # - create groups of chunk, based on their label:
+  # setup, solution, and checkers
+  # - reproduce the knitr context of evaluation
+  # - for each group, run an evaluate_exercise(),
+  # using the teacher's solution as an input
+  # - If this evaluate_exercise() works, then the solution
+  # is correct
+
+  # Setting learnr options
+  learnr::tutorial_options()
+
+  # When exiting the fun, we delete all the code store
+  # inside the knitr environment
+  on.exit({
+    knitr::knit_code$restore()
+  })
+
+  # Splitting the file using knitr.
+  # This will register code inside  `knitr::knit_code`
+  # The results is a list containing all the elements from the Rmd
+  # (i.e code + title + yaml)
+  res <- knitr:::split_file(
+    xfun::read_utf8(path),
+    patterns = knitr::all_patterns$md
+  )
+
+  # Given that we are in learnr, we only want the chunks that have a label
+  # This should have the same length as knitr::knit_code$get()
+  # For some reasons the result of split_file is not the same as what
+  # knitr::knit_code$get() returns
+  usefull_chunks <- res[
+    ! vapply(
+      res,
+      function(.x) is.null(.x$params$label),
+      FUN.VALUE = logical(1)
+    )
+  ]
+
+  # Here length(knitr::knit_code$get()) == length(usefull_chunks)
+
+  # Setting the names of the chunks using the labels
+  names(usefull_chunks) <- vapply(
+    usefull_chunks,
+    function(.x) .x$params$label,
+    FUN.VALUE = character(1)
+  )
+  # Here all(names(knitr::knit_code$get()) == names(usefull_chunks))
+
+  # As the result of split_file does not contain the code, we add a code
+  # elements
+  for (i in seq_along(usefull_chunks)){
+
+    usefull_chunks[[i]]$code <- knitr::knit_code$get(
+      usefull_chunks[[i]]$params$label
     )
+
   }
 
-  # Getting the old chunk hook, and reset it on exit
-  hook_old <- knitr::knit_hooks$get("chunk")
-  on.exit(
-    knitr::knit_hooks$set(chunk = hook_old)
+  # We extract the default setup chunk, and remove it from the list
+  setup_chunk <- usefull_chunks[["setup"]]
+  usefull_chunks[["setup"]] <- NULL
+
+  # At this point, all `usefull_chunks` contain two elements:
+  # - params, which are the chunk parameters as a lit
+  # - code, which contain the code and the parameters as attributes
+  # We now need to build N exercise objects, we N is the number of
+  # groups of chunks inside the Rmd
+  #
+  # Once we have successfully built the exercise object, we can
+  # safely pass it to `evaluate_exercise`
+
+  # Grep the "titles" of the groups. The length of this  object will
+  # correspond to the number of time we'll build an exercise object
+  # and evaluate_exercise() it
+  chunks_ <- grep(
+    "(-solution)|(-check)|(-hint)|(-setup)",
+    names(usefull_chunks),
+    invert = TRUE,
+    value = TRUE
   )
 
-  # Setting a hook on every chunk
-  knitr::knit_hooks$set(chunk = function(x, options) {
-    # It the chunk is a setup or solution chunk, we add it to
-    # the temp .R script
-    if(grepl("(\\-*setup|\\-solution)$", options$label)){
-      write_there(
-        sprintf(
-          "# %s ----",
-          options$label
-        )
-      )
-      if (verbose){
-        write_there(
-          sprintf(
-            'cli::cat_rule("Checking chunk %s")',
-            options$label
-          )
-        )
+
+  for (i in seq_along(usefull_chunks)){
+    # Get all the knitr options
+    options <- knitr::opts_chunk$get()
+    # If any of this options is overriden at the chunk level,
+    # we override it. The idea here is to be able to build a chunk
+    # with all the options
+    for (
+      param in names(usefull_chunks[[i]]$params)
+    ){
+      options[[
+        param
+      ]] <- usefull_chunks[[i]]$params[[
+        param
+      ]]
+    }
+
+    # We set some options based on
+    # https://github.com/rstudio/learnr/blob/master/R/knitr-hooks.R#L142
+    exercise_chunk <- is_exercise_chunk(options)
+    exercise_support_chunk <- is_exercise_support_chunk(options)
+    exercise_setup_chunk <- is_exercise_support_chunk(options, type = "setup")
+
+    if (exercise_chunk) {
+      learnr:::initialize_tutorial()
+      options$echo <- TRUE
+      options$include <- TRUE
+      options$highlight <- FALSE
+      options$comment <- NA
+      if (!is.null(options$exercise.eval)){
+        options$eval <- options$exercise.eval
+      } else {
+        options$eval <- FALSE
       }
-      write_there(
-        options$code
+
+    }
+
+    if (exercise_support_chunk) {
+      options$echo <- TRUE
+      options$include <- TRUE
+      options$eval <- FALSE
+      options$highlight <- FALSE
+    }
+
+    if (
+      is_exercise_support_chunk(
+        options,
+        type = c("code-check", "error-check", "check")
       )
-      if (verbose){
-        write_there(
-          'cli::cat_bullet("Ok", col = "green", bullet = "tick");cli::cat_line(" ")'
-        )
+    ) {
+      options$include <- FALSE
+    }
+
+    if (exercise_setup_chunk) {
+      # figure out the default behavior
+      exercise_eval <- knitr::opts_chunk$get('exercise.eval')
+      if (is.null(exercise_eval))
+        exercise_eval <- FALSE
+
+      # look for chunks that name this as their setup chunk
+      labels <- exercise_chunks_for_setup_chunk(options$label)
+      if (grepl("-setup$", options$label))
+        labels <- c(labels, sub("-setup$", "", options$label))
+      labels <- paste0('"', labels, '"')
+      labels <- paste0('c(', paste(labels, collapse = ', ') ,')')
+      label_query <- paste0("knitr::all_labels(label %in% ", labels, ", ",
+                            "identical(exercise.eval, ", !exercise_eval, "))")
+
+      default_reversed <- length(eval(parse(text = label_query))) > 0
+
+      if (default_reversed) {
+        exercise_eval <- !exercise_eval
       }
+
+
+      # set the eval property as appropriate
+      options$eval <- exercise_eval
+      options$echo <- FALSE
     }
-    hook_old(x, options)
-  })
 
-  # Trick knitr into thinking we are in a shiny_prerender context
-  hook_runtime<- knitr::knit_hooks$get("rmarkdown.runtime")
-  on.exit(
-    knitr::knit_hooks$set("rmarkdown.runtime" = hook_runtime)
-  )
-  knitr::opts_knit$set(rmarkdown.runtime = "shiny_prerendered")
-
-  # We don't need the knitted output so we unlink it immediatly
-  unlink(knitr::knit(path, quiet = TRUE))
-
-  # Trying to source the temp R script
-  tc <- try( source(tempr) )
-  unlink(tempr)
-
-  cli::cat_line(" ")
-  cli::cat_rule("Check finished")
-  cli::cat_line(" ")
-
-  if (
-    inherits(tc, "try-error")
-  ){
-    cli::cat_bullet(
-      "Running setup and/or solution chunks failed",
-      col = "red",
-      bullet = "cross"
-    )
-    return(invisible(FALSE))
+    # Add the options list to the chunk
+    usefull_chunks[[i]]$options <- options
+
+    # The exercise object will need a  label and the engine,
+    # we set them both here
+    usefull_chunks[[i]]$label <-  usefull_chunks[[i]]$params$label
+    usefull_chunks[[i]]$engine <-
+      usefull_chunks[[i]]$options$engine <- usefull_chunks[[i]]$params$engine %||% "r"
+
   }
 
-  cli::cat_bullet(
-    "Successfully run setup and/or solution chunks",
-    col = "green",
-    bullet = "tick"
-  )
+  # Now that we have manipulated the chunks, we can build the
+  # exercise objects
+
+  for (chunk_ in chunks_){
+    # Restoring  knitr code stock
+    knitr::knit_code$restore()
+    # Grep all the related chunks (i.e setup, checker, etc)
+    all_related <- grep(chunk_, names(usefull_chunks), value = TRUE)
 
-  cli::cat_line(" ")
+    # If there is a setup chunk, we make sure it's the first of the list
+    if (any(grepl("setup", all_related))){
+      setup_ <- grepl("setup", all_related)
+      all_related <- c(
+        all_related[setup_],
+        all_related[!setup_]
+      )
+    }
+
+    # This function allow to grab a chunk based on
+    # its pattern
+    grab_chunk <- function(
+      pattern
+    ){
+      res <- usefull_chunks[
+        all_related[
+          grepl(
+            sprintf("%s-%s", chunk_, pattern),
+            all_related
+          )
+        ]
+      ]
+      if (length(res)){
+        return(res)
+      }
+      list(
+        list()
+      )
+    }
+
+    # Now we are building the exercise object, and it
+    # will be sent to evaluate_code()
+    #
+    # An exercise object needs the following elements:
+    # - exercise$label
+    # - exercise$code, which will be the solution code here
+    # - exercise$restore
+    # - exercise$timestamp
+    # - exercise$global_setup if any
+    # - exercise$setup if any
+    # - exercise$code_check if any
+    # - exercise$chunks that contains the setup chunks
+    # - exercise$check if any
+    # - exercise$engine
+    #
+
+    # We keep the  chunk we need: setup, and solution
+    chunks_needed <- list()
+
+    if (length(grab_chunk("setup")[[1]])){
+      chunks_needed[[1]] <- grab_chunk("setup")[[1]]
+    }
+    if (length(grab_chunk("solution")[[1]])){
+      chunks_needed[[
+        length(chunks_needed) + 1
+      ]] <- grab_chunk("solution")[[1]]
+    }
+    exercise_blt <- list(
+      # Getting the label
+      label = {
+        chunk_
+      },
+      code = {
+        paste0(grab_chunk("solution")[[1]]$code, "\n\n", collapse = "\n")
+      },
+      restore = {
+        FALSE
+      },
+      timestamp = {
+        as.numeric(Sys.time())
+      },
+      global_setup = {
+        paste0(setup_chunk$code, collapse = "\n")
+      },
+      setup = {
+        paste0(grab_chunk("setup")[[1]]$code, collapse = "\n")
+      },
+      chunks = {
+        chunks_needed
+      },
+      solution  = {
+        grab_chunk("solution")[[1]]
+      },
+      code_check  = {
+        #grab_chunk("code-check")[[1]]
+      },
+      options = {
+        usefull_chunks[chunk_][[1]]$options
+      },
+      engine = {
+        usefull_chunks[chunk_][[1]]$enginee %||% "r"
+      },
+      version = "1"
+    )
+
+    res <- evaluate_exercise(
+      exercise_blt,
+      envir = new.env()
+    )
+    if (is.null(res$error_message)){
+      cli::cat_bullet(
+        sprintf("Exercise '%s' checked", chunk_),
+        bullet = "tick", bullet_col = "green", col = "green"
+      )
+    } else {
+      cli::cat_bullet(
+        sprintf("Exercise '%s' failed", chunk_),
+        bullet = "tick",
+        bullet_col = "red",
+        col = "red"
+      )
+      print(res$error_message)
+    }
+  }
 
   return(invisible(TRUE))
 }
diff --git a/sandbox/sandbox-exos.Rmd b/sandbox/sandbox-exos.Rmd
new file mode 100644
index 000000000..963395968
--- /dev/null
+++ b/sandbox/sandbox-exos.Rmd
@@ -0,0 +1,124 @@
+---
+title: "Tutorial"
+output: 
+  learnr::tutorial:
+    progressive: true
+    allow_skip: true
+runtime: shiny_prerendered
+---
+
+```{r setup, include=FALSE}
+library(learnr)
+library(gradethis)
+library(nycflights13)
+gradethis_setup()
+knitr::opts_chunk$set(echo = FALSE)
+```
+
+## Flight
+
+
+```{r filter, exercise=TRUE}
+# filter the flights table to include only United and American flights
+flights
+```
+
+```{r filter-setup}
+# filter the flights table to include only United and American flights
+library(nycflights13)
+```
+
+```{r filter-hint-1}
+filter(flights, ...)
+```
+
+```{r filter-hint-2}
+filter(flights, UniqueCarrier=="AA")
+```
+
+```{r filter-hint-3}
+filter(flights, UniqueCarrier=="AA" | UniqueCarrier=="UA")
+```
+
+```{r filter-solution}
+# filter the flights table to include only United and American flights
+flights
+```
+
+
+## grade_code
+
+```{r grade_code, exercise = TRUE}
+
+```
+
+```{r grade_code-solution}
+sqrt(log(2))
+```
+
+```{r grade_code-check}
+grade_code("Don't worry, things will soon get harder.")
+```
+
+## mtcars
+
+```{r mtcars, exercise = TRUE}
+
+```
+
+```{r mtcars-solution}
+mtcars
+```
+
+```{r mtcars-check}
+grade_result(
+  fail_if(~identical(.result, cars), "This is the cars (not mtcars) dataset."),
+  pass_if(~identical(.result, mtcars))
+)
+```
+
+## SQL
+
+```{sql sql-not-r, exercise = TRUE, connection="mammals"}
+SELECT *
+FROM `surveys`
+LIMIT 10
+```
+
+```{sql sql-not-r-solution}
+mtcars
+```
+
+```{sql sql-not-r-error-check}
+grade_code(incorrect = "This code produces an error (press 'Run Code' to see it).", glue_incorrect = "{ .message } { .incorrect }")
+```
+
+```{sql sql-not-r-check}
+grade_result(
+  fail_if(~identical(.result, cars), "This is the cars (not mtcars) dataset."),
+  pass_if(~identical(.result, mtcars))
+)
+```
+
+## mtcars2
+
+```{r mtcars2, exercise = TRUE}
+
+```
+
+```{r mtcars2-solution}
+mtcars
+```
+
+```{r mtcars2-error-check}
+grade_code(incorrect = "This code produces an error (press 'Run Code' to see it).", glue_incorrect = "{ .message } { .incorrect }")
+```
+
+```{r mtcars2-check}
+grade_result(
+  fail_if(~identical(.result, cars), "This is the cars (not mtcars) dataset."),
+  pass_if(~identical(.result, mtcars))
+)
+```
+
+

From 93a44c75c9fc24adf35ddabcdfbf7195a84e2a27 Mon Sep 17 00:00:00 2001
From: colin <colin@thinkr.fr>
Date: Fri, 23 Oct 2020 16:50:59 +0200
Subject: [PATCH 4/6] added example + kept the  TODO in code_check

---
 R/check_exercise.R | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/R/check_exercise.R b/R/check_exercise.R
index 58a067b3e..831b740b4 100644
--- a/R/check_exercise.R
+++ b/R/check_exercise.R
@@ -78,7 +78,8 @@ is_exercise_setup_chunk <- function(label) {
 #'
 #' @examples
 #' if (interactive()){
-#'   check_exercise("sandbox/sandbox.Rmd")
+#'   check_exercise("sandbox/sandbox")
+#'   check_exercise("sandbox/sandbox-exos.Rmd")
 #' }
 check_exercise <- function(
   path
@@ -365,6 +366,7 @@ check_exercise <- function(
         grab_chunk("solution")[[1]]
       },
       code_check  = {
+        # TODO
         #grab_chunk("code-check")[[1]]
       },
       options = {

From 7b09892e905e9f06c26fca870991fbd2c9a5f589 Mon Sep 17 00:00:00 2001
From: colin <colin@thinkr.fr>
Date: Fri, 23 Oct 2020 16:53:28 +0200
Subject: [PATCH 5/6] Correct return for check_exercise

---
 R/check_exercise.R | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/R/check_exercise.R b/R/check_exercise.R
index 831b740b4..df5df8815 100644
--- a/R/check_exercise.R
+++ b/R/check_exercise.R
@@ -273,6 +273,10 @@ check_exercise <- function(
 
   }
 
+  # Setting something to return that will be turned to FALSE
+  # if any check fail
+  all_passed <- TRUE
+
   # Now that we have manipulated the chunks, we can build the
   # exercise objects
 
@@ -395,8 +399,9 @@ check_exercise <- function(
         col = "red"
       )
       print(res$error_message)
+      all_passed <- FALSE
     }
   }
 
-  return(invisible(TRUE))
+  return(invisible(all_passed))
 }

From d86bc7fcc63c0eab6035eb670873eccadf9b2410 Mon Sep 17 00:00:00 2001
From: colin <colin@thinkr.fr>
Date: Fri, 23 Oct 2020 16:58:41 +0200
Subject: [PATCH 6/6] redoc

---
 man/check_exercise.Rd | 9 ++++-----
 1 file changed, 4 insertions(+), 5 deletions(-)

diff --git a/man/check_exercise.Rd b/man/check_exercise.Rd
index 6cc4218c9..1e30be8a3 100644
--- a/man/check_exercise.Rd
+++ b/man/check_exercise.Rd
@@ -4,12 +4,10 @@
 \alias{check_exercise}
 \title{Check the code from an Exercise}
 \usage{
-check_exercise(path, verbose = TRUE)
+check_exercise(path)
 }
 \arguments{
 \item{path}{Path to the Markdown file containing the RMarkdown.}
-
-\item{verbose}{Should the test output information on the console?}
 }
 \value{
 TRUE or FALSE invisibly.
@@ -17,11 +15,12 @@ TRUE or FALSE invisibly.
 \description{
 This function will take all the chunks with a label that matches \code{setup} or
 \code{-solution}, put them in a separate script and try to run them all.
-This will allow teachers to check that their setup and solution chunks
+This allows teachers to check that their setup and solution chunks
 contain valid code.
 }
 \examples{
 if (interactive()){
-  check_exercise("sandbox/sandbox.Rmd")
+  check_exercise("sandbox/sandbox")
+  check_exercise("sandbox/sandbox-exos.Rmd")
 }
 }