🚸 learn: Create initial levy CLI (#336)

Create initial `levy` CLI for local development and preview of learn platform
question.

`learnevy` currently has following sub-commands:

- `export` answerkey
- `verify`
- `seal`
- `unseal`

The target structure of Evy learn materials is (inspired by https://khanacademy.org):

    Course -> Unit -> Exercise -> Question

We directly map this hierarchy to the file system:

    course-1/
      unit-1/
        exercise-1/
          questions/
              question-1.md
              question-2.md
              ...

Questions are encoded in markdown files. Each question has a YAML front matter section, e.g.:

     type: question
     difficulty: easy # easy, medium, hard
     answer-type: single-choice # single-choice, multiple-choice, free-text, multiple-free-texts, program
     answer: c

This merges the following commits:
* build-tools: Move markdown utilities into shared md package
* learn: Create initial levy CLI
* learn: Add HTML output

     .golangci.yaml                                |   5 +
     .prettierignore                               |   1 +
     Makefile                                      |  10 +-
     build-tools/gengodoc.awk                      |   2 +-
     build-tools/md/main.go                        |   9 +-
     build-tools/site-gen/main.go                  |   2 +-
     learn/.golangci.yaml                          |   1 +
     learn/README.md                               |  55 +++
     learn/cmd/levy/answerkey-sample.json.js       |  22 +
     learn/cmd/levy/main.go                        | 227 ++++++++++
     learn/go.mod                                  |  20 +
     learn/go.sum                                  |  20 +
     learn/pkg/question/answer.go                  | 195 ++++++++
     learn/pkg/question/encrypt.go                 | 231 ++++++++++
     learn/pkg/question/encrypt_test.go            |  40 ++
     learn/pkg/question/frontmatter.go             | 142 ++++++
     learn/pkg/question/markdown.go                |  82 ++++
     learn/pkg/question/question.go                | 424 ++++++++++++++++++
     learn/pkg/question/question_test.go           | 275 ++++++++++++
     learn/pkg/question/renderer.go                | 281 ++++++++++++
     .../course1/unit1/err-exercise1/questions/dot |   1 +
     .../questions/err-false-negative.md           |  27 ++
     .../questions/err-false-positive.md           |  27 ++
     .../unit1/err-exercise1/questions/err-img1.md |  21 +
     .../unit1/err-exercise1/questions/err-img2.md |  33 ++
     .../unit1/err-exercise1/questions/err-img3.md |  33 ++
     .../unit1/err-exercise1/questions/err-img4.md |  33 ++
     .../unit1/err-exercise1/questions/err-img5.md |  33 ++
     .../unit1/err-exercise1/questions/err-img6.md |  21 +
     .../unit1/err-exercise1/questions/err-img7.md |  21 +
     .../questions/err-inconsistent1.md            |  20 +
     .../questions/err-inconsistent2.md            |  19 +
     .../err-exercise1/questions/err-link1.md      |  17 +
     .../err-exercise1/questions/err-link2.md      |  17 +
     .../err-exercise1/questions/err-link3.md      |  17 +
     .../err-exercise1/questions/err-link4.md      |  17 +
     .../err-exercise1/questions/err-link5.md      |  17 +
     .../err-exercise1/questions/err-link6.md      |  17 +
     .../unit1/err-exercise1/questions/print       |   1 +
     .../course1/unit1/exercise1/exercise1.md      |  10 +
     .../unit1/exercise1/questions/dot/dot.a.evy   |   3 +
     .../exercise1/questions/dot/dot.a.evy.svg     |   4 +
     .../unit1/exercise1/questions/dot/dot.b.evy   |   3 +
     .../exercise1/questions/dot/dot.b.evy.svg     |   4 +
     .../unit1/exercise1/questions/dot/dot.c.evy   |   3 +
     .../exercise1/questions/dot/dot.c.evy.svg     |   4 +
     .../unit1/exercise1/questions/dot/dot.d.evy   |   3 +
     .../exercise1/questions/dot/dot.d.evy.svg     |   4 +
     .../exercise1/questions/print/print.a.evy     |   1 +
     .../exercise1/questions/print/print.b.evy     |   1 +
     .../exercise1/questions/print/print.c.evy     |   1 +
     .../exercise1/questions/print/print.d.evy     |   1 +
     .../exercise1/questions/question-img1.md      |  21 +
     .../exercise1/questions/question-img2.md      |  33 ++
     .../exercise1/questions/question-link1.md     |  19 +
     .../exercise1/questions/question-link2.md     |  17 +
     .../exercise1/questions/question-link3.md     |  17 +
     .../exercise1/questions/question-link4.md     |  17 +
     .../exercise1/questions/question1-sealed.md   |  27 ++
     .../unit1/exercise1/questions/question1.md    |  19 +
     .../unit1/exercise1/questions/question2.md    |  27 ++
     .../golden/answerkey-question-img1.json       |  11 +
     .../golden/answerkey-question-img2.json       |  11 +
     .../golden/answerkey-question-link1.json      |  11 +
     .../golden/answerkey-question-link2.json      |  11 +
     .../golden/answerkey-question-link3.json      |  11 +
     .../golden/answerkey-question-link4.json      |  11 +
     .../golden/answerkey-question1-sealed.json    |  11 +
     .../testdata/golden/answerkey-question1.json  |  11 +
     .../testdata/golden/answerkey-question2.json  |  11 +
     .../testdata/golden/form-question-img1.html   |  42 ++
     .../testdata/golden/form-question-img2.html   |  42 ++
     .../testdata/golden/form-question-link1.html  |  43 ++
     .../testdata/golden/form-question-link2.html  |  42 ++
     .../testdata/golden/form-question-link3.html  |  32 ++
     .../testdata/golden/form-question-link4.html  |  32 ++
     .../testdata/golden/form-question1.html       |  24 +
     .../testdata/golden/form-question2.html       |  32 ++
     .../testdata/golden/question-img1.html        |  95 ++++
     .../testdata/golden/question-img2.html        |  95 ++++
     .../testdata/golden/question-link1.html       |  96 ++++
     .../testdata/golden/question-link2.html       |  95 ++++
     .../testdata/golden/question-link3.html       |  85 ++++
     .../testdata/golden/question-link4.html       |  85 ++++
     .../question/testdata/golden/question1.html   |  77 ++++
     .../question/testdata/golden/question2.html   |  85 ++++
     main.go                                       |   2 +-
     pkg/cli/runtime.go                            |   7 +
     build-tools/md/walkmd.go => pkg/md/md.go      |  20 +-
     89 files changed, 3789 insertions(+), 16 deletions(-)

Pull-request: #336

.golangci.yaml

@@ -4,7 +4,10 @@ issues:
     - "^don't use ALL_CAPS"
     - "^ST1003: should not use ALL_CAPS"
     - "^var-naming: don't use ALL_CAPS"
+    - "^G302: Expect file permissions to be 0600 or less"
     - "^G304: Potential file inclusion via variable"
+    - "^G306: Expect WriteFile permissions to be 0600 or less"
+    - "^G301: Expect directory permissions to be 0750 or less"
 
 linters:
   enable-all: true
@@ -26,6 +29,7 @@ linters:
     - godox
     - golint
     - gomnd
+    - gomoddirectives
     - ifshort
     - inamedparam
     - interfacer
@@ -42,6 +46,7 @@ linters:
     - scopelint
     - structcheck
     - tagalign
+    - tagliatelle
     - testpackage
     - varcheck
     - varnamelen

.prettierignore

@@ -2,3 +2,4 @@
 resets.css
 /pkg/cli/testdata/*.svg
 /pkg/cli/svg/testdata/*.svg
+/learn/pkg/question/testdata/

Makefile

@@ -31,6 +31,7 @@ clean::
 # --- Build --------------------------------------------------------------------
 GO_LDFLAGS = -X main.version=$(VERSION)
 CMDS = .
+LEARN_CMDS = ./cmd/levy
 
 ## Build full evy binaries
 build-full: embed | $(O)
@@ -39,14 +40,17 @@ build-full: embed | $(O)
 ## Build evy binaries without web content embedded
 build-go: $(O)
 	go build -o $(O) -ldflags='$(GO_LDFLAGS)' $(CMDS)
+	cd learn; go build -o ../$(O) -ldflags='$(GO_LDFLAGS)' $(LEARN_CMDS)
 
 ## Build and install binaries in $GOBIN
 install-full: embed
 	go install -tags full -ldflags='$(GO_LDFLAGS)' $(CMDS)
+	cd learn; go install -ldflags='$(GO_LDFLAGS)' $(LEARN_CMDS)
 
 ## Build and install binaries without embedded frontend in $GOBIN
 install:
 	go install -ldflags='$(GO_LDFLAGS)' $(CMDS)
+	cd learn; go install -ldflags='$(GO_LDFLAGS)' $(LEARN_CMDS)
 
 # Use `go version` to ensure the right go version is installed when using tinygo.
 go-version:
@@ -82,11 +86,13 @@ clean::
 
 # --- Test ---------------------------------------------------------------------
 COVERFILE = $(O)/coverage.txt
+LEARNCOVERFILE = $(O)/learn-coverage.txt
 EXPORTDIR = $(O)/export-test
 
 ## Run non-tinygo tests and generate a coverage file
 test-go: | $(O)
 	go test -coverprofile=$(COVERFILE) ./...
+	cd learn; go test -coverprofile=../$(LEARNCOVERFILE) ./...
 
 ## Test evy CLI
 test-cli: build-full
@@ -103,6 +109,7 @@ test-tiny: go-version | $(O)
 ## Check that test coverage meets the required level
 check-coverage: test-go
 	@go tool cover -func=$(COVERFILE) | $(CHECK_COVERAGE) || $(FAIL_COVERAGE)
+	@go tool cover -func=$(LEARNCOVERFILE) | $(CHECK_COVERAGE) || $(FAIL_COVERAGE)
 
 ## Show test coverage in your browser
 cover: test-go
@@ -119,6 +126,7 @@ EVY_FILES = $(shell find frontend/play/samples -name '*.evy')
 ## Lint go source code
 lint-go:
 	golangci-lint run
+	cd learn; golangci-lint run
 
 ## Format evy sample code
 fmt-evy:
@@ -148,7 +156,7 @@ usage: install
 	$(foreach md,$(USAGEFILES),$(USAGE_CMD)$(nl))
 
 GODOC_CMD = ./build-tools/gengodoc.awk $(filename) > $(O)/out.go && mv $(O)/out.go $(filename)
-GODOCFILES = main.go
+GODOCFILES = main.go learn/cmd/levy/main.go
 godoc: install
 	$(foreach filename,$(GODOCFILES),$(GODOC_CMD)$(nl))
 

build-tools/gengodoc.awk

@@ -5,7 +5,7 @@ $0 ~ "//\tUsage:" {
 }
 $0 ~ "^(package |// [[])" && in_usage {
         system(cmd " --help | sed -e '/./s|^|//\t|' -e 's|^$|//|'")
-        printf "//\n"
+        if ($1 == "//") printf "//\n"
         in_usage = 0
 }
 

build-tools/md/main.go

@@ -18,6 +18,7 @@ import (
 	"strings"
 	"text/template"
 
+	"evylang.dev/evy/pkg/md"
 	"github.com/alecthomas/kong"
 	"rsc.io/markdown"
 )
@@ -70,7 +71,7 @@ func (a *app) copy() ([]string, error) {
 
 		if d.IsDir() {
 			// use MkdirAll in case the directory already exists
-			return os.MkdirAll(destfile, 0o777) //nolint:gosec
+			return os.MkdirAll(destfile, 0o777)
 		}
 
 		if filepath.Ext(filename) == ".md" {
@@ -132,7 +133,7 @@ func updateASTs(asts map[string]*markdown.Document) {
 			continue
 		}
 		w := &walker{anchorIDs: map[string]bool{}}
-		walk(ast, w.walk)
+		md.Walk(ast, w.walk)
 		headings[mdf] = w.headings
 	}
 	for _, sbf := range sidebarFiles {
@@ -141,7 +142,7 @@ func updateASTs(asts map[string]*markdown.Document) {
 		// we need to walk sidebars _after_ sidebar update with heading
 		// insertion because we look up the inserted headings by markdown and
 		// not html filename.
-		walk(asts[sbf], w.walk)
+		md.Walk(asts[sbf], w.walk)
 	}
 }
 
@@ -218,7 +219,7 @@ type heading struct {
 	heading  *markdown.Heading
 }
 
-func (w *walker) walk(n node) {
+func (w *walker) walk(n md.Node) {
 	switch n := n.(type) {
 	case *markdown.Document:
 		removeTOC(n)

build-tools/site-gen/main.go

@@ -94,7 +94,7 @@ func (a *app) copyTree() error {
 
 		switch mode := d.Type() & fs.ModeType; mode {
 		case fs.ModeDir:
-			return os.Mkdir(destfile, 0o777) //nolint:gosec // erroneous linter
+			return os.Mkdir(destfile, 0o777)
 		case fs.ModeSymlink:
 			if err := checkSymlink(srcfile); err != nil {
 				return err

learn/.golangci.yaml

@@ -0,0 +1 @@
+../.golangci.yaml

learn/README.md

@@ -0,0 +1,55 @@
+# Learn
+
+The learn directory contains all essential code and content for the Evy courses
+on https://learn.evy.dev. This includes the markdown files, images, Evy code
+samples, but also Firebase and Firestore utilities for answer verification
+and progress tracking. However, frontend code is not included in this directory.
+
+The target structure of Evy learn materials is (inspired by https://khanacademy.org):
+
+    Course -> Unit -> Exercise -> Question
+
+We directly map this hierarchy to the file system:
+
+    courses/
+      fundamentals/                # course
+        sequences/                 # unit
+          print/                   # exercise
+            questions/
+                print1.md          # question
+                print2.md
+                ...
+
+Questions are encoded in markdown files. Each question has a YAML front matter section, e.g.:
+
+     type: question
+     difficulty: easy # "easy", "medium", "hard", "retriable"
+     answer-type: single-choice # single-choice, multiple-choice, free-text, multiple-free-texts, program
+     answer: c
+
+## `levy`
+
+`levy` is a tool for creating and previewing Evy practice and learn materials.
+It currently supports the following sub-commands:
+
+- `export` the answer key to JSON
+- `verify`
+- `seal`
+- `unseal`
+
+Try it with
+
+    make install
+    levy export answerkey pkg/question/testdata/course1/unit1/exercise1/questions/question1.md
+    levy seal pkg/question/testdata/course1/unit1/exercise1/questions/question1.md
+
+For sample error messages in case of failed verification, try
+
+    levy verify pkg/question/testdata/course1/unit1/err-exercise1/questions/err-false-negative.md
+
+To unseal sealed questions you need the secret private key. Here is a sample
+usage with a private-key that is a test key. Do not use this key in
+production!
+
+    export EVY_LEARN_PRIVATE_KEY="MIIEpQIBAAKCAQEAuNEufiuryg/OZPKVUbaIRam1UNqju5binwrRzsOGWkM6DYKqxW2tA+O7dhg9do/Jm0lr+rkVqf8CR/HejD08n9OTsHe0NeblLwZncQX1J3ayyGsu+xAFxQ0hvFfG+Vy8KXJAgug6CCsaiVgBwOWPdfEOqEDv5S5XlnwQh9dxWB8m/1CTDmqSdIhYnzQQp13ZyumCRgrIHKSYPR3KCZD8KLRvkoIrF0DU18f6ASO7wjv7FBhgQ2ZAR/Yud/h6ceQKvAW0W3MmPiJblZhbrsPQGi7eZZo4K8aAvuzQmcYq17/E/e6MnOweoyik4lIAG0uGa7FiY5f9NVuir7JPA2lCLwIDAQABAoIBAQCJHEcNu4BbC5bnNUCpum0moVyue0X1KV8+9lvotQ27cRxkYYgnp9IvjIfKePlAODQtTC8bdqwnzdP3Y+zixZtwRxrOVEARrRZh6LJdGzpg6KKCJWJZR+2/3pokjEpFPRMq/GP3uikzXib1taC3ZpcjvI5PLL3MnLDGJ4xr+t1Pral8BXSILhUSQzgMFAB8+5V+zWnUPuPzCeym3VeYpZSdbSsR+CZnxy4vbB4cSj97M1MgBTOPocduE5cRrE8mumAk93dzBmKH+/potjLOMhCiJJFVtPO9GXLLduLAH9qKwSk2vJytIX8KwYFTCve2EKhMB9ydBhk09zVoELUle0mhAoGBAOc9agk5CahkNOVO1E3Cw1zK+Da+2LXYk2HhjTpOTkr2lKji8v1eSDkk5R72ZfPrI5s8sBrkW0OqPJVXDnmho78quWHTwxvJrnrIcuZLa1Kn4H+cHN81J9jGcim7kLPTZUcnU0RMR7Xn3lT61H5lB3LSFplRq52tqS5AaxaksS0tAoGBAMybQjceAVTihCHKFkaFV8Ys2dm5p5ejCzYklY+jA0UdTmHT6kmr13KIA6k61+s8kyZDaGutZ6lRyHuCfotL6j6jr8rsn/EbDikZ4/XhhO9+B+xJMXolKLFA+/pBPxNs7KLSjZ3mH7N0qzxbQzyVWF4BhSxTxIjWEGAtc1ZUJN5LAoGBAMYPFzRhE0GU2q2RkEwuRnDDNEiHvEw8/Td4HiPTkEGq4/ens2KKj6fKTyju+LIsM6oyF9BgyT6yoAN1tmM9rGf/qxr8av/xBa4K5EcWUA1S1vnV9/DCsad9iajvC2jK5tND/pDgGQfYWtlEoh7EX9Xb1hlqF2kNpnuEF3UkiNDdAoGBAJzuMFlKAEd0/VdVQsSQHYR4fhbKmMprWXwLj1L9+tIV6jqKaVZcIQFNZVF1OorIiSx94ydDdxCdE6H3sstwTJgCwCBqYTpyP+gyXXAHqwhtp/IJKZO/0HgzmZCWXqStlMpFqC0FhicEQxol/WoIOiDQFa6sCT/Sv/iko6QBIc4FAoGAMSC5SUsgUiHo6gvp2put1ySmJIVj3roqI6mAndi2hLVMalF1Q5F4X4HVHWqOj7QA7zpf3ATotCI4AbmfOwpFCZ4rEP0QsbV2uZ/3NhxwAE1MWrv+ht2ONe74sOYg7Z+XAjD7TW7We3KTewerVnC/VotKZ+3Eq2FgelSYDvlNmoQ="
+    levy unseal pkg/question/testdata/course1/unit1/exercise1/questions/question1-sealed.md

learn/cmd/levy/answerkey-sample.json.js

@@ -0,0 +1,22 @@
+// Demo json data in JS form for readability:
+// comments, unquoted keys, optional trailing commas
+{
+  // course: fundamentals
+  fundamentals: {
+    // unit: sequence
+    sequence: {
+      // exercise: print
+      print: {
+        // question: simple
+        simple: {
+          // answer-type: single-choice, answer: d
+          single: "d"
+        }
+        // question: lines
+        lines: {
+          single: "d"
+        }
+      }
+    }
+  }
+}