use kwg for word graph

.circleci/config.yml

@@ -7,45 +7,48 @@ orbs:
 jobs:
   build:
     environment:
-      LEXICON_PATH: /opt/lexica/
-      STRATEGY_PARAMS_PATH: /opt/data/strategy
-      LETTER_DISTRIBUTION_PATH: /opt/data/letterdistributions
-      AUTH_KEY: abcdef
-    docker:
-      - image: golang:alpine
-    working_directory: /opt/macondo
+      LEXICON_PATH: /home/circleci/lexica
+    machine: # executor type
+      image: ubuntu-2004:current
+      docker_layer_caching: true
+    working_directory: ~/macondo
     steps:
       - checkout
       - add_ssh_keys:
           fingerprints:
             - "bf:37:1b:e1:b2:b0:ec:20:92:43:20:86:40:7c:c4:e0"
-      - run:
-          name: Install git, ssh, build-base, bash (for tests)
-          command: apk update && apk upgrade && apk add git openssh build-base bash curl
-      - run:
-          name: Install ssl
-          command: apk add --update ca-certificates openssl && update-ca-certificates
       - run: if [ ! -n "$(grep "^github.com " ~/.ssh/known_hosts)" ]; then ssh-keyscan github.com >> ~/.ssh/known_hosts 2>/dev/null; fi
       - run:
           name: Clone lexicon repo
-          command: git clone git@github.com:domino14/word-game-lexica /opt/word-game-lexica
-      - run: mkdir $LEXICON_PATH
-      - run: mkdir $LEXICON_PATH/gaddag
-      - run: mkdir $LEXICON_PATH/dawg
-      - run: mv /opt/word-game-lexica/*.txt $LEXICON_PATH
-      - run: cp -r /opt/macondo/data /opt/data
-      - run: go build cmd/shell/main.go
-      # build GADDAGs
-      - restore_cache:
-          keys:
-            - gaddag-cache
-      - run: cd testcommon && go generate 
-      - run: cd /opt/macondo && go test $(go list ./... | grep -v wasm)
-      - save_cache:
-          key: gaddag-cache
-          paths:
-            - /opt/lexica/gaddag
-            - /opt/lexica/dawg
+          command: git clone git@github.com:domino14/word-game-lexica ~/word-game-lexica
+      - run:
+          name: Clone wolges
+          command: git clone https://github.com/andy-k/wolges ~/wolges
+      - run: mkdir ~/lexica
+      - run: mkdir ~/lexica/gaddag
+      - run: mv ~/word-game-lexica/*.txt ~/lexica
+      - run: cp -r ~/macondo/data ~/data
+      - run: docker build -t macondo -f Dockerfile-dev .
+      - run: docker run --rm macondo make
+      # # build GADDAGs
+      # - restore_cache:
+      #     keys:
+      #       - kwg-cache
+      - run: cp Dockerfile-kbuilder ~/wolges
+      - run: cd ~/wolges && docker build -t kbuilder -f Dockerfile-kbuilder .
+      - run: cd ~/macondo/.circleci && ./create_lexicon_files.sh
+      - run: >
+          docker run --rm -e LEXICON_PATH=/opt/lexica
+          -e STRATEGY_PARAMS_PATH=/opt/data/strategy
+          -e DATA_PATH=/opt/data
+          -e LETTER_DISTRIBUTION_PATH=/opt/data/letterdistributions
+          -v ~/data:/opt/data
+          -v ~/lexica:/opt/lexica
+          macondo go test $(go list ./... | grep -v wasm)
+      # - save_cache:
+      #     key: kwg-cache
+      #     paths:
+      #       - /opt/lexica/gaddag
       - discord/status: &discord-webhook-setting
           webhook: "${DISCORD_WEBHOOK}"
           success_message: ":tada: A $CIRCLE_JOB job has succeeded! (Branch: $CIRCLE_BRANCH)"

.circleci/create_lexicon_files.sh

@@ -0,0 +1,70 @@
+#!/bin/bash
+
+set -euo pipefail
+
+ls $LEXICON_PATH
+
+for lex in "NWL20" "NWL18" "America" "CSW21" "CSW19" "ECWL"
+do
+    awk '{print $1}' "$LEXICON_PATH/$lex.txt" > "$LEXICON_PATH/$lex-stripped.txt"
+    awk '{print toupper($0)}' "$LEXICON_PATH/$lex-stripped.txt" > "$LEXICON_PATH/$lex-toupper.txt"
+    echo "lex $lex"
+
+    CONTAINER_ID="$(docker create kbuilder -- english-kwg /home/in.txt /home/out.kwg )"
+    trap "docker rm $CONTAINER_ID" EXIT
+    echo "$CONTAINER_ID"
+
+    docker cp "$LEXICON_PATH/$lex-toupper.txt" "$CONTAINER_ID:/home/in.txt"
+    docker start "$CONTAINER_ID"
+    docker attach "$CONTAINER_ID" || true
+    docker cp "$CONTAINER_ID:/home/out.kwg" "$LEXICON_PATH/gaddag/$lex.kwg"
+
+    docker rm "$CONTAINER_ID"
+    trap "" EXIT
+
+    echo "after $lex"
+done
+
+for lex in "OSPS44"
+do
+    awk '{print $1}' "$LEXICON_PATH/$lex.txt" > "$LEXICON_PATH/$lex-stripped.txt"
+    echo "lex $lex"
+
+    CONTAINER_ID="$(docker create kbuilder -- polish-kwg /home/in.txt /home/out.kwg )"
+    trap "docker rm $CONTAINER_ID" EXIT
+    echo "$CONTAINER_ID"
+
+    docker cp "$LEXICON_PATH/$lex-stripped.txt" "$CONTAINER_ID:/home/in.txt"
+    docker start "$CONTAINER_ID"
+    docker attach "$CONTAINER_ID" || true
+    docker cp "$CONTAINER_ID:/home/out.kwg" "$LEXICON_PATH/gaddag/$lex.kwg"
+
+    docker rm "$CONTAINER_ID"
+    trap "" EXIT
+
+    echo "after $lex"
+done
+
+
+for lex in "NSF22"
+do
+    awk '{print $1}' "$LEXICON_PATH/$lex.txt" > "$LEXICON_PATH/$lex-stripped.txt"
+    awk '{print toupper($0)}' "$LEXICON_PATH/$lex-stripped.txt" > "$LEXICON_PATH/$lex-toupper.txt"
+    echo "lex $lex"
+
+    CONTAINER_ID="$(docker create kbuilder -- norwegian-kwg /home/in.txt /home/out.kwg )"
+    trap "docker rm $CONTAINER_ID" EXIT
+    echo "$CONTAINER_ID"
+
+    docker cp "$LEXICON_PATH/$lex-toupper.txt" "$CONTAINER_ID:/home/in.txt"
+    docker start "$CONTAINER_ID"
+    docker attach "$CONTAINER_ID" || true
+    docker cp "$CONTAINER_ID:/home/out.kwg" "$LEXICON_PATH/gaddag/$lex.kwg"
+
+    docker rm "$CONTAINER_ID"
+    trap "" EXIT
+
+    echo "after $lex"
+done
+
+echo "done creating kwgs"

Dockerfile-dev

@@ -1,7 +1,8 @@
 FROM golang:alpine
+
+RUN apk add --update make
 ENV GOPATH=/go
 ADD . /go/src/github.com/domino14/macondo
 WORKDIR /go/src/github.com/domino14/macondo
 
-EXPOSE 8088
-CMD go run main.go -dawgpath=/dawgs/
+EXPOSE 8088

Dockerfile-kbuilder

@@ -0,0 +1,7 @@
+# Dockerfile for Kurnia files (KWG, KLV)
+# See github.com/andy-k/wolges
+FROM rust:1.68.0
+ADD . /workdir
+WORKDIR /workdir
+RUN ["cargo", "build", "--release", "--bin", "buildlex"]
+ENTRYPOINT ["cargo", "run", "--release", "--bin", "buildlex"]

Makefile

@@ -1,6 +1,6 @@
 everything: all wasm
 
-all: macondo_shell macondo_bot bot_shell gaddag_maker make_leaves_structure analyze
+all: macondo_shell macondo_bot bot_shell analyze
 
 .PHONY: wasm
 
@@ -19,11 +19,11 @@ macondo_bot:
 bot_shell:
 	go build -trimpath -o bin/bot_shell cmd/bot_shell/main.go
 
-gaddag_maker:
-	go build -trimpath -o bin/make_gaddag cmd/make_gaddag/main.go
+# gaddag_maker:
+# 	go build -trimpath -o bin/make_gaddag cmd/make_gaddag/main.go
 
-make_leaves_structure:
-	go build -trimpath -o bin/make_leaves_structure cmd/make_leaves_structure/main.go
+# make_leaves_structure:
+# 	go build -trimpath -o bin/make_leaves_structure cmd/make_leaves_structure/main.go
 
 wasm:
 	GOOS=js GOARCH=wasm go build -trimpath -o ../liwords/liwords-ui/public/wasm/macondo.wasm wasm/*.go

ai/bot/bot_player.go

@@ -20,6 +20,7 @@ type BotConfig struct {
 	config.Config
 	PEGAdjustmentFile string
 	LeavesFile        string
+	MinSimPlies       int
 }
 
 type BotTurnPlayer struct {
@@ -29,6 +30,7 @@ type BotTurnPlayer struct {
 	simmer      *montecarlo.Simmer
 	simmerCalcs []equity.EquityCalculator
 	simThreads  int
+	minSimPlies int
 
 	inferencer *rangefinder.RangeFinder
 }
@@ -87,12 +89,15 @@ func addBotFields(p *turnplayer.BaseTurnPlayer, conf *BotConfig, botType pb.BotR
 	// If it is a simming bot, add more fields.
 	if hasSimming(botType) {
 		c, err := equity.NewCombinedStaticCalculator(
-			p.LexiconName(), p.Config(), equity.LeaveFilename, equity.PEGAdjustmentFilename)
+			p.LexiconName(), p.Config(), "", equity.PEGAdjustmentFilename)
 		if err != nil {
 			return nil, err
 		}
 		btp.simmer = &montecarlo.Simmer{}
 		btp.simmerCalcs = []equity.EquityCalculator{c}
+		if conf.MinSimPlies > 0 {
+			btp.SetMinSimPlies(conf.MinSimPlies)
+		}
 	}
 	if hasEndgame(botType) {
 		btp.endgamer = &alphabeta.Solver{}
@@ -146,3 +151,7 @@ func (p *BotTurnPlayer) SetBotType(b pb.BotRequest_BotCode) {
 func (p *BotTurnPlayer) SetSimThreads(t int) {
 	p.simThreads = t
 }
+
+func (p *BotTurnPlayer) SetMinSimPlies(t int) {
+	p.minSimPlies = t
+}

ai/bot/elite.go

@@ -6,13 +6,13 @@ import (
 
 	"github.com/rs/zerolog/log"
 
-	"github.com/domino14/macondo/alphabet"
 	"github.com/domino14/macondo/equity"
-	"github.com/domino14/macondo/gaddag"
 	"github.com/domino14/macondo/game"
+	"github.com/domino14/macondo/kwg"
 	"github.com/domino14/macondo/montecarlo"
 	"github.com/domino14/macondo/move"
 	"github.com/domino14/macondo/movegen"
+	"github.com/domino14/macondo/tilemapping"
 )
 
 const InferencesSimLimit = 400
@@ -42,7 +42,7 @@ func eliteBestPlay(ctx context.Context, p *BotTurnPlayer) (*move.Move, error) {
 		if tr > 0 {
 			log.Debug().Msg("assigning all unseen to opp")
 			// bag is actually empty. Assign all of unseen to the opponent.
-			mls := make([]alphabet.MachineLetter, tr)
+			mls := make([]tilemapping.MachineLetter, tr)
 			err := p.Game.Bag().Draw(tr, mls)
 			if err != nil {
 				return nil, err
@@ -60,7 +60,11 @@ func eliteBestPlay(ctx context.Context, p *BotTurnPlayer) (*move.Move, error) {
 		simPlies = unseen
 	} else {
 		moves = p.GenerateMoves(40)
-		simPlies = 2
+		if p.minSimPlies > 2 {
+			simPlies = p.minSimPlies
+		} else {
+			simPlies = 2
+		}
 	}
 	log.Debug().Int("simPlies", simPlies).
 		Int("simThreads", p.simThreads).
@@ -82,7 +86,7 @@ func endGameBest(ctx context.Context, p *BotTurnPlayer, endgamePlies int) (*move
 		// Just return the static best play if we don't have an endgame engine.
 		return p.GenerateMoves(1)[0], nil
 	}
-	gd, err := gaddag.Get(p.Game.Config(), p.Game.LexiconName())
+	gd, err := kwg.Get(p.Game.Config(), p.Game.LexiconName())
 	if err != nil {
 		return nil, err
 	}