Permalink
Browse files

v2.0.8

  • Loading branch information...
JensNevens committed Jul 30, 2018
1 parent fa04c56 commit 83c635f49467bfd01437b9c72d02fc78cb08fa19
Showing with 8,608 additions and 5,068 deletions.
  1. +1 −0 .gitignore
  2. +1 −1 VERSION
  3. 0 {sharing → applications}/fcg-for-planning/example.lisp
  4. 0 {sharing → applications}/fcg-for-planning/goal-extraction.lisp
  5. 0 {sharing → applications}/fcg-for-planning/goal-tests.lisp
  6. 0 {sharing → applications}/fcg-for-planning/monitors.lisp
  7. 0 {sharing → applications}/fcg-for-planning/node-tests.lisp
  8. 0 {sharing → applications}/fcg-for-planning/package.lisp
  9. 0 {sharing → applications}/fcg-for-planning/pancakes.lisp
  10. 0 {sharing → applications}/fcg-for-planning/plan-extraction.lisp
  11. 0 {sharing → applications}/fcg-for-planning/planning.asd
  12. 0 {sharing → applications}/fcg-for-planning/planning.lisp
  13. 0 {sharing → applications}/fcg-for-planning/state-extraction.lisp
  14. +2 −2 grammars/Portuguese/start.lisp
  15. +1 −1 grammars/Spanish/start.lisp
  16. +6,616 −4,837 libraries/asdf.lisp
  17. +3 −0 scripts/linux/ccl-init-babel.lisp
  18. +5 −31 scripts/linux/install.sh
  19. +3 −0 scripts/osx/ccl-init-babel.lisp
  20. +27 −56 scripts/osx/install.sh
  21. +3 −2 systems/corpus-processing/corpus-processing.asd
  22. +15 −7 systems/corpus-processing/corpus-processing.lisp
  23. +373 −0 systems/corpus-processing/json-corpus-processing.lisp
  24. +11 −3 systems/fcg/construction-inventories/construction-inventory.lisp
  25. +1 −1 systems/fcg/construction-inventories/hashed-construction-set.lisp
  26. +4 −1 systems/fcg/construction-inventory-processor/construction-inventory-processor.lisp
  27. +2 −2 systems/fcg/evaluation/evaluate-grammar.lisp
  28. +175 −90 systems/fcg/fcg-light/fcg-light-construction.lisp
  29. +2 −1 systems/fcg/fcg-light/fcg-light-to-fcg2.lisp
  30. +8 −2 systems/fcg/fcg-light/monitors-fcg-light.lisp
  31. +2 −1 systems/fcg/parse-and-produce/de-render.lisp
  32. +26 −11 systems/fcg/unify-and-merge/match-structures.lisp
  33. +25 −3 systems/nlp-tools/penelope-interface.lisp
  34. +58 −0 systems/nlp-tools/utils.lisp
  35. +20 −0 systems/utils/clos.lisp
  36. +1 −1 systems/utils/misc-utils.lisp
  37. +7 −4 systems/utils/queue.lisp
  38. +10 −3 systems/utils/symbols-and-strings.lisp
  39. +11 −8 systems/utils/tree.lisp
  40. +1 −0 systems/utils/utils.asd
  41. +339 −0 tutorial/grammar-writing/clevr-demo.lisp
  42. +170 −0 tutorial/grammar-writing/creative-resultative-cxn.lisp
  43. +264 −0 tutorial/grammar-writing/referent-grammar.lisp
  44. +421 −0 tutorial/grammar-writing/the-x-est-y-in-the-z.lisp
View
@@ -20,6 +20,7 @@
*.asd~
*.ps
*.64xfasl
*.lisp#
.DS_Store
**/.tmp
View
@@ -1 +1 @@
v2.0.7
v2.0.8
@@ -1,7 +1,7 @@
;; Portuguese grammar for clitics
;; Tânia Marques and Katrien Beuls (2016)
;(asdf:operate 'asdf:load-op 'portuguese-grammar)
;(ql:quickload :portuguese-grammar)
(in-package :fcg)
@@ -37,7 +37,7 @@
(set-configuration (visualization-configuration *portuguese-corpus-grammar*) :with-search-debug-data t)
;(add-element (make-html *portuguese-corpus-grammar*))
(activate-monitor trace-fcg-light)
(activate-monitor trace-fcg)
(clear-page))
@@ -1,5 +1,5 @@
;(asdf:operate 'asdf:load-op :spanish-grammar)
;(ql:quickload :spanish-grammar)
(in-package :fcg)
(defparameter *test-sentences* (babel-pathname :directory '("grammars" "Spanish")
View

Large diffs are not rendered by default.

Oops, something went wrong.
@@ -12,3 +12,6 @@
;;; load babel2
(load "/home/<USER>/Babel2/libraries/asdf")
(load "/home/<USER>/Babel2/init-babel")
;;; load cl-ppcre
(ql:quickload :cl-ppcre)
View
@@ -137,12 +137,7 @@ echo "You can also use the professional editor Lispworks for development."
echo "However, the free version of Lispworks won't work with Babel due to memory limits."
echo "We can't install Lispworks for you, but we can set up the config for Babel."
echo
echo "Note that you will need to install additional packages when using Sublime Text to enjoy the full Lisp experience"
echo "First, install Package Control. Use this to install the following packages:"
echo "SublimeREPL, rainbowth, Load File to REPL, BracketHighlighter and lispindent"
echo "PLEASE BE NOTED that development in Sublime Text is not fully supported by our team"
echo
read -n1 -rsp "Press S to skip, L for Lispworks, T for Sublime Text, Ctrl+C to abort, or any other key for Emacs: " KEY
read -n1 -rsp "Press S to skip, L for Lispworks, Ctrl+C to abort, or any other key for Emacs: " KEY
echo
if [ "$KEY" = 'S' -o "$KEY" = 's' ]; then
echo "Skipping editor."
@@ -152,13 +147,6 @@ elif [ "$KEY" = 'L' -o "$KEY" = 'l' ]; then
echo "== 4: Lispworks settings =="
home_settings ".lispworks" "lispworks-babel" "Lispworks"
echo
elif [ "$KEY" = 'T' -o "$KEY" = 't' ]; then
echo
echo "== 4: Sublime Text editor =="
wget -qO - https://download.sublimetext.com/sublimehq-pub.gpg | sudo apt-key add -
echo "deb https://download.sublimetext.com/ apt/stable/" | sudo tee /etc/apt/sources.list.d/sublime-text.list
sudo apt update && sudo apt install sublime-text
echo
else
echo
echo "== 4: Emacs editor =="
@@ -201,24 +189,10 @@ if [ $? -eq 0 ]; then
echo "Babel2 detected, skipping."
else
echo "Babel2 has not been found."
echo "You can download Babel2 as a zip archive or clone it from Git"
echo "To get the latest version, we recommend cloning from Git"
read -n1 -rsp "Press Z for the zip archive or C for the Git clone: " KEY
if [ "$KEY" = 'Z' -o "$KEY" = 'z' ]; then
echo "Downloading the zip archive"
## curl -s https://api.github.com/repos/EvolutionaryLinguisticsAssociation/Babel2/tags | jq '.[0].zipball_url'
apt_install "unzip" "unzip"
(wget https://github.com/EvolutionaryLinguisticsAssociation/Babel2/archive/v2.0.6.zip &&
unzip v2.0.6.zip -d ~/Babel2 &&
mv ~/Babel2/Babel2-2.0.6 ~/Babel2 &&
rm v2.0.6.zip)
echo "Installed."
elif [ "$KEY" = 'C' -o "$KEY" = 'c' ]; then
echo "Cloning the git repository"
apt_install "git" "git"
(git clone https://github.com/EvolutionaryLinguisticsAssociation/Babel2.git ~/Babel2)
echo "Installed"
fi
echo "Cloning the latest version from Git..."
apt_install "git" "git"
(git clone https://github.com/EvolutionaryLinguisticsAssociation/Babel2.git ~/Babel2)
echo "Installed"
fi
echo
@@ -8,3 +8,6 @@
;;; load babel2
(load "/Users/<USER>/Babel2/libraries/asdf")
(load "/Users/<USER>/Babel2/init-babel")
;;; load cl-ppcre
(ql:quickload :cl-ppcre)
View
83 scripts/osx/install.sh 100644 → 100755
@@ -49,7 +49,6 @@ function home_settings {
mv "$NEWFILE" "$OLDFILE"
echo "Config placed."
fi
rm "$NEWFILE"
cd "$OLDPWD"
}
@@ -74,26 +73,6 @@ function brew_install {
fi
}
function brew_cask_install {
# $1: Command to be checked
# $2: Name of package
# $3: Options if any
command -v $1
if [ $? -eq 0 ]; then
echo "$2 detected, skipping."
else
echo "$2 has not been found."
echo "Installing..."
brew cask install $2 $3
if [ $? -eq 0 ]; then
echo "Installed."
else
echo "Something wrong has happened, aborting. Contact the Babel team for help."
exit 1
fi
fi
}
##
## END OF FUNCTIONS DEFINITION
##
@@ -150,16 +129,25 @@ echo
echo "== 3: CCL Lisp compiler =="
## brew_install "ccl" "clozure-cl"
curl -OL "https://github.com/Clozure/ccl/releases/download/v1.11.5/ccl-1.11.5-darwinx86.tar.gz"
tar -xvzf ccl-1.11.5-darwinx86.tar.gz
mv ./ccl/scripts/ccl /usr/local/bin/
mv ./ccl/scripts/ccl64 /usr/local/bin/
mv ./ccl/ /usr/local/src/
command -v ccl
if [ $? -eq 0 ]; then
echo "CCL detected, skipping."
else
echo "Downloading and installing CCL"
curl -OL "https://github.com/Clozure/ccl/releases/download/v1.11.5/ccl-1.11.5-darwinx86.tar.gz"
tar -xvzf ccl-1.11.5-darwinx86.tar.gz
mv ./ccl/scripts/ccl /usr/local/bin/
mv ./ccl/scripts/ccl64 /usr/local/bin/
if [ ! -d "/usr/local/src/ccl" ]; then
sudo mkdir -p /usr/local/src/ccl
fi
sudo mv -v ./ccl/* /usr/local/src/ccl/
fi
echo
echo "== 4: Gnuplot =="
if [ -d "$Aquaterm.app" ]; then
if [ -d "/Applications/Aquaterm.app" ]; then
echo "AquaTerm detected, skipping"
else
brew cask install aquaterm
@@ -180,13 +168,6 @@ echo "You can also use the professional editor Lispworks for development."
echo "However, the free version of Lispworks won't work with Babel due to memory limits."
echo "We can't install Lispworks for you, but we can set up the config for Babel."
echo
echo "Note that, when choosing Sublime Text, this will not be installed through this script."
echo "You can go to Sublime Text's website and download the latest version there."
echo "Also note that you will need to install additional Sublime Text packages to enjoy the full Lisp experience"
echo "First, install Package Control. Use this to install the following packages:"
echo "SublimeREPL, rainbowth, Load File to REPL, BracketHighlighter and lispindent"
echo "PLEASE BE NOTED that development in Sublime Text is not fully supported by the Babel2 team"
echo
read -n1 -rsp "Press S to skip, L for Lispworks, Ctrl+C to abort, or any other key for Emacs: " KEY
echo
if [ "$KEY" = 'S' -o "$KEY" = 's' ]; then
@@ -200,11 +181,11 @@ elif [ "$KEY" = 'L' -o "$KEY" = 'l' ]; then
else
echo
echo "== 6: Emacs editor =="
brew tap railwaycat/emacsmacport
if [ -d "$Emacs.app" ]; then
if [ -d "/Applications/Emacs.app" ]; then
echo "Emacs detected, skipping"
else
brew install emacs-mac
brew tap railwaycat/emacsmacport
brew_install emacs emacs-mac
ln -s /usr/local/opt/emacs-mac/Emacs.app /Applications
fi
echo
@@ -242,22 +223,10 @@ if [ $? -eq 0 ]; then
echo "Babel2 detected, skipping."
else
echo "Babel2 has not been found."
echo "You can download Babel2 as an archive or clone it from Git"
echo "To get the latest version, we recommend cloning from Git"
read -n1 -rsp "Press Z for the archive or C for the Git clone: " KEY
if [ "$KEY" = 'Z' -o "$KEY" = 'z' ]; then
echo "Downloading the archive"
(curl -OL https://github.com/EvolutionaryLinguisticsAssociation/Babel2/archive/v2.0.6.tar.gz &&
mkdir ~/Babel2 &&
tar -xvzf v2.0.6.tar.gz -C ~/Babel2 --strip-components 1
rm v2.0.6.tar.gz)
echo "Installed."
elif [ "$KEY" = 'C' -o "$KEY" = 'c' ]; then
echo "Cloning the git repository"
brew_install "git" "git"
(git clone https://github.com/EvolutionaryLinguisticsAssociation/Babel2.git ~/Babel2)
echo "Installed"
fi
echo "Cloning the latest version from Git..."
brew_install "git" "git"
(git clone https://github.com/EvolutionaryLinguisticsAssociation/Babel2.git ~/Babel2)
echo "Installed"
fi
echo
@@ -274,13 +243,15 @@ else
echo "Quicklisp has not been found"
echo "Installing..."
curl -O "https://beta.quicklisp.org/quicklisp.lisp"
echo "If CCL does not exit automaticlly, enter (ccl:quit)"
ccl -l quicklisp.lisp -e "(quicklisp-quickstart:install)(ql:quickload :cl-ppcre)(ql:quickload :cl-who)(quit)" -b
echo "The script will now open a CCL session to install Quicklisp"
echo "If CCL does not exit automatically, enter (quit)"
read -n1 -rsp "Press any key to continue or Ctrl+C to abort: "
ccl -l quicklisp.lisp -e '(quicklisp-quickstart:install)' -e '(ql:quickload :cl-ppcre)' -e '(ql:quickload :cl-who)' -e '(quit)' -b
rm quicklisp.lisp
echo "Installed."
fi
echo "== 10: Testing babel2 =="
echo "== 10: Testing Babel2 =="
# We omit the browser test because it entails keeping ccl running and it complicates matters
# it usually never fails anyway (compared to possible graphviz/gnuplot issues)
echo "The test consists in opening a PDF file with a diagram and displaying a window with a sinus function."
@@ -1,8 +1,9 @@
(in-package :asdf)
(defsystem :corpus-processing
:depends-on (:utils)
:depends-on (:utils :cl-fad :cl-json :trivial-timeout)
:components
((:file "corpus-processing")))
(#+:lw(:file "corpus-processing")
(:file "json-corpus-processing")))
@@ -9,13 +9,14 @@
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
;; Load with:
;; (asdf:operate 'asdf:load-op :corpus-processing)
;; (ql:quickload :corpus-processing)
(in-package :utils)
(export '(process-corpus))
(defun process-corpus (&key function
function-kwargs
inputfile
outputfile
(number-of-threads 4)
@@ -29,6 +30,7 @@
(format t "~%~%Inputfile: ~a" inputfile)
(format t "~%Outputfile: ~a" outputfile)
(format t "~%Applying function: ~a" function)
(format t "~%Passing function arguments: ~a" function-kwargs)
(format t "~%~%Threads: ~a" number-of-threads)
(format t "~%Lines per thread: ~a" number-of-lines-per-thread)
(format t "~%Lines per batch: ~a" (* number-of-threads number-of-lines-per-thread))
@@ -51,12 +53,12 @@
(with-open-file (stream inputfile)
(dotimes (n number-of-complete-batches)
(format t "~%Started complete batch ~a/~a..." (+ 1 n) number-of-complete-batches)
(process-batch-multi-threaded function stream
(process-batch-multi-threaded function function-kwargs stream
number-of-threads number-of-lines-per-batch outputfile)
(format t "~%(Finished)"))
(when (> number-of-lines-in-last-batch 0)
(format t "~%Started extra batch...")
(process-batch-multi-threaded function stream
(process-batch-multi-threaded function function-kwargs stream
number-of-threads number-of-lines-in-last-batch outputfile)
(format t "~%(Finished)"))))
(let ((finish-time (get-universal-time)))
@@ -66,7 +68,7 @@
(format t "~%~%***************** Finished Corpus Processing *****************~%~%"))
(defun process-batch-multi-threaded (function stream number-of-threads number-of-lines outputfile)
(defun process-batch-multi-threaded (function function-kwargs stream number-of-threads number-of-lines outputfile)
"takes a batch of lines, divides them over threads, applies function on each line and writes
the output to outputfile"
(let ((list-of-thread-batches nil))
@@ -86,7 +88,7 @@
(let* ((mailbox (make-mailbox :name "batch-mailbox"))
(thread-list (mapcar #'(lambda (thread-batch)
(process-run-function "line-processing" '()
#'process-list-of-lines function thread-batch mailbox))
#'process-list-of-lines function function-kwargs thread-batch mailbox))
list-of-thread-batches))
(mailbox-messages nil))
;; Wait for messages
@@ -128,9 +130,15 @@
:remove-empty-subseqs t)))))
number-of-lines))
(defun process-list-of-lines (function list-of-lines mailbox)
(defun process-list-of-lines (function function-kwargs list-of-lines mailbox)
"applies function to list-of-lines and returns the processed list-of-lines"
(mailbox-send mailbox (cons (car list-of-lines) (mapcar function (cdr list-of-lines)))))
(mailbox-send mailbox
(cons (car list-of-lines)
(mapcar #'(lambda (line)
(if function-kwargs
(apply function line function-kwargs)
(funcall function line)))
(cdr list-of-lines)))))
Oops, something went wrong.

0 comments on commit 83c635f

Please sign in to comment.