[Testing:Plagiarism] Test remaining tokenizers

.flake8

@@ -1,7 +1,7 @@
 [flake8]
 max-line-length = 100
 exclude=
-    tokenizer/python/input.py
+    tests/data/tokenizer/python/input.py
 
 per-file-ignores =
     tokenizer/mips/mips_tokenizer.py:W605

.github/workflows/lichen_run.yml

@@ -10,10 +10,7 @@ jobs:
       - uses: actions/setup-python@v2
         with:
           python-version: '3.6'
-      - name: Install Dependencies
-        run: |
-          sudo apt install libboost-all-dev
-      - name: Create Directory Structure
+      - name: Install Lichen
         run: |
           sudo bash ./tests/setup.sh
       - name: Run Tests

install_lichen.sh

@@ -10,12 +10,24 @@ fi
 
 echo -e "Installing lichen... "
 
-lichen_repository_dir=/usr/local/submitty/GIT_CHECKOUT/Lichen/
-lichen_installation_dir=/usr/local/submitty/Lichen/
+lichen_repository_dir=/usr/local/submitty/GIT_CHECKOUT/Lichen
+lichen_installation_dir=/usr/local/submitty/Lichen
 
-nlohmann_dir=${lichen_repository_dir}/../vendor/nlohmann/json/
+nlohmann_dir=${lichen_repository_dir}/../vendor/nlohmann/json
 
 
+########################################################################################################################
+# install dependencies
+
+# install clang
+apt-get install -y clang-6.0
+
+# boost
+apt-get install -y libboost-all-dev
+
+# python requirements
+pip install -r ${lichen_repository_dir}/requirements.txt
+
 ########################################################################################################################
 # get tools/source code from other repositories
 

requirements.txt

@@ -0,0 +1,11 @@
+# Python requirements for Lichen
+
+
+# Python tokenization
+parso==0.8.2
+
+# C/C++ tokenization
+clang==11.0
+
+# Java tokenization
+javac_parser==1.0.0