Sync Fork

.github/workflows/mlora-test.yml

@@ -0,0 +1,55 @@
+name: Test LLM Models
+
+on: [push, pull_request]
+
+jobs:
+  test-llama:
+    runs-on: self-hosted
+    container:
+      image: mikecovlee/mlora:0.3.1
+      volumes:
+        - /home/lab/models:/host_models/
+      options: --gpus "device=1"
+    steps:
+      - uses: actions/checkout@v3
+      - name: finetune lora
+        run: |
+          python launch.py gen --template lora --tasks ./data/dummy_data.json --micro_batch_size 4
+          python launch.py train --base_model /host_models/TinyLlama-1.1B-intermediate-step-1431k-3T --attn_impl eager --dtype fp16
+      - name: inference with lora
+        run: |
+          python generate.py --base_model /host_models/TinyLlama-1.1B-intermediate-step-1431k-3T --template "./template/alpaca.json" --lora_weights "./casual_0" --instruction "What is m-LoRA?" --max_seq_len 64
+
+  test-gemma:
+    runs-on: self-hosted
+    container:
+      image: mikecovlee/mlora:0.3.1
+      volumes:
+        - /home/lab/models:/host_models/
+      options: --gpus "device=1"
+    steps:
+      - uses: actions/checkout@v3
+      - name: finetune lora
+        run: |
+          python launch.py gen --template lora --tasks ./data/dummy_data.json --micro_batch_size 4
+          python launch.py train --base_model /host_models/gemma-2b --attn_impl eager --dtype fp16
+      - name: inference with lora
+        run: |
+          python generate.py --base_model /host_models/gemma-2b --template "./template/alpaca.json" --lora_weights "./casual_0" --instruction "What is m-LoRA?" --max_seq_len 64
+
+  test-phi:
+    runs-on: self-hosted
+    container:
+      image: mikecovlee/mlora:0.3.1
+      volumes:
+        - /home/lab/models:/host_models/
+      options: --gpus "device=1"
+    steps:
+      - uses: actions/checkout@v3
+      - name: finetune lora
+        run: |
+          python launch.py gen --template lora_phi --tasks ./data/dummy_data.json --micro_batch_size 4
+          python launch.py train --base_model /host_models/phi-2 --attn_impl eager --dtype fp16
+      - name: inference with lora
+        run: |
+          python generate.py --base_model /host_models/phi-2 --template "./template/alpaca.json" --lora_weights "./casual_0" --instruction "What is m-LoRA?" --max_seq_len 64

.github/workflows/pre-commit

@@ -1,5 +1,5 @@
 #!/bin/bash
 
-flake8 . --count --show-source --statistics --max-line-length=127 --max-complexity 15 --ignore=E722,W504
+flake8 . --count --show-source --statistics --max-line-length=127 --max-complexity 15 --ignore=E722,W504,W503
 
 pytest

.github/workflows/python-test-dev.yml → .github/workflows/python-test.yml

@@ -1,12 +1,9 @@
 # This workflow will install Python dependencies, run tests and lint with a single version of Python
 # For more information see: https://docs.github.com/en/actions/automating-builds-and-tests/building-and-testing-python
 
-name: Test on Development Branches
+name: Test on Main
 
-on:
-  push:
-    branches:
-      - '*_dev'
+on: [push, pull_request]
 
 permissions:
   contents: read

.gitignore

@@ -174,6 +174,11 @@ template/*
 !data/dummy_data.json
 !template/test_data_demo.json
 !template/template_demo.json
+data_train.json
 
 # macOS junk files
-.DS_Store
+.DS_Store
+
+# LoRA adapters
+adapter_model.bin
+adapter_config.json

.launcher/lora.json

@@ -0,0 +1,35 @@
+{
+    "cutoff_len": 512,
+    "save_step": 1000,
+    "train_lora_candidate_num": 2,
+    "train_lora_simultaneously_num": 2,
+    "train_strategy": "optim",
+    "lora": [
+        {
+            "name": "lora",
+            "task_name": "",
+            "optim": "adamw",
+            "scheduler_type": "constant",
+            "warmup_steps": 0,
+            "lr": 2e-4,
+            "batch_size": 16,
+            "micro_batch_size": 8,
+            "test_batch_size": 32,
+            "num_epochs": 2,
+            "r": 80,
+            "lora_alpha": 160,
+            "lora_dropout": 0.05,
+            "target_modules": {
+                "q_proj": true,
+                "k_proj": true,
+                "v_proj": true,
+                "o_proj": true,
+                "w1_proj": true,
+                "w2_proj": true,
+                "w3_proj": true
+            },
+            "group_by_length": false,
+            "expand_side": "right"
+        }
+    ]
+}

.launcher/lora_phi.json

@@ -0,0 +1,34 @@
+{
+    "cutoff_len": 512,
+    "save_step": 1000,
+    "train_lora_candidate_num": 2,
+    "train_lora_simultaneously_num": 2,
+    "train_strategy": "optim",
+    "lora": [
+        {
+            "name": "lora",
+            "task_name": "",
+            "optim": "adamw",
+            "scheduler_type": "constant",
+            "warmup_steps": 0,
+            "lr": 2e-4,
+            "batch_size": 16,
+            "micro_batch_size": 8,
+            "test_batch_size": 32,
+            "num_epochs": 2,
+            "r": 80,
+            "lora_alpha": 160,
+            "lora_dropout": 0.05,
+            "target_modules": {
+                "q_proj": true,
+                "k_proj": true,
+                "v_proj": true,
+                "dense": true,
+                "fc1": true,
+                "fc2": true
+            },
+            "group_by_length": false,
+            "expand_side": "right"
+        }
+    ]
+}

.launcher/mixlora.json

@@ -0,0 +1,38 @@
+{
+    "cutoff_len": 512,
+    "save_step": 1000,
+    "train_lora_candidate_num": 2,
+    "train_lora_simultaneously_num": 2,
+    "train_strategy": "optim",
+    "lora": [
+        {
+            "name": "mixlora",
+            "task_name": "",
+            "optim": "adamw",
+            "scheduler_type": "constant",
+            "warmup_steps": 0,
+            "lr": 2e-4,
+            "batch_size": 16,
+            "micro_batch_size": 8,
+            "test_batch_size": 32,
+            "num_epochs": 2,
+            "r": 16,
+            "lora_alpha": 32,
+            "lora_dropout": 0.05,
+            "target_modules": {
+                "q_proj": true,
+                "k_proj": true,
+                "v_proj": true,
+                "o_proj": true,
+                "w1_proj": true,
+                "w2_proj": true,
+                "w3_proj": true
+            },
+            "routing_strategy": "mixtral",
+            "num_experts": 8,
+            "top_k": 2,
+            "group_by_length": false,
+            "expand_side": "right"
+        }
+    ]
+}

.launcher/mixlora_phi.json

@@ -0,0 +1,37 @@
+{
+    "cutoff_len": 512,
+    "save_step": 1000,
+    "train_lora_candidate_num": 2,
+    "train_lora_simultaneously_num": 2,
+    "train_strategy": "optim",
+    "lora": [
+        {
+            "name": "mixlora",
+            "task_name": "",
+            "optim": "adamw",
+            "scheduler_type": "constant",
+            "warmup_steps": 0,
+            "lr": 2e-4,
+            "batch_size": 16,
+            "micro_batch_size": 8,
+            "test_batch_size": 32,
+            "num_epochs": 2,
+            "r": 16,
+            "lora_alpha": 32,
+            "lora_dropout": 0.05,
+            "target_modules": {
+                "q_proj": true,
+                "k_proj": true,
+                "v_proj": true,
+                "dense": true,
+                "fc1": true,
+                "fc2": true
+            },
+            "routing_strategy": "mixtral",
+            "num_experts": 8,
+            "top_k": 2,
+            "group_by_length": false,
+            "expand_side": "right"
+        }
+    ]
+}