Skip to content

Commit

Permalink
update exp result and docs
Browse files Browse the repository at this point in the history
  • Loading branch information
@chanlevan
chanlevan committed Mar 12, 2019
1 parent 664910a commit 6f82523
Show file tree
Hide file tree
Showing 4 changed files with 40 additions and 17 deletions.
2 changes: 1 addition & 1 deletion README.md
View file
Expand Up @@ -132,7 +132,7 @@ setx AMPLIGRAPH_DATA_HOME /YOUR/PATH/TO/datasets
| |FB15k |WN18 |WN18RR |FB15K-237|
|----------|------|-------|-------|---------|
| TransE | 0.55 | 0.50 | 0.23 | 0.27(p) |
| DistMult | 0.79 | 0.83 | 0.44 | 0.26(s) |
| DistMult | 0.79 | 0.83 | 0.44 | 0.29(s) |
| ComplEx | 0.79 | 0.94 | 0.44 | 0.27(s) |
| HolE | 0.80 | 0.94 | 0.47 | 0.20(n) |
| RotatE | .?? | .?? | .?? | .?? |
Expand Down
43 changes: 33 additions & 10 deletions docs/experiments.rst
View file
Expand Up @@ -133,9 +133,11 @@ WN18
seed: 0
========== ======== ====== ====== ====== ====== =========================

WN18RR
WN18RR
------

We removed 420 triples (210 in validation, 210 in test) containing unseen entities

========== ========= ====== ====== ====== ====== ========================
Model MR MRR H @ 1 H @ 3 H @ 10 Hyperparameters
========== ========= ====== ====== ====== ====== ========================
Expand Down Expand Up @@ -194,9 +196,11 @@ TransE 1532.28 0.23 0.07 0.34 0.50 batches_count: 100;
========== ========= ====== ====== ====== ====== ========================


FB15K-237
FB15K-237
---------

We removed 37 triples (9 in validation, 28 in test set) containing unseen entities

========= ======== ====== ====== ====== ====== ==========================
Model MR MRR H @ 1 H @ 3 H @ 10 Hyperparameters
========= ======== ====== ====== ====== ====== ==========================
Expand All @@ -217,21 +221,24 @@ TransE 373.63 0.27 0.18 0.30 0.44 batches_count: 10;
lambda: 0.0001;
seed: 0

DistMult 269.68 0.26 0.17 0.29 0.44 batches_count: 6;
DistMult 441.22 0.29 0.20 0.32 0.48 batches_count: 50;
embedding_model_params:
norm: 1;
epochs: 4000;
eta: 20;
k: 200;
eta: 50;
k: 400;
loss: self_adversarial;
loss_params:
alpha: 1;
margin: 1;
optimizer: adam;
optimizer_params:
lr: 0.0005;
lr: 0.0001;
regularizer: L2;
regularizer_params:
lambda: 0.0001;
lambda: 1.0e-05;
seed: 0

ComplEx 606.17 0.27 0.18 0.29 0.45 batches_count: 100;
epochs: 4000;
eta: 20;
Expand All @@ -258,9 +265,25 @@ ComplEx 606.17 0.27 0.18 0.29 0.45 batches_count: 100;
seed: 0
========= ======== ====== ====== ====== ====== ==========================

Results in the table above can be reproduced by running the script below:
Results in the table above can be reproduced by running as below:

`$ cd experiements`

To run all experiements:

`$ python predictive_performance.py`

To run experiments of single dataset:

`$ python predictive_performance.py -d dataset`

To run experiements of single model:

`$ python predictive_performance.py -m model`

To run experiements of single model with single dataset:

`$ cd experiments && python predictive_performance.py -d dataset -m model`
`$ python predictive_performance.py -m model -d dataset`


Runtime Performance
Expand Down
4 changes: 2 additions & 2 deletions experiments/config.json
View file
Expand Up @@ -35,10 +35,10 @@
},
"FB15K-237":{
"TRANSE": {"batches_count": 10, "epochs": 4000, "k": 150, "eta": 5, "loss": "pairwise", "loss_params": {"margin": 0.5}, "embedding_model_params": {"norm": 1}, "regularizer": "L2", "regularizer_params": {"lambda": 0.0001}, "optimizer": "adam", "optimizer_params": {"lr": 0.0001}, "verbose": false, "seed": 0},
"DISTMULT": {"batches_count":6,"seed":0,"epochs":4000,"k":200,"eta":20,"regularizer":"L2","regularizer_params": {"lambda": 0.0001},"loss": "self_adversarial", "loss_params": {"margin": 1},"optimizer":"adam","optimizer_params":{"lr":0.0005},"verbose":false},
"DISTMULT": {"batches_count":50,"seed":0,"epochs":4000,"k":400,"eta":50,"regularizer":"L2","regularizer_params": {"lambda": 0.0001},"loss": "self_adversarial", "loss_params": {"margin": 1, "alpha": 1},"optimizer":"adam","optimizer_params":{"lr":0.0005},"verbose":false},
"COMPLEX": {"batches_count":100,"seed":0,"epochs":4000,"k":200,"eta":50,"loss": "self_adversarial", "loss_params": {"margin": 1},"optimizer":"adam","optimizer_params":{"lr":0.0005},"verbose":false},
"HOLE": {"batches_count":100,"seed":0,"epochs":4000,"k":200,"eta":20,"regularizer":"L2","regularizer_params": {"lambda": 0.00001},"loss": "nll", "optimizer":"adam","optimizer_params":{"lr":0.0005},"verbose":false}
}
}
},
"no_early_stopping": ["FB15K_TRANSE"]
}
8 changes: 4 additions & 4 deletions experiments/predictive_performance.py
View file
Expand Up @@ -7,7 +7,7 @@
import argparse, os, json, sys
import numpy as np
import logging
logging.basicConfig(level=logging.INFO)
logging.basicConfig(level=logging.DEBUG)

from os import path
from beautifultable import BeautifulTable
Expand Down Expand Up @@ -72,7 +72,7 @@ def clean_data(train, valid, test, throw_valid = False):
c_if+=1
count_test = count_test + 1
filtered_test = np.delete(test, idxs_test, axis=0)
logging.debug("fit validation case: shape test: {0} - filtered test: {1}: {2} unseen entities cleaned".format(test.shape, filtered_test.shape, c_if))
logging.debug("fit validation case: shape test: {0} - filtered test: {1}: {2} triples with unseen entties removed".format(test.shape, filtered_test.shape, c_if))
return valid, filtered_test
else:
#filter valid
Expand All @@ -89,7 +89,7 @@ def clean_data(train, valid, test, throw_valid = False):
c_if+=1
count_valid = count_valid + 1
filtered_valid = np.delete(valid, idxs_valid, axis=0)
logging.debug("not fitting validation case: shape valid: {0} - filtered valid: {1}: {2} unseen entities cleaned".format(valid.shape, filtered_valid.shape, c_if))
logging.debug("not fitting validation case: shape valid: {0} - filtered valid: {1}: {2} triples with unseen entties removed".format(valid.shape, filtered_valid.shape, c_if))
# filter test
ent_test_diff_train = test_ent - train_ent

Expand All @@ -106,7 +106,7 @@ def clean_data(train, valid, test, throw_valid = False):
c_if+=1
count_test = count_test + 1
filtered_test = np.delete(test, idxs_test, axis=0)
logging.debug("not fitting validation case: shape test: {0} - filtered test: {1}: {2} unseen entities cleaned".format(test.shape, filtered_test.shape, c_if))
logging.debug("not fitting validation case: shape test: {0} - filtered test: {1}: {2} triples with unseen entties removed".format(test.shape, filtered_test.shape, c_if))
return filtered_valid, filtered_test

def run_single_exp(config, dataset, model):
Expand Down

0 comments on commit 6f82523

Please sign in to comment.