From 6f8252332d4e080715b66caec698e81fd223aebd Mon Sep 17 00:00:00 2001 From: "Le Van, Chan V" Date: Tue, 12 Mar 2019 11:28:12 +0000 Subject: [PATCH] update exp result and docs --- README.md | 2 +- docs/experiments.rst | 43 ++++++++++++++++++++------- experiments/config.json | 4 +-- experiments/predictive_performance.py | 8 ++--- 4 files changed, 40 insertions(+), 17 deletions(-) diff --git a/README.md b/README.md index 769301dd..e2323608 100755 --- a/README.md +++ b/README.md @@ -132,7 +132,7 @@ setx AMPLIGRAPH_DATA_HOME /YOUR/PATH/TO/datasets | |FB15k |WN18 |WN18RR |FB15K-237| |----------|------|-------|-------|---------| | TransE | 0.55 | 0.50 | 0.23 | 0.27(p) | -| DistMult | 0.79 | 0.83 | 0.44 | 0.26(s) | +| DistMult | 0.79 | 0.83 | 0.44 | 0.29(s) | | ComplEx | 0.79 | 0.94 | 0.44 | 0.27(s) | | HolE | 0.80 | 0.94 | 0.47 | 0.20(n) | | RotatE | .?? | .?? | .?? | .?? | diff --git a/docs/experiments.rst b/docs/experiments.rst index 5e1f5ee5..9699f409 100755 --- a/docs/experiments.rst +++ b/docs/experiments.rst @@ -133,9 +133,11 @@ WN18 seed: 0 ========== ======== ====== ====== ====== ====== ========================= -WN18RR +WN18RR ------ +We removed 420 triples (210 in validation, 210 in test) containing unseen entities + ========== ========= ====== ====== ====== ====== ======================== Model MR MRR H @ 1 H @ 3 H @ 10 Hyperparameters ========== ========= ====== ====== ====== ====== ======================== @@ -194,9 +196,11 @@ TransE 1532.28 0.23 0.07 0.34 0.50 batches_count: 100; ========== ========= ====== ====== ====== ====== ======================== -FB15K-237 +FB15K-237 --------- +We removed 37 triples (9 in validation, 28 in test set) containing unseen entities + ========= ======== ====== ====== ====== ====== ========================== Model MR MRR H @ 1 H @ 3 H @ 10 Hyperparameters ========= ======== ====== ====== ====== ====== ========================== @@ -217,21 +221,24 @@ TransE 373.63 0.27 0.18 0.30 0.44 batches_count: 10; lambda: 0.0001; seed: 0 -DistMult 269.68 0.26 0.17 0.29 0.44 batches_count: 6; + DistMult 441.22 0.29 0.20 0.32 0.48 batches_count: 50; + embedding_model_params: + norm: 1; epochs: 4000; - eta: 20; - k: 200; + eta: 50; + k: 400; loss: self_adversarial; loss_params: + alpha: 1; margin: 1; optimizer: adam; optimizer_params: - lr: 0.0005; + lr: 0.0001; regularizer: L2; regularizer_params: - lambda: 0.0001; + lambda: 1.0e-05; seed: 0 - + ComplEx 606.17 0.27 0.18 0.29 0.45 batches_count: 100; epochs: 4000; eta: 20; @@ -258,9 +265,25 @@ ComplEx 606.17 0.27 0.18 0.29 0.45 batches_count: 100; seed: 0 ========= ======== ====== ====== ====== ====== ========================== -Results in the table above can be reproduced by running the script below: +Results in the table above can be reproduced by running as below: + +`$ cd experiements` + +To run all experiements: + +`$ python predictive_performance.py` + +To run experiments of single dataset: + +`$ python predictive_performance.py -d dataset` + +To run experiements of single model: + +`$ python predictive_performance.py -m model` + +To run experiements of single model with single dataset: -`$ cd experiments && python predictive_performance.py -d dataset -m model` +`$ python predictive_performance.py -m model -d dataset` Runtime Performance diff --git a/experiments/config.json b/experiments/config.json index 016f926a..51e35534 100755 --- a/experiments/config.json +++ b/experiments/config.json @@ -35,10 +35,10 @@ }, "FB15K-237":{ "TRANSE": {"batches_count": 10, "epochs": 4000, "k": 150, "eta": 5, "loss": "pairwise", "loss_params": {"margin": 0.5}, "embedding_model_params": {"norm": 1}, "regularizer": "L2", "regularizer_params": {"lambda": 0.0001}, "optimizer": "adam", "optimizer_params": {"lr": 0.0001}, "verbose": false, "seed": 0}, - "DISTMULT": {"batches_count":6,"seed":0,"epochs":4000,"k":200,"eta":20,"regularizer":"L2","regularizer_params": {"lambda": 0.0001},"loss": "self_adversarial", "loss_params": {"margin": 1},"optimizer":"adam","optimizer_params":{"lr":0.0005},"verbose":false}, + "DISTMULT": {"batches_count":50,"seed":0,"epochs":4000,"k":400,"eta":50,"regularizer":"L2","regularizer_params": {"lambda": 0.0001},"loss": "self_adversarial", "loss_params": {"margin": 1, "alpha": 1},"optimizer":"adam","optimizer_params":{"lr":0.0005},"verbose":false}, "COMPLEX": {"batches_count":100,"seed":0,"epochs":4000,"k":200,"eta":50,"loss": "self_adversarial", "loss_params": {"margin": 1},"optimizer":"adam","optimizer_params":{"lr":0.0005},"verbose":false}, "HOLE": {"batches_count":100,"seed":0,"epochs":4000,"k":200,"eta":20,"regularizer":"L2","regularizer_params": {"lambda": 0.00001},"loss": "nll", "optimizer":"adam","optimizer_params":{"lr":0.0005},"verbose":false} - } + } }, "no_early_stopping": ["FB15K_TRANSE"] } \ No newline at end of file diff --git a/experiments/predictive_performance.py b/experiments/predictive_performance.py index da48a2d8..f08b99d9 100755 --- a/experiments/predictive_performance.py +++ b/experiments/predictive_performance.py @@ -7,7 +7,7 @@ import argparse, os, json, sys import numpy as np import logging -logging.basicConfig(level=logging.INFO) +logging.basicConfig(level=logging.DEBUG) from os import path from beautifultable import BeautifulTable @@ -72,7 +72,7 @@ def clean_data(train, valid, test, throw_valid = False): c_if+=1 count_test = count_test + 1 filtered_test = np.delete(test, idxs_test, axis=0) - logging.debug("fit validation case: shape test: {0} - filtered test: {1}: {2} unseen entities cleaned".format(test.shape, filtered_test.shape, c_if)) + logging.debug("fit validation case: shape test: {0} - filtered test: {1}: {2} triples with unseen entties removed".format(test.shape, filtered_test.shape, c_if)) return valid, filtered_test else: #filter valid @@ -89,7 +89,7 @@ def clean_data(train, valid, test, throw_valid = False): c_if+=1 count_valid = count_valid + 1 filtered_valid = np.delete(valid, idxs_valid, axis=0) - logging.debug("not fitting validation case: shape valid: {0} - filtered valid: {1}: {2} unseen entities cleaned".format(valid.shape, filtered_valid.shape, c_if)) + logging.debug("not fitting validation case: shape valid: {0} - filtered valid: {1}: {2} triples with unseen entties removed".format(valid.shape, filtered_valid.shape, c_if)) # filter test ent_test_diff_train = test_ent - train_ent @@ -106,7 +106,7 @@ def clean_data(train, valid, test, throw_valid = False): c_if+=1 count_test = count_test + 1 filtered_test = np.delete(test, idxs_test, axis=0) - logging.debug("not fitting validation case: shape test: {0} - filtered test: {1}: {2} unseen entities cleaned".format(test.shape, filtered_test.shape, c_if)) + logging.debug("not fitting validation case: shape test: {0} - filtered test: {1}: {2} triples with unseen entties removed".format(test.shape, filtered_test.shape, c_if)) return filtered_valid, filtered_test def run_single_exp(config, dataset, model):