Resolved #19. Added single experiment script.

Accenture · Mar 6, 2019 · cd02c3e · cd02c3e
1 parent cb43842
commit cd02c3e
Show file tree

Hide file tree

Showing 107 changed files with 381 additions and 793 deletions.
diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md
diff --git a/.github/ISSUE_TEMPLATE/feature_request.md b/.github/ISSUE_TEMPLATE/feature_request.md
diff --git a/.gitignore b/.gitignore
@@ -112,12 +112,8 @@ venv.bak/
 # pycharm
 .idea/
 
-#chan-hdt
-model
-.pytest_cache
-output.txt
-unittest_save_and_restore_models
-playground
-
 #vim
 .swp
+
+# experiments
+experiments/config_test.json
diff --git a/LICENSE b/LICENSE
diff --git a/MANIFEST.in b/MANIFEST.in
diff --git a/README.md b/README.md
@@ -128,17 +128,39 @@ setx AMPLIGRAPH_DATA_HOME /YOUR/PATH/TO/datasets
 ```
 
 
-## Predictive Power Evaluation (MRR Filtered)
-
-|          | WN18 | FB15k | FB15k | WN18RR |
-|----------|------|-------|-------|--------|
-| TransE   | .??  | .??   | .??   | .??    |
-| DistMult | .??  | .??   | .??   | .??    |
-| ComplEx  | .??  | .??   | .??   | .??    |
-| HolE     | .??  | .??   | .??   | .??    |
-| RotatE   | .??  | .??   | .??   | .??    |
-
-
+## Predictive Power Evaluation
+
+#### FB15K
+|Model|MR|MRR|H@1|H@3|H@10|
+|-----|--|---|---|---|----|
+|TransE|159.02|0.47|0.35|0.55|0.72|
+|DistMult|223.39|0.77|0.73|0.81|0.85|
+|ComplEx|221.21|0.79|0.76|0.82|0.85|
+|HolE|245.28|0.78|0.72|0.82|0.86|
+
+#### WN18
+|Model|MR|MRR|H@1|H@3|H@10|
+|-----|--|---|---|---|----|
+|TransE|.??|.??|.??|.??|.??|
+|DistMult|761.78|0.76|0.62|0.90|0.96|
+|ComplEx|656.88|0.96|0.96|0.97|0.97|
+|HolE|587.58|0.96|0.95|0.97|0.97|
+
+#### WN18RR
+|Model|MR|MRR|H@1|H@3|H@10|
+|-----|--|---|---|---|----|
+|TransE|.??|.??|.??|.??|.??|
+|DistMult|13047.16|0.41|0.39|0.42|0.46|
+|ComplEx|15547.66|0.43|0.40|0.43|0.48|
+|HolE|13364.40|0.43|0.41|0.44|0.48|
+
+#### FB15K-237
+|Model|MR|MRR|H@1|H@3|H@10|
+|-----|--|---|---|---|----|
+|TransE|.??|.??|.??|.??|.??|
+|DistMult|.??|.??|.??|.??|.??|
+|ComplEx|.??|.??|.??|.??|.??|
+|HolE|.??|.??|.??|.??|.??|
 
 ##  Documentation
 

diff --git a/ampligraph/__init__.py b/ampligraph/__init__.py
diff --git a/ampligraph/datasets/__init__.py b/ampligraph/datasets/__init__.py
diff --git a/ampligraph/datasets/datasets.py b/ampligraph/datasets/datasets.py
diff --git a/ampligraph/evaluation/__init__.py b/ampligraph/evaluation/__init__.py
diff --git a/ampligraph/evaluation/metrics.py b/ampligraph/evaluation/metrics.py
diff --git a/ampligraph/evaluation/protocol.py b/ampligraph/evaluation/protocol.py
diff --git a/ampligraph/latent_features/__init__.py b/ampligraph/latent_features/__init__.py
diff --git a/ampligraph/latent_features/loss_functions.py b/ampligraph/latent_features/loss_functions.py
diff --git a/ampligraph/latent_features/misc.py b/ampligraph/latent_features/misc.py
diff --git a/ampligraph/latent_features/model_utils.py b/ampligraph/latent_features/model_utils.py
diff --git a/ampligraph/latent_features/models.py b/ampligraph/latent_features/models.py
diff --git a/ampligraph/latent_features/pool_functions.py b/ampligraph/latent_features/pool_functions.py
diff --git a/ampligraph/latent_features/regularizers.py b/ampligraph/latent_features/regularizers.py
diff --git a/ampligraph/latent_features/score_functions.py b/ampligraph/latent_features/score_functions.py
diff --git a/docs/Makefile b/docs/Makefile
diff --git a/docs/ampligraph.datasets.rst b/docs/ampligraph.datasets.rst
diff --git a/docs/ampligraph.evaluation.rst b/docs/ampligraph.evaluation.rst
diff --git a/docs/ampligraph.latent_features.rst b/docs/ampligraph.latent_features.rst
diff --git a/docs/api.rst b/docs/api.rst
diff --git a/docs/biblio.rst b/docs/biblio.rst
diff --git a/docs/changelog.md b/docs/changelog.md
diff --git a/docs/conf.py b/docs/conf.py
diff --git a/docs/contacts.md b/docs/contacts.md
diff --git a/docs/dev.md b/docs/dev.md
diff --git a/docs/examples.md b/docs/examples.md
diff --git a/docs/experiments.rst b/docs/experiments.rst
@@ -5,41 +5,72 @@ Performance
 Predictive Performance
 ----------------------
 
-We report the filtered MRR on the most common datasets used in literature.
-
-+----------+------+-------+-------+--------+
-|          | WN18 | FB15k | FB15k | WN18RR |
-+----------+------+-------+-------+--------+
-| TransE   | .??  | .??   | .??   | .??    |
-+----------+------+-------+-------+--------+
-| DistMult | .??  | .??   | .??   | .??    |
-+----------+------+-------+-------+--------+
-| ComplEx  | .??  | .??   | .??   | .??    |
-+----------+------+-------+-------+--------+
-| HolE     | .??  | .??   | .??   | .??    |
-+----------+------+-------+-------+--------+
-| RotatE   | .??  | .??   | .??   | .??    |
-+----------+------+-------+-------+--------+
-
-The above results have been obtained with the following hyperparameters values:
-
-+----------+-----------------+
-|          | Hyperparameters |
-+----------+-----------------+
-| TransE   | .??             |
-+----------+-----------------+
-| DistMult | .??             |
-+----------+-----------------+
-| ComplEx  | .??             |
-+----------+-----------------+
-| HolE     | .??             |
-+----------+-----------------+
-| RotatE   | .??             |
-+----------+-----------------+
+We report the filtered MR, MRR, Hit at 1, 3 and 10 on the most common datasets used in literature.
 
+FB15K
+-----
+
++----------+--------+------+------+------+------+
+|  Model   |   MR   | MRR  | H @ 1| H @ 3|H @ 10|
++----------+--------+------+------+------+------+
+|  TransE  | 159.02 | 0.47 | 0.35 | 0.55 | 0.72 |
++----------+--------+------+------+------+------+
+| DistMult | 223.39 | 0.77 | 0.73 | 0.81 | 0.85 |
++----------+--------+------+------+------+------+
+| ComplEx  | 221.21 | 0.79 | 0.76 | 0.82 | 0.85 |
++----------+--------+------+------+------+------+
+|   HolE   | 245.28 | 0.78 | 0.72 | 0.82 | 0.86 |
++----------+--------+------+------+------+------+
+
+WN18
+----
+
++----------+--------+------+------+------+------+
+|  Model   |   MR   | MRR  | H @ 1| H @ 3|H @ 10|
++----------+--------+------+------+------+------+
+|  TransE  |  .??   | .??  | .??  | .??  | .??  |
++----------+--------+------+------+------+------+
+| DistMult | 761.78 | 0.76 | 0.62 | 0.90 | 0.96 |
++----------+--------+------+------+------+------+
+| ComplEx  | 656.88 | 0.96 | 0.96 | 0.97 | 0.97 |
++----------+--------+------+------+------+------+
+|   HolE   | 587.58 | 0.96 | 0.95 | 0.97 | 0.97 |
++----------+--------+------+------+------+------+
+
+WN18RR
+------
+We removed 187 triples containing unseen entities on test dataset.
+
++----------+----------+------+------+------+------+
+|  Model   |    MR    | MRR  | H @ 1| H @ 3|H @ 10|
++----------+----------+------+------+------+------+
+|  TransE  |   .??    | .??  | .??  | .??  | .??  |
++----------+----------+------+------+------+------+
+| DistMult | 13047.16 | 0.41 | 0.39 | 0.42 | 0.46 |
++----------+----------+------+------+------+------+
+| ComplEx  | 15547.66 | 0.43 | 0.40 | 0.43 | 0.48 |
++----------+----------+------+------+------+------+
+|   HolE   | 13364.4  | 0.43 | 0.41 | 0.44 | 0.48 |
++----------+----------+------+------+------+------+
+
+FB15K-237 
+---------
+We removed 27 triples containing unseen entities on test dataset.
+
++----------+-----+-----+-----+-----+------+
+|  Model   | MR  | MRR |H @ 1|H @ 3|H @ 10|
++----------+-----+-----+-----+-----+------+
+|  TransE  | .?? | .?? | .?? | .?? | .??  |
++----------+-----+-----+-----+-----+------+
+| DistMult | .?? | .?? | .?? | .?? | .??  |
++----------+-----+-----+-----+-----+------+
+| ComplEx  | .?? | .?? | .?? | .?? | .??  |
++----------+-----+-----+-----+-----+------+
+|   HolE   | .?? | .?? | .?? | .?? | .??  |
++----------+-----+-----+-----+-----+------+
 
 Results in the table above can be reproduced by running the script below:
-``$ ./predictive_performance.py -i dataset -m model``
+``$ ./predictive_performance.py -d dataset -m model``
 
 
 

diff --git a/docs/generated/ampligraph.datasets.load_ICEWS.rst b/docs/generated/ampligraph.datasets.load_ICEWS.rst
diff --git a/docs/generated/ampligraph.datasets.load_fb13.rst b/docs/generated/ampligraph.datasets.load_fb13.rst
diff --git a/docs/generated/ampligraph.datasets.load_fb15k.rst b/docs/generated/ampligraph.datasets.load_fb15k.rst
diff --git a/docs/generated/ampligraph.datasets.load_fb15k_237.rst b/docs/generated/ampligraph.datasets.load_fb15k_237.rst
diff --git a/docs/generated/ampligraph.datasets.load_from_csv.rst b/docs/generated/ampligraph.datasets.load_from_csv.rst
diff --git a/docs/generated/ampligraph.datasets.load_from_ntriples.rst b/docs/generated/ampligraph.datasets.load_from_ntriples.rst
diff --git a/docs/generated/ampligraph.datasets.load_from_rdf.rst b/docs/generated/ampligraph.datasets.load_from_rdf.rst
diff --git a/docs/generated/ampligraph.datasets.load_wn11.rst b/docs/generated/ampligraph.datasets.load_wn11.rst
diff --git a/docs/generated/ampligraph.datasets.load_wn18.rst b/docs/generated/ampligraph.datasets.load_wn18.rst
diff --git a/docs/generated/ampligraph.datasets.load_wn18rr.rst b/docs/generated/ampligraph.datasets.load_wn18rr.rst
diff --git a/docs/generated/ampligraph.datasets.load_yago3_10.rst b/docs/generated/ampligraph.datasets.load_yago3_10.rst
diff --git a/docs/generated/ampligraph.evaluation.create_mappings.rst b/docs/generated/ampligraph.evaluation.create_mappings.rst
diff --git a/docs/generated/ampligraph.evaluation.evaluate_performance.rst b/docs/generated/ampligraph.evaluation.evaluate_performance.rst
diff --git a/docs/generated/ampligraph.evaluation.generate_corruptions_for_eval.rst b/docs/generated/ampligraph.evaluation.generate_corruptions_for_eval.rst
diff --git a/docs/generated/ampligraph.evaluation.generate_corruptions_for_fit.rst b/docs/generated/ampligraph.evaluation.generate_corruptions_for_fit.rst
diff --git a/docs/generated/ampligraph.evaluation.hits_at_n_score.rst b/docs/generated/ampligraph.evaluation.hits_at_n_score.rst
diff --git a/docs/generated/ampligraph.evaluation.mar_score.rst b/docs/generated/ampligraph.evaluation.mar_score.rst
diff --git a/docs/generated/ampligraph.evaluation.mrr_score.rst b/docs/generated/ampligraph.evaluation.mrr_score.rst
diff --git a/docs/generated/ampligraph.evaluation.quality_loss_mse.rst b/docs/generated/ampligraph.evaluation.quality_loss_mse.rst
diff --git a/docs/generated/ampligraph.evaluation.rank_score.rst b/docs/generated/ampligraph.evaluation.rank_score.rst
diff --git a/docs/generated/ampligraph.evaluation.select_best_model_ranking.rst b/docs/generated/ampligraph.evaluation.select_best_model_ranking.rst
diff --git a/docs/generated/ampligraph.evaluation.to_idx.rst b/docs/generated/ampligraph.evaluation.to_idx.rst
diff --git a/docs/generated/ampligraph.latent_features.AbsoluteMarginLoss.rst b/docs/generated/ampligraph.latent_features.AbsoluteMarginLoss.rst
diff --git a/docs/generated/ampligraph.latent_features.ComplEx.rst b/docs/generated/ampligraph.latent_features.ComplEx.rst
diff --git a/docs/generated/ampligraph.latent_features.DistMult.rst b/docs/generated/ampligraph.latent_features.DistMult.rst
diff --git a/docs/generated/ampligraph.latent_features.EmbeddingModel.rst b/docs/generated/ampligraph.latent_features.EmbeddingModel.rst
diff --git a/docs/generated/ampligraph.latent_features.HolE.rst b/docs/generated/ampligraph.latent_features.HolE.rst
diff --git a/docs/generated/ampligraph.latent_features.L1Regularizer.rst b/docs/generated/ampligraph.latent_features.L1Regularizer.rst
diff --git a/docs/generated/ampligraph.latent_features.L2Regularizer.rst b/docs/generated/ampligraph.latent_features.L2Regularizer.rst
diff --git a/docs/generated/ampligraph.latent_features.Loss.rst b/docs/generated/ampligraph.latent_features.Loss.rst
diff --git a/docs/generated/ampligraph.latent_features.NLLLoss.rst b/docs/generated/ampligraph.latent_features.NLLLoss.rst
diff --git a/docs/generated/ampligraph.latent_features.NoRegularizer.rst b/docs/generated/ampligraph.latent_features.NoRegularizer.rst
diff --git a/docs/generated/ampligraph.latent_features.PairwiseLoss.rst b/docs/generated/ampligraph.latent_features.PairwiseLoss.rst
diff --git a/docs/generated/ampligraph.latent_features.Regularizer.rst b/docs/generated/ampligraph.latent_features.Regularizer.rst
diff --git a/docs/generated/ampligraph.latent_features.SelfAdverserialLoss.rst b/docs/generated/ampligraph.latent_features.SelfAdverserialLoss.rst
diff --git a/docs/generated/ampligraph.latent_features.TransE.rst b/docs/generated/ampligraph.latent_features.TransE.rst
diff --git a/docs/generated/ampligraph.latent_features.restore_model.rst b/docs/generated/ampligraph.latent_features.restore_model.rst
diff --git a/docs/generated/ampligraph.latent_features.save_model.rst b/docs/generated/ampligraph.latent_features.save_model.rst
diff --git a/docs/img/arch.png b/docs/img/arch.png
diff --git a/docs/img/concept_embeddings_wordnet3_1_v0_2.png b/docs/img/concept_embeddings_wordnet3_1_v0_2.png
diff --git a/docs/img/kg_lp.png b/docs/img/kg_lp.png
diff --git a/docs/img/kg_lp_step1.png b/docs/img/kg_lp_step1.png
diff --git a/docs/img/kg_lp_step2.png b/docs/img/kg_lp_step2.png
diff --git a/docs/img/kg_new.png b/docs/img/kg_new.png
diff --git a/docs/img/pipeline.png b/docs/img/pipeline.png
diff --git a/docs/index.rst b/docs/index.rst
diff --git a/docs/install.md b/docs/install.md
diff --git a/docs/references.bib b/docs/references.bib
diff --git a/experiments/How-to-run.md b/experiments/How-to-run.md
diff --git a/experiments/config.json b/experiments/config.json
@@ -0,0 +1,44 @@
+{
+    "CUDA_VISIBLE_DEVICES": "1",
+    "UNSEEN_DATASETS": ["WN18RR", "FB15K-237"],
+    "load_function_map": {
+        "WN18": "load_wn18",
+        "FB15K": "load_fb15k",
+        "FB15K-237": "load_fb15k_237",
+        "WN18RR": "load_wn18rr"
+    },
+    "model_name_map": {
+        "TRANSE": "TransE",
+        "COMPLEX": "ComplEx",
+        "DISTMULT": "DistMult",
+        "HOLE": "HolE"
+    },
+
+    "hyperparams" : {
+        "FB15K":{
+            "TRANSE":  {"batches_count": 10, "epochs": 4000, "k": 150, "eta": 5, "loss": "pairwise", "loss_params": {"margin": 0.5}, "embedding_model_params": {"norm": 1}, "regularizer": "L2", "regularizer_params": {"lambda": 0.0001}, "optimizer": "adam", "optimizer_params": {"lr": 0.0001}, "verbose": false, "seed": 0},
+            "DISTMULT": {"batches_count":50,"seed":0,"epochs":4000,"k":200,"eta":20,"regularizer":"None","regularizer_params" : {},"loss": "self_adverserial","loss_params": {"margin":1},"optimizer":"adam","optimizer_params":{"lr":0.0005},"verbose":false},
+            "COMPLEX": {"batches_count":100,"seed":0,"epochs":4000,"k":200,"eta":20,"regularizer":"None","regularizer_params" : {},"loss": "self_adverserial","loss_params":{"margin":1},"optimizer":"adam","optimizer_params":{"lr":0.0005},"verbose":true},
+            "HOLE": {"batches_count":50,"seed":0,"epochs":4000,"k":200,"eta":20,"regularizer":"None","regularizer_params" : {},"loss": "self_adverserial","loss_params": {"margin":1},"optimizer":"adam","optimizer_params":{"lr":0.0005},"verbose":false}        
+        },
+        "WN18":{
+            "TRANSE": null,
+            "DISTMULT": {"batches_count":50,"seed":0,"epochs":4000,"k":200,"eta":20,"regularizer":"None","regularizer_params" : {},"loss": "nll","loss_params": {"margin":1},"optimizer":"adam","optimizer_params":{"lr":0.0005}, "verbose": false},
+            "COMPLEX": {"batches_count":50,"seed":0,"epochs":4000,"k":200,"eta":20,"regularizer":"None","regularizer_params" : {},"loss": "nll","loss_params": {"margin":1},"optimizer":"adam","optimizer_params":{"lr":0.0005},"verbose":false},
+            "HOLE": {"batches_count":50,"seed":0,"epochs":4000,"k":200,"eta":20,"regularizer":"None","regularizer_params" : {},"loss": "self_adverserial","loss_params": {"margin":1},"optimizer":"adam","optimizer_params":{"lr":0.0005},"verbose":false}
+        },
+        "WN18RR":{
+            "TRANSE": null,
+            "DISTMULT": {"batches_count":25,"seed":0,"epochs":4000,"k":200,"eta":20,"regularizer":"None","regularizer_params" : {},"loss": "self_adverserial","loss_params": {"margin":1},"optimizer":"adam","optimizer_params":{"lr":0.0005},"verbose":false},
+            "COMPLEX": {"batches_count":10,"seed":0,"epochs":4000, "k":200,"eta":20,"regularizer":"None","regularizer_params" : {},"loss": "nll","loss_params": {"margin":1},"optimizer":"adam","optimizer_params":{"lr":0.0005}, "verbose":false},
+            "HOLE": {"batches_count":50,"seed":0,"epochs":4000,"k":200,"eta":20,"regularizer":"None","regularizer_params" : {},"loss": "self_adverserial","loss_params": {"margin":1},"optimizer":"adam","optimizer_params":{"lr":0.0005},"verbose":false}        
+        },
+        "FB15K-237":{
+            "TRANSE": {"batches_count": 10, "epochs": 3000, "k": 150, "eta": 10, "loss": "nll", "embedding_model_params": {"normalize_ent_emb": true}, "regularizer": "None", "regularizer_params": {"lambda": 0.0001}, "optimizer": "adam", "optimizer_params": {"lr": 0.0005}, "verbose": false, "seed": 0},
+            "DISTMULT": null,
+            "COMPLEX": null,
+            "HOLE": null        
+        }
+    },
+    "no_early_stopping": ["FB15K_TRANSE"]
+}
diff --git a/experiments/grid.json b/experiments/grid.json
diff --git a/experiments/grid_search_exp.py b/experiments/grid_search_exp.py
diff --git a/experiments/input.json b/experiments/input.json