Skip to content
Permalink
Browse files

Resolved #19. Added single experiment script.

  • Loading branch information...
chanlevan committed Mar 6, 2019
1 parent cb43842 commit cd02c3e04f852eb2b3cddc02b36c98db3a20dd9f
Showing with 381 additions and 793 deletions.
  1. 0 .github/ISSUE_TEMPLATE/bug_report.md
  2. 0 .github/ISSUE_TEMPLATE/feature_request.md
  3. +3 −7 .gitignore
  4. 0 LICENSE
  5. 0 MANIFEST.in
  6. +33 −11 README.md
  7. 0 ampligraph/__init__.py
  8. 0 ampligraph/datasets/__init__.py
  9. 0 ampligraph/datasets/datasets.py
  10. 0 ampligraph/evaluation/__init__.py
  11. 0 ampligraph/evaluation/metrics.py
  12. 0 ampligraph/evaluation/protocol.py
  13. 0 ampligraph/latent_features/__init__.py
  14. 0 ampligraph/latent_features/loss_functions.py
  15. 0 ampligraph/latent_features/misc.py
  16. 0 ampligraph/latent_features/model_utils.py
  17. 0 ampligraph/latent_features/models.py
  18. 0 ampligraph/latent_features/pool_functions.py
  19. 0 ampligraph/latent_features/regularizers.py
  20. 0 ampligraph/latent_features/score_functions.py
  21. 0 docs/Makefile
  22. 0 docs/ampligraph.datasets.rst
  23. 0 docs/ampligraph.evaluation.rst
  24. 0 docs/ampligraph.latent_features.rst
  25. 0 docs/api.rst
  26. 0 docs/biblio.rst
  27. 0 docs/changelog.md
  28. 0 docs/conf.py
  29. 0 docs/contacts.md
  30. 0 docs/dev.md
  31. 0 docs/examples.md
  32. +63 −32 docs/experiments.rst
  33. 0 docs/generated/ampligraph.datasets.load_ICEWS.rst
  34. 0 docs/generated/ampligraph.datasets.load_fb13.rst
  35. 0 docs/generated/ampligraph.datasets.load_fb15k.rst
  36. 0 docs/generated/ampligraph.datasets.load_fb15k_237.rst
  37. 0 docs/generated/ampligraph.datasets.load_from_csv.rst
  38. 0 docs/generated/ampligraph.datasets.load_from_ntriples.rst
  39. 0 docs/generated/ampligraph.datasets.load_from_rdf.rst
  40. 0 docs/generated/ampligraph.datasets.load_wn11.rst
  41. 0 docs/generated/ampligraph.datasets.load_wn18.rst
  42. 0 docs/generated/ampligraph.datasets.load_wn18rr.rst
  43. 0 docs/generated/ampligraph.datasets.load_yago3_10.rst
  44. 0 docs/generated/ampligraph.evaluation.create_mappings.rst
  45. 0 docs/generated/ampligraph.evaluation.evaluate_performance.rst
  46. 0 docs/generated/ampligraph.evaluation.generate_corruptions_for_eval.rst
  47. 0 docs/generated/ampligraph.evaluation.generate_corruptions_for_fit.rst
  48. 0 docs/generated/ampligraph.evaluation.hits_at_n_score.rst
  49. 0 docs/generated/ampligraph.evaluation.mar_score.rst
  50. 0 docs/generated/ampligraph.evaluation.mrr_score.rst
  51. 0 docs/generated/ampligraph.evaluation.quality_loss_mse.rst
  52. 0 docs/generated/ampligraph.evaluation.rank_score.rst
  53. 0 docs/generated/ampligraph.evaluation.select_best_model_ranking.rst
  54. 0 docs/generated/ampligraph.evaluation.to_idx.rst
  55. 0 docs/generated/ampligraph.latent_features.AbsoluteMarginLoss.rst
  56. 0 docs/generated/ampligraph.latent_features.ComplEx.rst
  57. 0 docs/generated/ampligraph.latent_features.DistMult.rst
  58. 0 docs/generated/ampligraph.latent_features.EmbeddingModel.rst
  59. 0 docs/generated/ampligraph.latent_features.HolE.rst
  60. 0 docs/generated/ampligraph.latent_features.L1Regularizer.rst
  61. 0 docs/generated/ampligraph.latent_features.L2Regularizer.rst
  62. 0 docs/generated/ampligraph.latent_features.Loss.rst
  63. 0 docs/generated/ampligraph.latent_features.NLLLoss.rst
  64. 0 docs/generated/ampligraph.latent_features.NoRegularizer.rst
  65. 0 docs/generated/ampligraph.latent_features.PairwiseLoss.rst
  66. 0 docs/generated/ampligraph.latent_features.Regularizer.rst
  67. 0 docs/generated/ampligraph.latent_features.SelfAdverserialLoss.rst
  68. 0 docs/generated/ampligraph.latent_features.TransE.rst
  69. 0 docs/generated/ampligraph.latent_features.restore_model.rst
  70. 0 docs/generated/ampligraph.latent_features.save_model.rst
  71. BIN docs/img/arch.png
  72. BIN docs/img/concept_embeddings_wordnet3_1_v0_2.png
  73. BIN docs/img/kg_lp.png
  74. BIN docs/img/kg_lp_step1.png
  75. BIN docs/img/kg_lp_step2.png
  76. BIN docs/img/kg_new.png
  77. BIN docs/img/pipeline.png
  78. 0 docs/index.rst
  79. 0 docs/install.md
  80. 0 docs/references.bib
  81. +0 −11 experiments/How-to-run.md
  82. +44 −0 experiments/config.json
  83. +0 −30 experiments/grid.json
  84. +0 −77 experiments/grid_search_exp.py
  85. +0 −25 experiments/input.json
  86. +0 −49 experiments/playground.py
  87. +238 −0 experiments/predictive_performance.py
  88. +0 −94 experiments/single_exp.py
  89. +0 −86 experiments/single_exp_0_es.py
  90. +0 −112 experiments/single_exp_append_unseen.py
  91. +0 −27 experiments/unseen_fb15k_237.csv
  92. +0 −187 experiments/unseen_wn18rr.csv
  93. +0 −45 experiments/utils.py
  94. 0 jenkins.sh
  95. 0 setup.py
  96. 0 tests/__init__.py
  97. 0 tests/ampligraph/__init__.py
  98. 0 tests/ampligraph/datasets/__init__.py
  99. 0 tests/ampligraph/datasets/test_datasets.py
  100. 0 tests/ampligraph/evaluation/__init__.py
  101. 0 tests/ampligraph/evaluation/test_metrics.py
  102. 0 tests/ampligraph/evaluation/test_protocol.py
  103. 0 tests/ampligraph/latent_features/__init__.py
  104. 0 tests/ampligraph/latent_features/test_misc.py
  105. 0 tests/ampligraph/latent_features/test_models.py
  106. 0 tests/ampligraph/latent_features/test_regularizers.py
  107. 0 tests/ampligraph/temporal/test_models_temp.py
No changes.
No changes.
10 .gitignore 100644 → 100755
@@ -112,12 +112,8 @@ venv.bak/
# pycharm
.idea/

#chan-hdt
model
.pytest_cache
output.txt
unittest_save_and_restore_models
playground

#vim
.swp

# experiments
experiments/config_test.json
0 LICENSE 100644 → 100755
No changes.
0 MANIFEST.in 100644 → 100755
No changes.
44 README.md 100644 → 100755
@@ -128,17 +128,39 @@ setx AMPLIGRAPH_DATA_HOME /YOUR/PATH/TO/datasets
```


## Predictive Power Evaluation (MRR Filtered)

| | WN18 | FB15k | FB15k | WN18RR |
|----------|------|-------|-------|--------|
| TransE | .?? | .?? | .?? | .?? |
| DistMult | .?? | .?? | .?? | .?? |
| ComplEx | .?? | .?? | .?? | .?? |
| HolE | .?? | .?? | .?? | .?? |
| RotatE | .?? | .?? | .?? | .?? |


## Predictive Power Evaluation

#### FB15K
|Model|MR|MRR|H@1|H@3|H@10|
|-----|--|---|---|---|----|
|TransE|159.02|0.47|0.35|0.55|0.72|
|DistMult|223.39|0.77|0.73|0.81|0.85|
|ComplEx|221.21|0.79|0.76|0.82|0.85|
|HolE|245.28|0.78|0.72|0.82|0.86|

#### WN18
|Model|MR|MRR|H@1|H@3|H@10|
|-----|--|---|---|---|----|
|TransE|.??|.??|.??|.??|.??|
|DistMult|761.78|0.76|0.62|0.90|0.96|
|ComplEx|656.88|0.96|0.96|0.97|0.97|
|HolE|587.58|0.96|0.95|0.97|0.97|

#### WN18RR
|Model|MR|MRR|H@1|H@3|H@10|
|-----|--|---|---|---|----|
|TransE|.??|.??|.??|.??|.??|
|DistMult|13047.16|0.41|0.39|0.42|0.46|
|ComplEx|15547.66|0.43|0.40|0.43|0.48|
|HolE|13364.40|0.43|0.41|0.44|0.48|

#### FB15K-237
|Model|MR|MRR|H@1|H@3|H@10|
|-----|--|---|---|---|----|
|TransE|.??|.??|.??|.??|.??|
|DistMult|.??|.??|.??|.??|.??|
|ComplEx|.??|.??|.??|.??|.??|
|HolE|.??|.??|.??|.??|.??|

## Documentation

0 ampligraph/__init__.py 100644 → 100755
No changes.
0 ampligraph/datasets/__init__.py 100644 → 100755
No changes.
0 ampligraph/datasets/datasets.py 100644 → 100755
No changes.
0 ampligraph/evaluation/__init__.py 100644 → 100755
No changes.
0 ampligraph/evaluation/metrics.py 100644 → 100755
No changes.
0 ampligraph/evaluation/protocol.py 100644 → 100755
No changes.
No changes.
No changes.
No changes.
No changes.
No changes.
No changes.
No changes.
No changes.
0 docs/Makefile 100644 → 100755
No changes.
0 docs/ampligraph.datasets.rst 100644 → 100755
No changes.
0 docs/ampligraph.evaluation.rst 100644 → 100755
No changes.
No changes.
0 docs/api.rst 100644 → 100755
No changes.
0 docs/biblio.rst 100644 → 100755
No changes.
0 docs/changelog.md 100644 → 100755
No changes.
0 docs/conf.py 100644 → 100755
No changes.
0 docs/contacts.md 100644 → 100755
No changes.
0 docs/dev.md 100644 → 100755
No changes.
0 docs/examples.md 100644 → 100755
No changes.
95 docs/experiments.rst 100644 → 100755
@@ -5,41 +5,72 @@ Performance
Predictive Performance
----------------------

We report the filtered MRR on the most common datasets used in literature.

+----------+------+-------+-------+--------+
| | WN18 | FB15k | FB15k | WN18RR |
+----------+------+-------+-------+--------+
| TransE | .?? | .?? | .?? | .?? |
+----------+------+-------+-------+--------+
| DistMult | .?? | .?? | .?? | .?? |
+----------+------+-------+-------+--------+
| ComplEx | .?? | .?? | .?? | .?? |
+----------+------+-------+-------+--------+
| HolE | .?? | .?? | .?? | .?? |
+----------+------+-------+-------+--------+
| RotatE | .?? | .?? | .?? | .?? |
+----------+------+-------+-------+--------+

The above results have been obtained with the following hyperparameters values:

+----------+-----------------+
| | Hyperparameters |
+----------+-----------------+
| TransE | .?? |
+----------+-----------------+
| DistMult | .?? |
+----------+-----------------+
| ComplEx | .?? |
+----------+-----------------+
| HolE | .?? |
+----------+-----------------+
| RotatE | .?? |
+----------+-----------------+
We report the filtered MR, MRR, Hit at 1, 3 and 10 on the most common datasets used in literature.

FB15K
-----

+----------+--------+------+------+------+------+
| Model | MR | MRR | H @ 1| H @ 3|H @ 10|
+----------+--------+------+------+------+------+
| TransE | 159.02 | 0.47 | 0.35 | 0.55 | 0.72 |
+----------+--------+------+------+------+------+
| DistMult | 223.39 | 0.77 | 0.73 | 0.81 | 0.85 |
+----------+--------+------+------+------+------+
| ComplEx | 221.21 | 0.79 | 0.76 | 0.82 | 0.85 |
+----------+--------+------+------+------+------+
| HolE | 245.28 | 0.78 | 0.72 | 0.82 | 0.86 |
+----------+--------+------+------+------+------+

WN18
----

+----------+--------+------+------+------+------+
| Model | MR | MRR | H @ 1| H @ 3|H @ 10|
+----------+--------+------+------+------+------+
| TransE | .?? | .?? | .?? | .?? | .?? |
+----------+--------+------+------+------+------+
| DistMult | 761.78 | 0.76 | 0.62 | 0.90 | 0.96 |
+----------+--------+------+------+------+------+
| ComplEx | 656.88 | 0.96 | 0.96 | 0.97 | 0.97 |
+----------+--------+------+------+------+------+
| HolE | 587.58 | 0.96 | 0.95 | 0.97 | 0.97 |
+----------+--------+------+------+------+------+

WN18RR
------
We removed 187 triples containing unseen entities on test dataset.

+----------+----------+------+------+------+------+
| Model | MR | MRR | H @ 1| H @ 3|H @ 10|
+----------+----------+------+------+------+------+
| TransE | .?? | .?? | .?? | .?? | .?? |
+----------+----------+------+------+------+------+
| DistMult | 13047.16 | 0.41 | 0.39 | 0.42 | 0.46 |
+----------+----------+------+------+------+------+
| ComplEx | 15547.66 | 0.43 | 0.40 | 0.43 | 0.48 |
+----------+----------+------+------+------+------+
| HolE | 13364.4 | 0.43 | 0.41 | 0.44 | 0.48 |
+----------+----------+------+------+------+------+

FB15K-237
---------
We removed 27 triples containing unseen entities on test dataset.

+----------+-----+-----+-----+-----+------+
| Model | MR | MRR |H @ 1|H @ 3|H @ 10|
+----------+-----+-----+-----+-----+------+
| TransE | .?? | .?? | .?? | .?? | .?? |
+----------+-----+-----+-----+-----+------+
| DistMult | .?? | .?? | .?? | .?? | .?? |
+----------+-----+-----+-----+-----+------+
| ComplEx | .?? | .?? | .?? | .?? | .?? |
+----------+-----+-----+-----+-----+------+
| HolE | .?? | .?? | .?? | .?? | .?? |
+----------+-----+-----+-----+-----+------+

Results in the table above can be reproduced by running the script below:
``$ ./predictive_performance.py -i dataset -m model``
``$ ./predictive_performance.py -d dataset -m model``



No changes.
No changes.
No changes.
No changes.
No changes.
No changes.
No changes.
No changes.
No changes.
No changes.
No changes.
No changes.
No changes.
No changes.
No changes.
No changes.
No changes.
No changes.
No changes.
No changes.
No changes.
No changes.
No changes.
No changes.
No changes.
No changes.
No changes.
No changes.
No changes.
No changes.
No changes.
No changes.
No changes.
No changes.
No changes.
No changes.
No changes.
No changes.
0 docs/img/arch.png 100644 → 100755
No changes.
No changes.
0 docs/img/kg_lp.png 100644 → 100755
No changes.
0 docs/img/kg_lp_step1.png 100644 → 100755
No changes.
0 docs/img/kg_lp_step2.png 100644 → 100755
No changes.
0 docs/img/kg_new.png 100644 → 100755
No changes.
0 docs/img/pipeline.png 100644 → 100755
No changes.
0 docs/index.rst 100644 → 100755
No changes.
0 docs/install.md 100644 → 100755
No changes.
0 docs/references.bib 100644 → 100755
No changes.

This file was deleted.

Oops, something went wrong.
@@ -0,0 +1,44 @@
{
"CUDA_VISIBLE_DEVICES": "1",
"UNSEEN_DATASETS": ["WN18RR", "FB15K-237"],
"load_function_map": {
"WN18": "load_wn18",
"FB15K": "load_fb15k",
"FB15K-237": "load_fb15k_237",
"WN18RR": "load_wn18rr"
},
"model_name_map": {
"TRANSE": "TransE",
"COMPLEX": "ComplEx",
"DISTMULT": "DistMult",
"HOLE": "HolE"
},

"hyperparams" : {
"FB15K":{
"TRANSE": {"batches_count": 10, "epochs": 4000, "k": 150, "eta": 5, "loss": "pairwise", "loss_params": {"margin": 0.5}, "embedding_model_params": {"norm": 1}, "regularizer": "L2", "regularizer_params": {"lambda": 0.0001}, "optimizer": "adam", "optimizer_params": {"lr": 0.0001}, "verbose": false, "seed": 0},
"DISTMULT": {"batches_count":50,"seed":0,"epochs":4000,"k":200,"eta":20,"regularizer":"None","regularizer_params" : {},"loss": "self_adverserial","loss_params": {"margin":1},"optimizer":"adam","optimizer_params":{"lr":0.0005},"verbose":false},
"COMPLEX": {"batches_count":100,"seed":0,"epochs":4000,"k":200,"eta":20,"regularizer":"None","regularizer_params" : {},"loss": "self_adverserial","loss_params":{"margin":1},"optimizer":"adam","optimizer_params":{"lr":0.0005},"verbose":true},
"HOLE": {"batches_count":50,"seed":0,"epochs":4000,"k":200,"eta":20,"regularizer":"None","regularizer_params" : {},"loss": "self_adverserial","loss_params": {"margin":1},"optimizer":"adam","optimizer_params":{"lr":0.0005},"verbose":false}
},
"WN18":{
"TRANSE": null,
"DISTMULT": {"batches_count":50,"seed":0,"epochs":4000,"k":200,"eta":20,"regularizer":"None","regularizer_params" : {},"loss": "nll","loss_params": {"margin":1},"optimizer":"adam","optimizer_params":{"lr":0.0005}, "verbose": false},
"COMPLEX": {"batches_count":50,"seed":0,"epochs":4000,"k":200,"eta":20,"regularizer":"None","regularizer_params" : {},"loss": "nll","loss_params": {"margin":1},"optimizer":"adam","optimizer_params":{"lr":0.0005},"verbose":false},
"HOLE": {"batches_count":50,"seed":0,"epochs":4000,"k":200,"eta":20,"regularizer":"None","regularizer_params" : {},"loss": "self_adverserial","loss_params": {"margin":1},"optimizer":"adam","optimizer_params":{"lr":0.0005},"verbose":false}
},
"WN18RR":{
"TRANSE": null,
"DISTMULT": {"batches_count":25,"seed":0,"epochs":4000,"k":200,"eta":20,"regularizer":"None","regularizer_params" : {},"loss": "self_adverserial","loss_params": {"margin":1},"optimizer":"adam","optimizer_params":{"lr":0.0005},"verbose":false},
"COMPLEX": {"batches_count":10,"seed":0,"epochs":4000, "k":200,"eta":20,"regularizer":"None","regularizer_params" : {},"loss": "nll","loss_params": {"margin":1},"optimizer":"adam","optimizer_params":{"lr":0.0005}, "verbose":false},
"HOLE": {"batches_count":50,"seed":0,"epochs":4000,"k":200,"eta":20,"regularizer":"None","regularizer_params" : {},"loss": "self_adverserial","loss_params": {"margin":1},"optimizer":"adam","optimizer_params":{"lr":0.0005},"verbose":false}
},
"FB15K-237":{
"TRANSE": {"batches_count": 10, "epochs": 3000, "k": 150, "eta": 10, "loss": "nll", "embedding_model_params": {"normalize_ent_emb": true}, "regularizer": "None", "regularizer_params": {"lambda": 0.0001}, "optimizer": "adam", "optimizer_params": {"lr": 0.0005}, "verbose": false, "seed": 0},
"DISTMULT": null,
"COMPLEX": null,
"HOLE": null
}
},
"no_early_stopping": ["FB15K_TRANSE"]
}

This file was deleted.

Oops, something went wrong.

This file was deleted.

Oops, something went wrong.

This file was deleted.

Oops, something went wrong.
Oops, something went wrong.

0 comments on commit cd02c3e

Please sign in to comment.
You can’t perform that action at this time.