Fix empty html docs, choking on the pyre extensions (#177)

facebookresearch · Jan 7, 2022 · 9cf3c38 · 9cf3c38
1 parent c17ec1f
commit 9cf3c38
Show file tree

Hide file tree

Showing 2 changed files with 38 additions and 0 deletions.
diff --git a/docs/requirements.txt b/docs/requirements.txt
@@ -6,3 +6,4 @@ sphinxcontrib-programoutput==0.16
 git+https://github.com/pytorch/pytorch_sphinx_theme.git#egg=pytorch_sphinx_theme
 torch>=1.6.0
 numpy>=1.19.5
+pyre-extensions == 0.0.23
diff --git a/xformers/factory/model_factory.py b/xformers/factory/model_factory.py
@@ -30,6 +30,43 @@ class xFormerConfig:
     the encoder and decoder positional encoding, as proposed for instance by
     `Using the Output Embedding to Improve Language Models`, Press et al.
 
+    A full config example is for instance as follows:
+
+    ::
+
+        xformer_config = [
+            {
+                "reversible": False,  # Turn on to test the effect of using reversible layers
+                "block_type": "encoder",
+                "num_layers": LAYERS,
+                "dim_model": EMB,
+                "layer_norm_style": "pre",
+                "position_encoding_config": {
+                    "name": "vocab",
+                    "seq_len": CONTEXT,
+                    "vocab_size": VOCAB_SIZE,
+                },
+                "multi_head_config": {
+                    "num_heads": NUM_HEADS,
+                    "residual_dropout": RES_DROP,
+                    "use_rotary_embeddings": True,
+                    "attention": {
+                        "name": ATTENTION_MECHANISM_STR,
+                        "dropout": ATTN_DROP,
+                        "causal": True,
+                        "seq_len": CONTEXT,
+                    },
+                },
+                "feedforward_config": {
+                    "name": "FusedMLP",  # Use MLP if Triton is not available
+                    "dropout": MLP_DROP,
+                    "activation": "gelu",
+                    "hidden_layer_multiplier": MLP_MULTIPLIER,
+                },
+            }
+        ]
+
+
     .. _`Using the Output Embedding to Improve Language Models`: https://arxiv.org/pdf/1608.05859.pdf
     """