From c2ff15cf36f10e83406d7645900bb5a9ab7be515 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rgen=20Brandt?= Date: Fri, 29 Dec 2017 00:33:34 +0100 Subject: [PATCH] docs improved. --- README.md | 6 +- cf_client.rst | 2 - cf_worker.rst | 8 +-- cf_worker_configuration.rst | 10 ++- cf_worker_deployment.rst | 11 ---- cf_worker_usage.rst | 29 +++++++++ conf.py | 2 +- cre.rst | 6 +- effi.rst | 11 ++++ effi_configuration.rst | 2 + effi_format.rst | 117 ++++++++++++++++++++++++++++++++++++ effi_usage.rst | 2 + index.rst | 5 +- syntax.rst | 4 +- 14 files changed, 185 insertions(+), 30 deletions(-) delete mode 100644 cf_client.rst delete mode 100644 cf_worker_deployment.rst create mode 100644 cf_worker_usage.rst create mode 100644 effi.rst create mode 100644 effi_configuration.rst create mode 100644 effi_format.rst create mode 100644 effi_usage.rst diff --git a/README.md b/README.md index 9580ec8..4b1a4cd 100644 --- a/README.md +++ b/README.md @@ -9,8 +9,10 @@ - [Sphinx](http://www.sphinx-doc.org/) - sudo apt-get install python-pip - pip install sphinx sphinx-rtd-theme +```bash +sudo apt-get install python-pip +pip install sphinx sphinx-rtd-theme +``` ## Authors diff --git a/cf_client.rst b/cf_client.rst deleted file mode 100644 index 07336a9..0000000 --- a/cf_client.rst +++ /dev/null @@ -1,2 +0,0 @@ -Cuneiform Client -================ \ No newline at end of file diff --git a/cf_worker.rst b/cf_worker.rst index 3238732..41b62b6 100644 --- a/cf_worker.rst +++ b/cf_worker.rst @@ -1,10 +1,10 @@ -Cuneiform Worker -================ +Cuneiform Worker (cf_worker) +============================ -This section describes the Cuneiform worker application :code:`cf_worker`. +This section describes the Cuneiform worker application (cf_worker). .. toctree:: :maxdepth: 2 - cf_worker_deployment + cf_worker_usage cf_worker_configuration diff --git a/cf_worker_configuration.rst b/cf_worker_configuration.rst index b4b767f..8d8dc1e 100644 --- a/cf_worker_configuration.rst +++ b/cf_worker_configuration.rst @@ -12,14 +12,17 @@ If no other configuration is provided, the Cuneiform worker application uses the { "n_wrk": "auto", - "cre_node": "node" + "cre_node": "node", + "wrk_dir": "./cuneiform/tmp", + "repo_dir": "./cuneiform/repo", + "data_dir": "./data" } This configuration can be superseded by - creating a *global* configuration file with a subset of the above entries in :code:`/usr/local/etc/cuneiform/cf_worker.json` - creating a *user-specific* configuration file with a subset of the above entries in :code:`~/.config/cuneiform/cf_worker.json` -- providing a custom configuration file from the command line (see :ref:`cf_worker_deployment`) +- providing a custom configuration file provided at the command line (see :ref:`cf_worker_deployment`) - providing explicit command line flags (see :ref:`cf_worker_deployment`) Configuration Entries @@ -29,6 +32,9 @@ The Cuneiform worker application has the following configuration entries: - :code:`n_wrk` The number of worker processes to start under this application master. A positive integer or the string :code:`"auto"`. If the entry is set to :code:`"auto"`, the application tries to guess the number of cores on the machine and start as many workers. If the number of cores could not be determined, one worker process is started. - :code:`cre_node` The Erlang node name on which the CRE application is running. Any string or :code:`"node"`. If the entry is set to :code:`"node"` the CRE application is looked up on the current Erlang node. +- :code:`work_dir` Local directory where the worker application performs work, i.e., copies input data to, runs scripts, and copies output data from. +- :code:`repo_dir` Possibly remote repository directory where intermediate and output data is read from and written to. +- :code:`data_dir` Possibly remote input data directory. This directory is never altered by the cf_worker or any other Cuneiform-related application. .. attention:: Leaving the CRE node at the default value :code:`"node"` will most likely result in an error. Typically, if the CRE computer's hostname is :code:`my_host` you need to set the CRE Erlang node to :code:`"cre@my_host"`. However, you have the freedom to set this parameter as early as in the global configuration file or as late as on the command line. \ No newline at end of file diff --git a/cf_worker_deployment.rst b/cf_worker_deployment.rst deleted file mode 100644 index 3750376..0000000 --- a/cf_worker_deployment.rst +++ /dev/null @@ -1,11 +0,0 @@ -.. _cf_worker_deployment: - -Deployment -========== - -Command Line ------------- - -Driving the Application from Erlang ------------------------------------ - diff --git a/cf_worker_usage.rst b/cf_worker_usage.rst new file mode 100644 index 0000000..70815cb --- /dev/null +++ b/cf_worker_usage.rst @@ -0,0 +1,29 @@ +.. _cf_worker_deployment: + +Usage +===== + +The Cuneiform worker application (cf_worker) needs to connect to a :ref:`cre`. Thus, in order to start a Cuneiform worker, the Erlang node name of the CRE instance needs to be known (e.g., :code:`cre@x240`). + +Command Line +------------ + +.. code-block:: none + + Usage: cf_worker [-v] [-h] [-c ] [-r ] + [-n []] + + -v, --version Show cf_worker version. + -h, --help Show command line options. + -c, --suppl_file Supplementary configuration file. + -r, --cre_node Erlang node running the CRE application (must be + specified). + -n, --n_wrk Number of worker processes to start. 0 means + auto-detect available processors. [default: 0] + + The cre_node argument must be specified. + + +Driving the Application from Erlang +----------------------------------- + diff --git a/conf.py b/conf.py index d8c9aa8..a0a2b96 100644 --- a/conf.py +++ b/conf.py @@ -45,7 +45,7 @@ # General information about the project. project = u'Cuneiform' -copyright = u'2016-2017, Jörgen Brandt' +copyright = u'2013-2017, Jörgen Brandt' # The version info for the project you're documenting, acts as replacement for # |version| and |release|, also used in various other places throughout the diff --git a/cre.rst b/cre.rst index 6de4daa..2fa0355 100644 --- a/cre.rst +++ b/cre.rst @@ -1,2 +1,4 @@ -Common Runtime Environment -========================== \ No newline at end of file +.. _cre: + +Common Runtime Environment (cre) +================================ \ No newline at end of file diff --git a/effi.rst b/effi.rst new file mode 100644 index 0000000..94807e7 --- /dev/null +++ b/effi.rst @@ -0,0 +1,11 @@ +Erlang Foreign Function Interface (effi) +======================================== + +This section describes the Erlang Foreign Function Interface library (effi). + +.. toctree:: + :maxdepth: 2 + + effi_usage + effi_configuration + effi_format \ No newline at end of file diff --git a/effi_configuration.rst b/effi_configuration.rst new file mode 100644 index 0000000..ae12c3b --- /dev/null +++ b/effi_configuration.rst @@ -0,0 +1,2 @@ +Configuration +============= \ No newline at end of file diff --git a/effi_format.rst b/effi_format.rst new file mode 100644 index 0000000..cf7a5fd --- /dev/null +++ b/effi_format.rst @@ -0,0 +1,117 @@ +Exchange Formats +================ + +Effi is driven by task instantiations or applications. Running effi on an application results in a reply. Here, we describe the application format and the reply format. + +Example +------- + +.. code-block:: json + + { "app_id": "1234", + "lambda": { "lambda_name": "bowtie2-build", + "arg_type_lst": [{ "arg_name": "fa", + "arg_type": "File", + "is_list": false }], + "ret_type_lst": [{ "arg_name": "idx", + "arg_type": "File", + "is_list": false }], + "lang": "Bash", + "script": "bowtie2-build $fa bt2idx\nidx=idx.tar\ntar cf $idx --remove-files bt2idx.*\n" }, + "arg_bind_lst": [{ "arg_name": "fa", + "value": "chr22.fa" }] } + +.. code-block:: json + + { "app_id": "1234", + "extended_script": "...", + "output": "...", + "stat": { "tstart": "...", + "duration": "..." }, + "result": { "status": "ok", + "ret_bind_lst": [{ "arg_name": "idx", + "value": "idx.tar" }] } } + +Application Format +------------------ + +The application format is what effi consumes. An application :code:`App` has the following form: + +.. code-block:: none + + App ::= { "app_id": S, + "lambda": Lambda, + "arg_bind_lst": [Bind, ...] } + +The application's lambda expression :code:`Lambda` has the following form: + +.. code-block:: none + + Lambda ::= { "lambda_name": S, + "arg_type_lst": [TArg, ...], + "ret_type_lst": [TArg, ...], + "lang": Lang, + "script": S } + +A lambda expression's :code:`arg_type_lst` pair lists specifications for the input parameters while the :code:`ret_type_lst` pair lists specifications for the output parameters of a lambda. An input or output parameter specification :code:`TArg` has the following form: + +.. code-block:: none + + TArg ::= { "arg_name": S, + "arg_type": Type, + "is_list": B } + +The :code:`arg_type` pair provides the base type of the argument. The base type :code:`Type` has the following form: + +.. code-block:: none + + Type ::= "Bool" + | "Str" + | "File" + + +The :code:`lang` pair provides the programming language in which the script is written. The language :code:`Lang` has the following form: + +.. code-block:: none + + Lang ::= "Bash" + | "Octave" + | "Perl" + | "Python" + | "R" + | "Racket" + +A lambda expression contains a list of argument bindings :code:`Bind` of the following form: + +.. code-block:: none + + Bind ::= { "arg_name": S, "value": S} + | { "arg_name": S, "value": [S, ...] } + +.. code-block:: none + + B ::= true + | false + +.. code-block:: none + + S ::= "..." + +Reply Format +------------ + +The reply format is what effi produces. + +.. code-block:: none + + Reply ::= { "app_id": S, + "extended_script": S, + "stat": { "t_start": S, "duration": S }, + "result": Result } + + +.. code-block:: none + + Result ::= { "status": "ok", "output": S, "ret_bind_lst": [Bind, ...] } + | { "status": "error", "output": S } + diff --git a/effi_usage.rst b/effi_usage.rst new file mode 100644 index 0000000..9fa12ad --- /dev/null +++ b/effi_usage.rst @@ -0,0 +1,2 @@ +Usage +===== \ No newline at end of file diff --git a/index.rst b/index.rst index c38b275..ea0be9b 100644 --- a/index.rst +++ b/index.rst @@ -22,9 +22,6 @@ The sources for this documentation are available on setup tutorial syntax - Manual: effi - Manual: gen_pnet - Manual: lib_combin cre cf_worker - cf_client \ No newline at end of file + effi \ No newline at end of file diff --git a/syntax.rst b/syntax.rst index 37fa13b..dfc5bbf 100644 --- a/syntax.rst +++ b/syntax.rst @@ -1,5 +1,5 @@ -Cuneiform Syntax -================ +Cuneiform Syntax Reference +========================== This section gives a formal definition of the Cuneiform syntax. The entry point of a Cuneiform program is the :ref:`syntax_script`.