Merge pull request #549 from usc-isi-i2/dev

Dev

README.md

@@ -20,6 +20,37 @@ KGTK can process Wikidata-sized KGs with billions of edges on a laptop. We have
 
 KGTK is open source software, well documented, actively used and developed, and released using the MIT license. We invite the community to try KGTK. It is easy to get started with our tutorial notebooks available and executable online.
 
+## Installation
+
+> The following instructions install KGTK and the KGTK Jupyter Notebooks on
+    Linux and MacOS systems.
+
+    If you want to install KGTK on a Microsoft Windows system, please
+    contact the KGTK team.
+
+Our KGTK installations use a Conda virtual environment. If you
+don't have the Conda tools installed, follow this
+[guide](https://docs.conda.io/projects/conda/en/latest/user-guide/install/) to
+install it. We recommend installing Miniconda installation rather than the
+full Anaconda installation.
+
+Next, execute the following steps to install the latest stable release
+of KGTK:
+
+```bash
+conda create -n kgtk-env python=3.8
+conda activate kgtk-env
+conda install -c conda-forge graph-tool
+conda install -c conda-forge jupyterlab
+pip --no-cache install -U kgtk
+python -m spacy download en_core_web_sm
+```
+
+Please see our [installation document](/docs/install.md) for more details.  If
+you encounter problems with your installation, or are interested in a detailed
+explanation of these commands, [read more about the installation procedure
+here](KGTK-Installation-Procedure-Details.md).
+
 ## Getting started
 
 ### Online Documentation
@@ -31,21 +62,12 @@ https://kgtk.readthedocs.io/en/latest/
 
 ### KGTK Notebooks
 
-The [examples folder](examples/) provides a larger and constantly increasing number of easy-to-follow Jupyter Notebooks which showcase different functionalities of KGTK. These include computing:
-* Embeddings for ConceptNet nodes
-* Graph statistics over a curated subset of Wikidata
-* Reachable occupations for selected people in Wikidata
-* PageRank over Wikidata
-* etc.
+For examples of using KGTK, please see our [Tutorial Notebooks](https://github.com/usc-isi-i2/kgtk-notebooks ).
 
 ## Releases
 
 * See all [source code releases](https://github.com/usc-isi-i2/kgtk/releases)
 
-## Installation
-
-Please see our [installation document](/docs/install.md) for installation procedures.
-
 ## KGTK Text Search API
 
 The documentation for the KGTK Text Search API is [here](https://github.com/usc-isi-i2/kgtk-search)

docker/Dockerfile

@@ -28,6 +28,10 @@ RUN conda install -c conda-forge jupyterlab
 
 RUN pip install chardet
 
+RUN pip install gensim
+
+RUN pip install papermill
+
 ARG NB_USER=jovyan
 ARG NB_UID=1000
 ENV USER ${NB_USER}

docker/dev/Dockerfile

@@ -28,6 +28,10 @@ RUN conda install -c conda-forge jupyterlab
 
 RUN pip install chardet
 
+RUN pip install gensim
+
+RUN pip install papermill
+
 ARG NB_USER=jovyan
 ARG NB_UID=1000
 ENV USER ${NB_USER}

kgtk/__init__.py

@@ -1 +1 @@
-__version__ = '1.0.0'
+__version__ = '1.0.1'

kgtk/functions.py

@@ -23,6 +23,7 @@ def kgtk(arg1: typing.Union[str, pandas.DataFrame],
          auto_display_html: typing.Optional[bool] = None,
          auto_display_json: typing.Optional[bool] = None,
          auto_display_md: typing.Optional[bool] = None,
+         unquote_column_names: typing.Optional[bool] = None,
          bash_command: typing.Optional[str] = None,
          kgtk_command: typing.Optional[str] = None,
          )->typing.Optional[pandas.DataFrame]:
@@ -60,6 +61,10 @@ def kgtk(arg1: typing.Union[str, pandas.DataFrame],
 
         This parameter controls the processing of MarkDown output.  See below.
 
+    unquote_column_names=True/False (default True)
+
+        Convert string column names to symbols.
+
     bash_command=CMD (default 'bash')
 
         This parameter specifies the name of the shell interpreter.  If the
@@ -117,11 +122,12 @@ def kgtk(arg1: typing.Union[str, pandas.DataFrame],
     Environment Variables
     =========== =========
 
-    This modeule directly uses the following environment variables:
+    This module directly uses the following environment variables:
 
     KGTK_AUTO_DISPLAY_HTML
     KGTK_AUTO_DISPLAY_JSON
     KGTK_AUTO_DISPLAY_MD
+    KGTK_UNQUOTE_COLUMN_NAMES
     KGTK_BASH_COMMAND
     KGTK_KGTK_COMMAND
 
@@ -142,6 +148,8 @@ def kgtk(arg1: typing.Union[str, pandas.DataFrame],
         auto_display_json = os.getenv("KGTK_AUTO_DISPLAY_JSON", "true").lower() in ["true", "yes", "y"]
     if auto_display_md is None:
         auto_display_md = os.getenv("KGTK_AUTO_DISPLAY_MD", "false").lower() in ["true", "yes", "y"]
+    if unquote_column_names is None:
+        unquote_column_names = os.getenv("KGTK_UNQUOTE_COLUMN_NAMES", "true").lower() in ["true", "yes", "y"]
 
     # Why not os.getenv("KGTK_BASH_COMMAND", "bash")? Splitting it up makes
     # mypy happier.
@@ -189,6 +197,18 @@ def kgtk(arg1: typing.Union[str, pandas.DataFrame],
                               doublequote=False,
                               escapechar='\\',
                               )
+        if unquote_column_names:
+            # Pandas will have treated the column names as strings and quoted
+            # them.  By convention, KGTK column names are symbols.  So, we will
+            # remove double quotes from the outside of each column name.
+            #
+            # TODO: Handle the troublesome case of a double quote inside a column
+            # name.
+            header, body = in_tsv.split('\n', 1)
+            column_names = header.split('\t')
+            column_names = [x[1:-1] if x.startswith('"') else x for x in column_names ]
+            header = "\t".join(column_names)
+            in_tsv = header + "\n" + body
 
     # Execute the KGTK command pipeline:
     outbuf: StringIO = StringIO()
@@ -247,11 +267,20 @@ def kgtk(arg1: typing.Union[str, pandas.DataFrame],
         # Assume that anything else is KGTK formatted output.  Convert it to a
         # pandas DataFrame and return it.
         #
+        # TODO: Test this conversion with all KTK datatypes.  Language-qualified
+        # strings are problematic.  Check what happens to quantites, date/times,
+        # and locations.
+        #
         # TODO: Remove the escape character from internal `|` characters?
         # If we do that, should we detect KGTK lists and complain?
         # `\|` -> `|`
         outbuf.seek(0)
-        result = pandas.read_csv(outbuf, sep='\t')
+        result = pandas.read_csv(outbuf,
+                                 sep='\t',
+                                 quotechar='"',
+                                 doublequote=False,
+                                 escapechar='\\',
+                                 )
 
     outbuf.close()