Restructure library (#6)

* restructure library * add versions * update pip * fix tests path * pin macos image version * adjust imports
sintel-dev · May 1, 2024 · 314ca5c · 314ca5c
1 parent 1313afe
commit 314ca5c
Show file tree

Hide file tree

Showing 25 changed files with 84 additions and 223 deletions.
diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml
@@ -15,15 +15,17 @@ jobs:
     strategy:
       matrix:
         python-version: [3.8]
-        os: [ubuntu-latest, macos-latest, windows-latest]
+        os: [ubuntu-latest, macos-13, windows-latest]
     steps:
     - uses: actions/checkout@v1
     - name: Set up Python ${{ matrix.python-version }}
       uses: actions/setup-python@v2
       with:
         python-version: ${{ matrix.python-version }}
     - name: Install package
-      run: pip install invoke .[dev]
+      run: |
+          python -m pip install --upgrade pip
+          python -m pip install invoke .[dev]
     - name: invoke lint
       run: invoke lint
 
@@ -51,15 +53,17 @@ jobs:
     strategy:
       matrix:
         python-version: ['3.8', '3.9', '3.10', '3.11']
-        os: [ubuntu-20.04, macos-latest]
+        os: [ubuntu-20.04, macos-13]
     steps:
     - uses: actions/checkout@v1
     - name: Set up Python ${{ matrix.python-version }}
       uses: actions/setup-python@v2
       with:
         python-version: ${{ matrix.python-version }}
     - name: Install package and dependencies
-      run: pip install invoke rundoc .
+      run: |
+          python -m pip install --upgrade pip
+          python -m pip install invoke rundoc .
     - name: invoke readme
       run: invoke readme
 
@@ -69,15 +73,17 @@ jobs:
     strategy:
       matrix:
         python-version: ['3.8', '3.9', '3.10', '3.11']
-        os: [ubuntu-20.04, macos-latest, windows-latest]
+        os: [ubuntu-latest, macos-13, windows-latest]
     steps:
     - uses: actions/checkout@v1
     - name: Set up Python ${{ matrix.python-version }}
       uses: actions/setup-python@v2
       with:
         python-version: ${{ matrix.python-version }}
     - name: Install package and dependencies
-      run: pip install invoke .[test]
+      run: |
+          python -m pip install --upgrade pip
+          python -m pip install invoke .[test]
     - name: invoke unit
       run: invoke unit
 
@@ -115,4 +121,4 @@ jobs:
   #   - name: Install package and dependencies
   #     run: pip install invoke jupyter matplotlib .
   #   - name: invoke tutorials
-  #     run: invoke tutorials
+  #     run: invoke tutorials
diff --git a/docs/conf.py b/docs/conf.py
@@ -18,7 +18,7 @@
 # relative to the documentation root, use os.path.abspath to make it
 # absolute, like shown here.
 
-import sphinx_rtd_theme # For read the docs theme
+# import sphinx_rtd_theme # For read the docs theme
 
 import sigllm
 
@@ -32,16 +32,22 @@
 # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom ones.
 extensions = [
     'm2r2',
+    'nbsphinx',
     'sphinx.ext.autodoc',
+    'sphinx.ext.autosummary',
     'sphinx.ext.githubpages',
     'sphinx.ext.viewcode',
     'sphinx.ext.napoleon',
-    'autodocsumm',
 ]
 
-autodoc_default_options = {
-    'autosummary': True,
-}
+ipython_execlines = [
+    "import pandas as pd",
+    "pd.set_option('display.width', 1000000)",
+    "pd.set_option('max_columns', 1000)",
+]
+
+autosummary_generate = True
+autodoc_typehints = "none"
 
 # Add any paths that contain templates here, relative to this directory.
 templates_path = ['_templates']
@@ -53,6 +59,18 @@
 # The master toctree document.
 master_doc = 'index'
 
+# Jupyter Notebooks
+nbsphinx_execute = 'never'
+nbsphinx_prolog = """
+.. raw:: html
+    <style>
+        .nbinput .prompt,
+        .nboutput .prompt {
+            display: none;
+        }
+    </style>
+"""
+
 # General information about the project.
 project = 'sigllm'
 slug = 'sigllm'
@@ -94,8 +112,10 @@
 # The theme to use for HTML and HTML Help pages.  See the documentation for
 # a list of builtin themes.
 #
-html_theme = 'sphinx_rtd_theme'
-html_theme_path = [sphinx_rtd_theme.get_html_theme_path()]
+# html_theme = 'sphinx_rtd_theme'
+# html_theme_path = [sphinx_rtd_theme.get_html_theme_path()]
+
+html_theme = 'pydata_sphinx_theme'
 
 # Readthedocs additions
 html_context = {
@@ -110,8 +130,8 @@
 # theme further.  For a list of options available for each theme, see the
 # documentation.
 html_theme_options = {
-    'collapse_navigation': False,
-    'display_version': False,
+    "github_url": "https://github.com/sintel-dev/sigllm",
+    "show_prev_next": True,
 }
 
 # Add any paths that contain custom static files (such as style sheets) here,

diff --git a/setup.py b/setup.py
@@ -12,16 +12,16 @@
     history = history_file.read()
 
 install_requires = [
-    'numpy',
+    'numpy>=1.17.5,<2.15',
     'openai',
-    'pandas',
-    'orion-ml',
-    'matplotlib',
-    'scikit-learn',
+    'pandas>=1,<2',
+    'orion-ml>=0.5,<0.8',
+    'scikit-learn>=0.22,<1.2',
     'tiktoken',
     'transformers',
-    'torch',
-    'accelerate'
+    'torch>=1.4',
+    'accelerate',
+    'sentencepiece'
 ]
 
 setup_requires = [
@@ -41,10 +41,23 @@
     'watchdog>=0.8.3',
 
     # docs
+    'docutils>=0.12,<0.18',
     'm2r2>=0.2.5,<0.3',
+    'nbsphinx>=0.5.0,<0.7',
     'Sphinx>=3,<3.3',
-    'sphinx_rtd_theme>=0.2.4,<0.5',
-    'autodocsumm>=0.1.10',
+    'pydata-sphinx-theme<0.5',
+    'markupsafe<2.1.0',
+    'ipython>=6.5,<9',
+    'Jinja2>=2,<3',
+
+    # fails on Sphinx < v3.4
+    'alabaster<=0.7.12',
+    # fails on Sphins < v5.0
+    'sphinxcontrib-applehelp<1.0.8',
+    'sphinxcontrib-devhelp<1.0.6',
+    'sphinxcontrib-htmlhelp<2.0.5',
+    'sphinxcontrib-serializinghtml<1.1.10',
+    'sphinxcontrib-qthelp<1.0.7',
 
     # style check
     'flake8>=3.7.7,<4',
@@ -80,6 +93,12 @@
         'Programming Language :: Python :: 3.11',
     ],
     description='Signals plus LLMs',
+    entry_points={
+        'mlblocks': [
+            'primitives=sigllm:MLBLOCKS_PRIMITIVES',
+            'pipelines=sigllm:MLBLOCKS_PIPELINES'
+        ]
+    },
     extras_require={
         'test': tests_require,
         'dev': development_requires + tests_require,

diff --git a/sigllm/__init__.py b/sigllm/__init__.py
@@ -5,3 +5,12 @@
 __author__ = 'MIT Data To AI Lab'
 __email__ = 'dailabmit@gmail.com'
 __version__ = '0.0.1.dev0'
+
+import os
+
+_BASE_PATH = os.path.abspath(os.path.dirname(__file__))
+MLBLOCKS_PRIMITIVES = os.path.join(_BASE_PATH, 'primitives', 'jsons')
+MLBLOCKS_PIPELINES = tuple([
+    os.path.join(_BASE_PATH, 'pipelines', 'prompter'),
+    os.path.join(_BASE_PATH, 'pipelines', 'detector')
+])
diff --git a/sigllm/sigllm.py → sigllm/core.py b/sigllm/sigllm.py → sigllm/core.py
@@ -5,8 +5,8 @@
 
 This module contains functions that get LLM's anomaly detection results.
 """
-from anomalies import get_anomaly_list_within_seq, str2idx
-from data import sig2str
+from sigllm.primitives.prompting.anomalies import get_anomaly_list_within_seq, str2idx
+from sigllm.primitives.prompting.data import sig2str
 
 
 def get_anomalies(seq, msg_func, model_func, num_iters=1, alpha=0.5):

diff --git a/sigllm/pipelines/detector/.gitkeep b/sigllm/pipelines/detector/.gitkeep
diff --git a/sigllm/pipelines/prompter/.gitkeep b/sigllm/pipelines/prompter/.gitkeep
diff --git a/sigllm/primitives/__init__.py b/sigllm/primitives/__init__.py
diff --git a/sigllm/forecasting/__init__.py → sigllm/primitives/forecasting/__init__.py b/sigllm/forecasting/__init__.py → sigllm/primitives/forecasting/__init__.py
diff --git a/sigllm/forecasting/gpt.py → sigllm/primitives/forecasting/gpt.py b/sigllm/forecasting/gpt.py → sigllm/primitives/forecasting/gpt.py
diff --git a/sigllm/forecasting/gpt_messages.json → .../primitives/forecasting/gpt_messages.json b/sigllm/forecasting/gpt_messages.json → .../primitives/forecasting/gpt_messages.json
diff --git a/sigllm/forecasting/huggingface.py → sigllm/primitives/forecasting/huggingface.py b/sigllm/forecasting/huggingface.py → sigllm/primitives/forecasting/huggingface.py
diff --git a/sigllm/primitives/jsons/.gitkeep b/sigllm/primitives/jsons/.gitkeep
diff --git a/sigllm/primitives/postprocessing.py b/sigllm/primitives/postprocessing.py
diff --git a/sigllm/forecasting/preprocessing.py → sigllm/primitives/preprocessing.py b/sigllm/forecasting/preprocessing.py → sigllm/primitives/preprocessing.py
diff --git a/sigllm/primitives/prompting/__init__.py b/sigllm/primitives/prompting/__init__.py
diff --git a/sigllm/anomalies.py → sigllm/primitives/prompting/anomalies.py b/sigllm/anomalies.py → sigllm/primitives/prompting/anomalies.py
diff --git a/sigllm/data.py → sigllm/primitives/prompting/data.py b/sigllm/data.py → sigllm/primitives/prompting/data.py
diff --git a/sigllm/gpt.py → sigllm/primitives/prompting/gpt.py b/sigllm/gpt.py → sigllm/primitives/prompting/gpt.py
diff --git a/tests/primitives/forecasting/test_gpt.py b/tests/primitives/forecasting/test_gpt.py
diff --git a/tests/test_anomalies.py → tests/primitives/prompting/test_anomalies.py b/tests/test_anomalies.py → tests/primitives/prompting/test_anomalies.py
@@ -4,7 +4,7 @@
 import pandas as pd
 from pytest import fixture
 
-from sigllm.anomalies import (
+from sigllm.primitives.prompting.anomalies import (
     get_anomaly_list_within_seq, idx2time, merge_anomaly_seq, str2idx, str2sig,)
 
 

diff --git a/tests/test_data.py → tests/primitives/prompting/test_data.py b/tests/test_data.py → tests/primitives/prompting/test_data.py
@@ -3,7 +3,7 @@
 import numpy as np
 from pytest import fixture
 
-from sigllm.data import rolling_window_sequences, sig2str
+from sigllm.primitives.prompting.data import rolling_window_sequences, sig2str
 
 
 @fixture

diff --git a/tests/forecasting/test_preprocessing.py → tests/primitives/test_preprocessing.py b/tests/forecasting/test_preprocessing.py → tests/primitives/test_preprocessing.py
@@ -2,7 +2,7 @@
 
 import numpy as np
 
-from sigllm.forecasting.preprocessing import Signal2String
+from sigllm.primitives.preprocessing import Signal2String
 
 
 class Signal2StringTest(unittest.TestCase):