Merge pull request #114 from whylabs/WHY-1944_log_rotation

log rotation with time

.bumpversion.cfg

@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 0.1.12-dev0
+current_version = 0.1.13-dev0
 commit = True
 tag = False
 parse = (?P<major>\d+)\.(?P<minor>\d+)\.(?P<patch>\d+)(\-(?P<release>[a-z]+)(?P<build>\d+))?

.github/workflows/continuous-integration.yml

@@ -14,7 +14,7 @@ jobs:
       fail-fast: false
       max-parallel: 6
       matrix:
-        python-version: [ 3.6, 3.7,3.8]
+        python-version: [ 3.6,3.7,3.8]
         os: [ubuntu-latest, macOS-latest]
 
     steps:
@@ -39,6 +39,20 @@ jobs:
     - name: Install dependencies
       run: |
         python -m pip install --upgrade pip
-        pip install tox tox-gh-actions pytest pytest-cov coverage 
+        pip install tox tox-gh-actions pytest pytest-cov coverage  
     - name: Test with tox
-      run: make test-all
+      run: make test-all
+    - name: Coveralls Parallel
+      if: matrix.os == 'ubuntu-latest'
+      uses: AndreMiras/coveralls-python-action@develop
+      with:
+        flag-name: run-${{ matrix.os }}-${{matrix.python-version}}
+        parallel: true
+  finish:
+    needs: test
+    runs-on: ubuntu-latest
+    steps:
+    - name: Coveralls Finished
+      uses: AndreMiras/coveralls-python-action@develop
+      with:
+        parallel-finished: true

docs/conf.py

@@ -108,7 +108,7 @@ def setup(app):
 # built documents.
 #
 # The short X.Y version.
-version = " 0.1.12-dev0"
+version = " 0.1.13-dev0"
 # The full version, including alpha/beta/rc tags.
 release = ""  # Is set by calling `setup.py docs`
 

notebooks/.gitignore

@@ -1 +1,2 @@
+output/
 whylogs/

requirements-dev.txt

@@ -17,7 +17,7 @@ chardet==3.0.4
 click==8.0.0a1
 cloudpickle==1.6.0
 colorama==0.4.4
-coverage==4.5.4
+coverage==5.3
 cryptography==3.2.1
 cycler==0.10.0
 databricks-cli==0.14.1

requirements-test.txt

@@ -15,4 +15,6 @@ s3fs==0.4.2
 s3transfer==0.3.3
 six==1.15.0
 urllib3==1.26.2
+coverage==5.3
 whylabs-datasketches==2.0.0b7
+freezegun==1.0.0

scripts/.gitignore

@@ -1,2 +1,3 @@
+output
 output.*
 output_*

scripts/session_dataloging.py

@@ -0,0 +1,19 @@
+import pandas as pd
+import time
+from whylogs.core.datasetprofile import dataframe_profile
+from whylogs import get_or_create_session
+
+if __name__ == "__main__":
+    df = pd.read_csv("data/lending-club-accepted-10.csv")
+
+    session = get_or_create_session()
+    with session.logger("test", with_rotation_time='s',cache=1) as logger:
+        profile = logger.log_dataframe(df)
+        time.sleep(2)
+        profile = logger.log_dataframe(df)
+        profile = logger.log_dataframe(df)
+        time.sleep(2)
+        profile = logger.log_dataframe(df)
+
+
+

setup.cfg

@@ -4,7 +4,7 @@
 
 [metadata]
 name = whylogs
-version = 0.1.12-dev0
+version = 0.1.13-dev0
 description = Add a short description here!
 author = Andy Dang🤖
 author-email = andy@whylabs.ai
@@ -78,7 +78,7 @@ testing =
     twine
     wheel
     setuptools_black
-    coverage<5  # Required for pycharm to run tests with coverage
+    coverage==5.3  # Required for pycharm to run tests with coverage
 viz =
     matplotlib
 mlflow =

src/whylogs/_version.py

@@ -1,3 +1,3 @@
 """WhyLabs version number."""
 
-__version__ = "0.1.12-dev0"
+__version__ = "0.1.13-dev0"

src/whylogs/app/config.py

@@ -124,11 +124,15 @@ def __init__(
         pipeline: str,
         writers: List[WriterConfig],
         verbose: bool = False,
+        with_rotation_time: str =None,
+        cache :int = None,
     ):
         self.project = project
         self.pipeline = pipeline
         self.verbose = verbose
         self.writers = writers
+        self.with_rotation_time = with_rotation_time
+        self.cache =cache 
 
     def to_yaml(self, stream=None):
         """
@@ -187,6 +191,8 @@ class SessionConfigSchema(Schema):
 
     project = fields.Str(required=True)
     pipeline = fields.Str(required=True)
+    with_rotation_time=fields.Str(required=False, validate=validate.OneOf(["s","m","h","d"]))
+    cache= fields.Int(required=False)
     verbose = fields.Bool(missing=False)
     writers = fields.List(
         fields.Nested(WriterConfigSchema),
@@ -220,21 +226,25 @@ def load_config():
     import os
 
     logger = getLogger(__name__)
-    cfg_candidates = [
-        os.environ.get("WHYLOGS_CONFIG"),
-        WHYLOGS_YML,
-        os.path.join(os.path.expanduser("~"), WHYLOGS_YML),
-        os.path.join("/opt/whylogs/", WHYLOGS_YML),
-    ]
-
-    for fpath in cfg_candidates:
+    cfg_candidates = {
+    "enviroment":  os.environ.get("WHYLOGS_CONFIG"),
+    "current_dir":    WHYLOGS_YML,
+    "home_dir":    os.path.join(os.path.expanduser("~"), WHYLOGS_YML),
+    "opt" :   os.path.join("/opt/whylogs/", WHYLOGS_YML),
+    }
+
+    location_found=None
+
+    for k, fpath in cfg_candidates.items():
         logger.debug(f"Attempting to load config file: {fpath}")
         if fpath is None or not os.path.isfile(fpath):
             continue
 
         try:
             with open(fpath, "rt") as f:
-                return SessionConfig.from_yaml(f)
+                session_config=SessionConfig.from_yaml(f)
+                location_found= {k, fpath} 
+                return session_config
         except IOError as e:
             logger.warning("Failed to load YAML config", e)
             pass