Merge branch 'feature/pipeline-refactor'

azure-pipelines.yml

@@ -7,8 +7,6 @@ variables:
     invoke pytest coverage pytest-cov pytest-doctestplus
     pandas scipy pyarrow numpy
     numba cython cffi
-  pip.extras: >
-    hpfrec implicit
 
 jobs:
 
@@ -18,3 +16,4 @@ jobs:
 # - template: build-support/test-windows-vanilla.yml
 - template: build-support/test-windows-conda.yml
 - template: build-support/test-mac-conda.yml
+- template: build-support/publish.yml

build-support/nix-coverage.yml

@@ -0,0 +1,28 @@
+steps:
+- script: |
+    source activate lkpy
+    coverage xml
+    coverage html
+  displayName: 'Process code coverage results'
+
+- script: |
+    source activate lkpy
+    pip install codecov
+
+    echo "Building for $BUILD_REASON"
+    cc_args=
+    if [ -n "$SYSTEM_PULLREQUEST_PULLREQUESTNUMBER" ]; then
+      ccargs="--pr $SYSTEM_PULLREQUEST_PULLREQUESTNUMBER"
+    fi
+    if [ -z "$CODECOV_TOKEN" ]; then
+      echo "no CODECOV_TOKEN :(" >&2
+    fi
+
+    codecov -c "$BUILD_SOURCEVERSION" -b $BUILD_SOURCEBRANCH $cc_args
+  displayName: 'Upload Coverage'
+
+- task: PublishCodeCoverageResults@1
+  inputs:
+    codeCoverageTool: Cobertura
+    summaryFileLocation: '$(System.DefaultWorkingDirectory)/coverage.xml'
+    reportDirectory: '$(System.DefaultWorkingDirectory)/htmlcov'

build-support/publish.yml

@@ -0,0 +1,84 @@
+
+jobs:
+- job: SourcePkg
+  pool:
+    vmImage: ubuntu-16.04
+
+  steps:
+  - bash: echo "##vso[task.prependpath]/usr/share/miniconda/bin"
+    displayName: Add conda to PATH
+
+  - script: conda create -n build -qy python=3.7
+    displayName: Create Conda env
+
+  - script: |
+      source activate build
+      python setup.py sdist -d $(Build.ArtifactStagingDirectory)
+    displayName: Build sdist
+
+  - task: PublishBuildArtifacts@1
+    inputs:
+      artifactName: 'source'
+
+- job: LinuxPkg
+  dependsOn: [SourcePkg, LinuxConda]
+  pool:
+    vmImage: ubuntu-16.04
+
+  steps:
+  - bash: echo "##vso[task.prependpath]/usr/share/miniconda/bin"
+    displayName: Add conda to PATH
+
+  - script: conda create -n build -qy python=3.7 conda-build anaconda-client
+    displayName: Create Conda env
+
+  - script: |
+      source activate build
+      conda build --output-folder $(Build.ArtifactStagingDirectory) conda
+    displayName: Build Conda packages
+
+  - task: PublishBuildArtifacts@1
+    inputs:
+      artifactName: 'conda'
+
+- job: WinPkg
+  dependsOn: [SourcePkg, WindowsConda]
+  pool:
+    vmImage: vs2017-win2016
+
+  steps:
+  - powershell: Write-Host "##vso[task.prependpath]$env:CONDA\Scripts"
+    displayName: Add conda to PATH
+
+  - script: conda create -n build -qy python=3.7 conda-build anaconda-client
+    displayName: Create Conda env
+
+  - script: |
+      call activate build
+      conda build --output-folder $(Build.ArtifactStagingDirectory) conda
+    displayName: Build Conda packages
+
+  - task: PublishBuildArtifacts@1
+    inputs:
+      artifactName: 'conda'
+
+- job: MacPkg
+  dependsOn: [SourcePkg, MacConda]
+  pool:
+    vmImage: 'macOS-10.13'
+
+  steps:
+  - bash: echo "##vso[task.prependpath]$CONDA/bin"
+    displayName: Add conda to PATH
+
+  - script: conda create -n build -qy python=3.7 conda-build anaconda-client
+    displayName: Create Conda env
+
+  - script: |
+      source activate build
+      conda build --output-folder $(Build.ArtifactStagingDirectory) conda
+    displayName: Build Conda packages
+  
+  - task: PublishBuildArtifacts@1
+    inputs:
+      artifactName: 'conda'

build-support/test-linux-conda.yml

@@ -16,12 +16,8 @@ jobs:
 
   - script: |
       conda create -n lkpy -qy $(conda.deps)
+      conda install -n lkpy -qy -c mdekstrand implicit hpfrec
     displayName: Create Anaconda environment
-  
-  - script: |
-      source activate lkpy
-      pip install $(pip.extras)
-    displayName: 'Extra PyPI deps'
 
   - script: |
       source activate lkpy
@@ -58,33 +54,7 @@ jobs:
       env NUMBA_DISABLE_JIT=1 python3 -m pytest --cov=lenskit --cov-append -m 'not slow'
     displayName: 'Non-JIT Test Coverage'
 
-  - script: |
-      source activate lkpy
-      coverage xml
-      coverage html
-    displayName: 'Process code coverage results'
-
-  - script: |
-      source activate lkpy
-      pip install codecov
-      
-      echo "Building for $BUILD_REASON"
-      cc_args=
-      if [ -n "$SYSTEM_PULLREQUEST_PULLREQUESTNUMBER" ]; then
-        ccargs="--pr $SYSTEM_PULLREQUEST_PULLREQUESTNUMBER"
-      fi
-      if [ -z "$CODECOV_TOKEN" ]; then
-        echo "no CODECOV_TOKEN :(" >&2
-      fi
-
-      codecov -c "$BUILD_SOURCEVERSION" -b $BUILD_SOURCEBRANCH $cc_args
-    displayName: 'Update CodeCov'
-
-  - task: PublishCodeCoverageResults@1
-    inputs:
-      codeCoverageTool: Cobertura
-      summaryFileLocation: '$(System.DefaultWorkingDirectory)/coverage.xml'
-      reportDirectory: '$(System.DefaultWorkingDirectory)/htmlcov'
+  - template: ./nix-coverage.yml
 
   - script: |
       source activate lkpy

build-support/test-linux-vanilla.yml

@@ -47,30 +47,4 @@ jobs:
       testResultsFiles: 'build/test-results.xml'
       testRunTitle: 'Publish test results for Python $(python.version)'
 
-  - script: |
-      source activate lkpy
-      coverage xml
-      coverage html
-    displayName: 'Process code coverage results'
-  
-  - script: |
-      source activate lkpy
-      pip install codecov
-
-      echo "Building for $BUILD_REASON"
-      cc_args=
-      if [ -n "$SYSTEM_PULLREQUEST_PULLREQUESTNUMBER" ]; then
-        ccargs="--pr $SYSTEM_PULLREQUEST_PULLREQUESTNUMBER"
-      fi
-      if [ -z "$CODECOV_TOKEN" ]; then
-        echo "no CODECOV_TOKEN :(" >&2
-      fi
-
-      codecov -c "$BUILD_SOURCEVERSION" -b $BUILD_SOURCEBRANCH $cc_args
-    displayName: 'Upload Coverage'
-
-  - task: PublishCodeCoverageResults@1
-    inputs:
-      codeCoverageTool: Cobertura
-      summaryFileLocation: '$(System.DefaultWorkingDirectory)/coverage.xml'
-      reportDirectory: '$(System.DefaultWorkingDirectory)/htmlcov'
+  - template: ./nix-coverage.yml

build-support/test-odd-environs.yml

@@ -20,7 +20,7 @@ jobs:
     
   - script: |
       mkdir -p build
-      python3 -m pytest --junitxml=build/test-results.xml
+      python3 -m pytest --junitxml=build/test-results.xml --cov=lenskit
     displayName: 'pytest'
 
   - task: PublishTestResults@2
@@ -29,22 +29,4 @@ jobs:
       testResultsFiles: 'build/test-results.xml'
       testRunTitle: 'Publish test results for Python $(python.version)'
 
-  - script: |
-      env NUMBA_DISABLE_JIT=1 python3 -m pytest --cov=lenskit --cov-report=xml --cov-report=html -m 'not eval'
-    displayName: 'Test Coverage'
-
-  - script: |
-      echo "Fetching Codecov script"
-      curl -o /tmp/codecov.sh https://codecov.io/bash
-      
-      echo "Building for $BUILD_REASON"
-      cc_args=
-      if [ -n "$SYSTEM_PULLREQUEST_PULLREQUESTNUMBER" ]; then
-        ccargs="-P $SYSTEM_PULLREQUEST_PULLREQUESTNUMBER"
-      fi
-      if [ -z "$CODECOV_TOKEN" ]; then
-        echo "no CODECOV_TOKEN :(" >&2
-      fi
-
-      bash /tmp/codecov.sh -C "$BUILD_SOURCEVERSION" -B $BUILD_SOURCEBRANCH $cc_args
-    displayName: 'Upload Coverage'
+  - template: ./nix-coverage.yml

build-support/test-windows-conda.yml

@@ -17,13 +17,10 @@ jobs:
   - script: conda update -y conda
     displayName: Update Conda
 
-  - script: conda create -n lkpy -qy $(conda.deps)
-    displayName: Create Conda environment
-
   - script: |
-      call activate lkpy
-      pip install $(pip.extras)
-    displayName: 'Extra PyPI deps'
+      conda create -n lkpy -qy $(conda.deps)
+      conda install -n lkpy -qy -c mdekstrand implicit hpfrec
+    displayName: Create Conda environment
 
   - script: |
       call activate lkpy

conda/meta.yaml

@@ -45,4 +45,4 @@ test:
     - pytest
     - pytest-arraydiff
   commands:
-    - pytest tests
+    - pytest tests -m "not slow"

doc/implicit.rst

@@ -3,8 +3,9 @@ Implicit
 
 .. module:: lenskit.algorithms.implicit
 
-This module provides a LensKit bridge to the implicit_ library implementing
-several implicit-feedback recommenders.
+This module provides a LensKit bridge to Ben Frederickson's implicit_ library
+implementing some implicit-feedback recommender algorithms, with an emphasis
+on matrix factorization.
 
 .. _implicit: https://implicit.readthedocs.io/en/latest/
 

tests/test_hpf.py

@@ -1,7 +1,7 @@
 import logging
 import pickle
 
-from lenskit.algorithms import hpf
+from lenskit.algorithms import hpf, basic
 
 import pandas as pd
 import numpy as np
@@ -44,38 +44,11 @@ def test_hpf_train_large(tmp_path):
     assert a2.n_users == algo.n_users
     assert a2.n_items == algo.n_items
 
+    csel = basic.UnratedItemCandidateSelector()
+    csel.fit(ratings)
+    rec = basic.TopN(algo, csel)
 
-@mark.slow
-@mark.eval
-@mark.skipif(not have_hpfrec, reason='hpfrec not installed')
-@mark.skipif(not lktu.ml100k.available, reason='ML100K data not present')
-def test_hpf_batch_accuracy():
-    import lenskit.crossfold as xf
-    from lenskit import batch, topn
-
-    ratings = lktu.ml100k.load_ratings()
-
-    algo = hpf.HPF(25)
-
-    def eval(train, test):
-        _log.info('running training')
-        train['rating'] = train.rating.astype(np.float_)
-        algo.fit(train)
-        users = test.user.unique()
-        _log.info('testing %d users', len(users))
-        candidates = topn.UnratedCandidates(train)
-        recs = batch.recommend(algo, users, 100, candidates)
-        return recs
-
-    folds = list(xf.partition_users(ratings, 5, xf.SampleFrac(0.2)))
-    test = pd.concat(f.test for f in folds)
-
-    recs = pd.concat(eval(train, test) for (train, test) in folds)
-
-    _log.info('analyzing recommendations')
-    rla = topn.RecListAnalysis()
-    rla.add_metric(topn.ndcg)
-    results = rla.compute(recs, test)
-    dcg = results.ndcg
-    _log.info('nDCG for users is %.4f', dcg.mean())
-    assert dcg.mean() > 0
+    for u in np.random.choice(ratings.user.unique(), size=50, replace=False):
+        recs = rec.recommend(u, 50)
+        assert len(recs) == 50
+        assert recs.item.nunique() == 50