Skip to content

Commit

Permalink
added test file output to .gitignore
Browse files Browse the repository at this point in the history
  • Loading branch information
mamonu committed Aug 21, 2020
1 parent 7b02acf commit baaa6d3
Show file tree
Hide file tree
Showing 4 changed files with 7 additions and 230 deletions.
3 changes: 3 additions & 0 deletions .gitignore
Expand Up @@ -143,3 +143,6 @@ dmypy.json

# gitignore exception for docs files (otherwise docs/build does not get saved in repo
!docs/*

# some files created by certain tests need to not be uploaded to the repo
saved_model.json
229 changes: 0 additions & 229 deletions saved_model.json

This file was deleted.

3 changes: 2 additions & 1 deletion splink/term_frequencies.py
Expand Up @@ -3,6 +3,7 @@

import logging
import math
import warnings

try:
from pyspark.sql.dataframe import DataFrame
Expand Down Expand Up @@ -54,7 +55,7 @@ def sql_gen_generate_adjusted_lambda(column_name, params, table_name="df_e"):
m = params.params["π"][f"gamma_{column_name}"]["prob_dist_match"][f"level_{max_level}"]["probability"]
u = params.params["π"][f"gamma_{column_name}"]["prob_dist_non_match"][f"level_{max_level}"]["probability"]

# ensure average adj calculation doesnt divide by zero
# ensure average adj calculation doesnt divide by zero (see issue 118)
if ( math.isclose((m+u), 0.0, rel_tol=1e-9, abs_tol=0.0)):
average_adjustment = 0.5
warnings.warn( f" Is most of column {column_name} or all of it comprised of NULL values??? There are levels where no comparisons are found.")
Expand Down
2 changes: 2 additions & 0 deletions tests/test_adj.py
Expand Up @@ -3,6 +3,7 @@
import pandas as pd
import pyspark.sql.functions as f
import pyspark
import warnings


@pytest.fixture(scope="module")
Expand Down Expand Up @@ -109,6 +110,7 @@ def test_freq_adj_divzero(spark, sparkdf):
# create column weird in a way that could trigger a div by zero on the average adj calculation before the fix
sparkdf = sparkdf.withColumn("weird",f.lit(None))


try:
linker = Splink(settings, spark, df=sparkdf)
notpassing = False
Expand Down

0 comments on commit baaa6d3

Please sign in to comment.