10 changes: 10 additions & 0 deletions ibis/backends/dask/execution/strings.py
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@
execute_series_translate_series_series,
execute_string_capitalize,
execute_string_contains,
execute_string_find,
execute_string_length_series,
execute_string_like_series_string,
execute_string_lower,
Expand Down Expand Up @@ -93,6 +94,15 @@
(dd.Series, type(None)) + integer_types,
(dd.Series, type(None)) + integer_types,
),
execute_string_find,
)
],
ops.StringContains: [
(
(
dd.Series,
(dd.Series, str),
),
execute_string_contains,
)
],
Expand Down
3 changes: 2 additions & 1 deletion ibis/backends/duckdb/compiler.py
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,8 @@ class DuckDBSQLExprTranslator(AlchemyExprTranslator):
@rewrites(ops.All)
@rewrites(ops.NotAny)
@rewrites(ops.NotAll)
def _any_all_no_op(expr):
@rewrites(ops.StringContains)
def _no_op(expr):
return expr


Expand Down
11 changes: 6 additions & 5 deletions ibis/backends/duckdb/registry.py
Original file line number Diff line number Diff line change
Expand Up @@ -148,7 +148,7 @@ def _regex_extract(t, expr):
operation_registry.update(
{
ops.ArrayColumn: _array_column,
ops.ArrayConcat: fixed_arity('array_concat', 2),
ops.ArrayConcat: fixed_arity(sa.func.array_concat, 2),
ops.DayOfWeekName: unary(sa.func.dayname),
ops.Literal: _literal,
ops.Log2: unary(sa.func.log2),
Expand All @@ -159,12 +159,13 @@ def _regex_extract(t, expr):
ops.Round: _round,
ops.StructField: _struct_field,
ops.TableColumn: _table_column,
ops.TimestampDiff: fixed_arity('age', 2),
ops.TimestampDiff: fixed_arity(sa.func.age, 2),
ops.TimestampFromUNIX: _timestamp_from_unix,
ops.Translate: fixed_arity('replace', 3),
ops.TimestampNow: fixed_arity('now', 0),
ops.Translate: fixed_arity(sa.func.replace, 3),
ops.TimestampNow: fixed_arity(sa.func.now, 0),
ops.RegexExtract: _regex_extract,
ops.RegexReplace: fixed_arity("regexp_replace", 3),
ops.RegexReplace: fixed_arity(sa.func.regexp_replace, 3),
ops.StringContains: fixed_arity(sa.func.contains, 2),
}
)

Expand Down
7 changes: 6 additions & 1 deletion ibis/backends/pandas/execution/strings.py
Original file line number Diff line number Diff line change
Expand Up @@ -118,14 +118,19 @@ def execute_string_repeat(op, data, times, **kwargs):
return data.str.repeat(times)


@execute_node.register(ops.StringContains, pd.Series, (pd.Series, str))
def execute_string_contains(_, data, needle, **kwargs):
return data.str.contains(needle)


@execute_node.register(
ops.StringFind,
pd.Series,
(pd.Series, str),
(pd.Series, type(None)) + integer_types,
(pd.Series, type(None)) + integer_types,
)
def execute_string_contains(op, data, needle, start, end, **kwargs):
def execute_string_find(op, data, needle, start, end, **kwargs):
return data.str.find(needle, start, end)


Expand Down
8 changes: 8 additions & 0 deletions ibis/backends/pyspark/compiler.py
Original file line number Diff line number Diff line change
Expand Up @@ -1906,3 +1906,11 @@ def compile_sql_view(t, expr, scope, timecontext, **kwargs):
result = backend._session.sql(op.query)
result.createOrReplaceTempView(op.name)
return result


@compiles(ops.StringContains)
def compile_string_contains(t, expr, scope, timecontext, **kwargs):
op = expr.op()
haystack = t.translate(op.haystack, scope, timecontext, **kwargs)
needle = t.translate(op.needle, scope, timecontext, **kwargs)
return haystack.contains(needle)
9 changes: 9 additions & 0 deletions ibis/expr/operations/strings.py
Original file line number Diff line number Diff line change
Expand Up @@ -248,3 +248,12 @@ class StringLength(UnaryOp):
@public
class StringAscii(UnaryOp):
output_dtype = dt.int32


@public
class StringContains(ValueOp):
haystack = rlz.string
needle = rlz.string

output_shape = rlz.shape_like("args")
output_dtype = dt.bool
4 changes: 3 additions & 1 deletion ibis/expr/types/strings.py
Original file line number Diff line number Diff line change
Expand Up @@ -180,7 +180,9 @@ def contains(self, substr: str | StringValue) -> ir.BooleanValue:
BooleanValue
Boolean indicating the presence of `substr` in the expression
"""
return self.find(substr) >= 0
import ibis.expr.operations as ops

return ops.StringContains(self, substr).to_expr()

def hashbytes(
self,
Expand Down
3 changes: 1 addition & 2 deletions ibis/tests/expr/test_string.py
Original file line number Diff line number Diff line change
Expand Up @@ -89,8 +89,7 @@ def test_join(table):

def test_contains(table):
expr = table.g.contains('foo')
expected = table.g.find('foo') >= 0
assert_equal(expr, expected)
assert isinstance(expr.op(), ops.StringContains)

with pytest.raises(TypeError):
'foo' in table.g
Expand Down
2 changes: 1 addition & 1 deletion ibis/tests/sql/test_compiler.py
Original file line number Diff line number Diff line change
Expand Up @@ -131,7 +131,7 @@ def test_pushdown_with_or():
expected = """\
SELECT *
FROM functional_alltypes
WHERE ((`double_col` > 3.14) AND (locate('foo', `string_col`) - 1 >= 0)) AND
WHERE ((`double_col` > 3.14) AND locate('foo', `string_col`) - 1 >= 0) AND
(((`int_col` - 1) = 0) OR (`float_col` <= 1.34))"""
assert result == expected

Expand Down
1 change: 1 addition & 0 deletions mkdocs.yml
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,7 @@ plugins:
- search
- mike
- autorefs
- git-revision-date-localized
- exclude:
glob:
- backends/template.md
Expand Down
6 changes: 3 additions & 3 deletions nix/sources.json

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

170 changes: 170 additions & 0 deletions patches/datafusion-macos.patch
Original file line number Diff line number Diff line change
@@ -0,0 +1,170 @@
diff --git a/Cargo.lock b/Cargo.lock
index d73083f..489f301 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -277,9 +277,9 @@ dependencies = [

[[package]]
name = "datafusion"
-version = "7.0.0"
+version = "7.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "30cf8e6735817bb021748d72cecc33e468d8775bf749470c52aa7f55ee5cdf9e"
+checksum = "79a0ea0a500cbfb6b683ad8cc6f403faa7c897432cc8ad0da40c09a9a705255f"
dependencies = [
"ahash",
"arrow",
@@ -384,9 +384,9 @@ dependencies = [

[[package]]
name = "flate2"
-version = "1.0.22"
+version = "1.0.23"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1e6988e897c1c9c485f43b47a529cef42fde0547f9d8d41a7062518f1d8fc53f"
+checksum = "b39522e96686d38f4bc984b9198e3a0613264abaebaff2c5c918bfa6b6da09af"
dependencies = [
"cfg-if",
"crc32fast",
@@ -701,9 +701,9 @@ dependencies = [

[[package]]
name = "libc"
-version = "0.2.121"
+version = "0.2.124"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "efaa7b300f3b5fe8eb6bf21ce3895e1751d9665086af2d64b42f19701015ff4f"
+checksum = "21a41fed9d98f27ab1c6d161da622a4fa35e8a54a8adc24bbf3ddd0ef70b0e50"

[[package]]
name = "libmimalloc-sys"
@@ -779,12 +779,11 @@ dependencies = [

[[package]]
name = "miniz_oxide"
-version = "0.4.4"
+version = "0.5.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a92518e98c078586bc6c934028adcca4c92a53d6a958196de835170a01d84e4b"
+checksum = "d2b29bd4bc3f33391105ebee3589c19197c4271e3e5a9ec9bfe8127eeff8f082"
dependencies = [
"adler",
- "autocfg",
]

[[package]]
@@ -1047,18 +1046,18 @@ checksum = "dbf0c48bc1d91375ae5c3cd81e3722dff1abcf81a30960240640d223f59fe0e5"

[[package]]
name = "proc-macro2"
-version = "1.0.36"
+version = "1.0.37"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c7342d5883fbccae1cc37a2353b09c87c9b0f3afd73f5fb9bba687a1f733b029"
+checksum = "ec757218438d5fda206afc041538b2f6d889286160d649a86a24d37e1235afd1"
dependencies = [
"unicode-xid",
]

[[package]]
name = "pyo3"
-version = "0.15.1"
+version = "0.15.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7cf01dbf1c05af0a14c7779ed6f3aa9deac9c3419606ac9de537a2d649005720"
+checksum = "d41d50a7271e08c7c8a54cd24af5d62f73ee3a6f6a314215281ebdec421d5752"
dependencies = [
"cfg-if",
"indoc",
@@ -1072,18 +1071,18 @@ dependencies = [

[[package]]
name = "pyo3-build-config"
-version = "0.15.1"
+version = "0.15.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "dbf9e4d128bfbddc898ad3409900080d8d5095c379632fbbfbb9c8cfb1fb852b"
+checksum = "779239fc40b8e18bc8416d3a37d280ca9b9fb04bda54b98037bb6748595c2410"
dependencies = [
"once_cell",
]

[[package]]
name = "pyo3-macros"
-version = "0.15.1"
+version = "0.15.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "67701eb32b1f9a9722b4bc54b548ff9d7ebfded011c12daece7b9063be1fd755"
+checksum = "00b247e8c664be87998d8628e86f282c25066165f1f8dda66100c48202fdb93a"
dependencies = [
"pyo3-macros-backend",
"quote",
@@ -1092,9 +1091,9 @@ dependencies = [

[[package]]
name = "pyo3-macros-backend"
-version = "0.15.1"
+version = "0.15.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f44f09e825ee49a105f2c7b23ebee50886a9aee0746f4dd5a704138a64b0218a"
+checksum = "5a8c2812c412e00e641d99eeb79dd478317d981d938aa60325dfa7157b607095"
dependencies = [
"proc-macro2",
"pyo3-build-config",
@@ -1104,9 +1103,9 @@ dependencies = [

[[package]]
name = "quote"
-version = "1.0.17"
+version = "1.0.18"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "632d02bff7f874a36f33ea8bb416cd484b90cc66c1194b1a1110d067a7013f58"
+checksum = "a1feb54ed693b93a84e14094943b84b7c4eae204c512b7ccb95ab0c66d278ad1"
dependencies = [
"proc-macro2",
]
@@ -1341,9 +1340,9 @@ checksum = "6bdef32e8150c2a081110b42772ffe7d7c9032b606bc226c8260fd97e0976601"

[[package]]
name = "syn"
-version = "1.0.90"
+version = "1.0.91"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "704df27628939572cd88d33f171cd6f896f4eaca85252c6e0a72d8d8287ee86f"
+checksum = "b683b2b825c8eef438b77c36a06dc262294da3d5a5813fac20da149241dcd44d"
dependencies = [
"proc-macro2",
"quote",
diff --git a/Cargo.toml b/Cargo.toml
index ba2f337..40cba00 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -35,7 +35,7 @@ datafusion = { version = "^7.0.0", features = ["pyarrow"] }
datafusion-expr = { version = "^7.0.0" }
datafusion-common = { version = "^7.0.0", features = ["pyarrow"] }
uuid = { version = "0.8", features = ["v4"] }
-mimalloc = { version = "*", default-features = false }
+mimalloc = { version = "*", optional = true, default-features = false }

[lib]
name = "datafusion_python"
diff --git a/src/lib.rs b/src/lib.rs
index 977d9e8..ca1cd17 100644
--- a/src/lib.rs
+++ b/src/lib.rs
@@ -15,6 +15,7 @@
// specific language governing permissions and limitations
// under the License.

+#[cfg(feature = "mimalloc")]
use mimalloc::MiMalloc;
use pyo3::prelude::*;

@@ -28,6 +29,7 @@ mod udaf;
mod udf;
pub mod utils;

+#[cfg(feature = "mimalloc")]
#[global_allocator]
static GLOBAL: MiMalloc = MiMalloc;

13 changes: 13 additions & 0 deletions patches/watchdog-force-kqueue.patch
Original file line number Diff line number Diff line change
@@ -0,0 +1,13 @@
diff --git a/setup.py b/setup.py
index 072dfc8..cb9aa7a 100644
--- a/setup.py
+++ b/setup.py
@@ -39,7 +39,7 @@ _apple_devices = ('appletv', 'iphone', 'ipod', 'ipad', 'watch')
is_macos = sys.platform == 'darwin' and not machine().lower().startswith(_apple_devices)

ext_modules = []
-if is_macos or os.getenv('FORCE_MACOS_MACHINE', '0') == '1':
+if False:
ext_modules = [
Extension(
name='_watchdog_fsevents',
27 changes: 23 additions & 4 deletions poetry-overrides.nix
Original file line number Diff line number Diff line change
Expand Up @@ -16,17 +16,30 @@ self: super:
});
});

datafusion = super.datafusion.overridePythonAttrs (attrs: {
datafusion = super.datafusion.overridePythonAttrs (attrs: rec {
inherit (attrs) version;
src = pkgs.fetchFromGitHub {
owner = "datafusion-contrib";
repo = "datafusion-python";
rev = attrs.version;
sha256 = "sha256-IWqlY4Cfil3cyQqXm+X9ViRYLzmNaiM3+i/7EyV5CK4=";
};

patches = (attrs.patches or [ ])
++ lib.optionals stdenv.isDarwin [ ./patches/datafusion-macos.patch ];

nativeBuildInputs = (attrs.nativeBuildInputs or [ ])
++ (with pkgs.rustPlatform; [ cargoSetupHook maturinBuildHook ]);

buildInputs = (attrs.buildInputs or [ ])
++ lib.optionals stdenv.isDarwin [ pkgs.libiconv ];

cargoDeps = pkgs.rustPlatform.fetchCargoTarball {
inherit (attrs) src;
sourceRoot = "${attrs.pname}-${attrs.version}";
sha256 = "sha256-SHVJWbQROQVQ9qZDTSvHz/O9irCyEPgcmDowerMPYeI=";
inherit src patches;
sha256 =
if stdenv.isDarwin
then "sha256-qDXfSisgQ4qr8Sky0aNns8LldiHYs/N1cNatNlwEE18="
else "sha256-bDuCbQYNai/mNrS2BqoW4qe7eLZcBhb7GhsFKn08G/U=";
};
});

Expand Down Expand Up @@ -54,4 +67,10 @@ self: super:
atpublic = super.atpublic.overridePythonAttrs (attrs: {
nativeBuildInputs = (attrs.nativeBuildInputs or [ ]) ++ [ self.pdm-pep517 ];
});

watchdog = super.watchdog.overrideAttrs (attrs: lib.optionalAttrs
(stdenv.isDarwin && lib.versionAtLeast attrs.version "2")
{
patches = (attrs.patches or [ ]) ++ [ ./patches/watchdog-force-kqueue.patch ];
});
}
126 changes: 100 additions & 26 deletions poetry.lock
5 changes: 3 additions & 2 deletions pyproject.toml
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
[tool.poetry]
name = "ibis-framework"
version = "3.0.0"
version = "3.0.1"
packages = [{ include = "ibis" }]
homepage = "https://ibis-project.org"
repository = "https://github.com/ibis-project/ibis"
Expand Down Expand Up @@ -83,13 +83,14 @@ mike = ">=1.1.2,<2"
mkdocs = ">=1.2.3,<2"
mkdocs-exclude = ">=1.0.2,<2"
mkdocs-gen-files = ">=0.3.4,<0.4.0"
mkdocs-git-revision-date-localized-plugin = ">=1.0.1,<2"
mkdocs-jupyter = ">=0.20.0,<1"
mkdocs-literate-nav = ">=0.4.1,<1"
mkdocs-macros-plugin = ">=0.6.3,<1"
mkdocs-material = ">=8.2.1,<9"
mkdocs-table-reader-plugin = ">=1.0.0,<2"
mkdocstrings = ">=0.17.0,<0.18.0"
mypy = "0.942"
mypy = "0.950"
platformdirs = ">=2,<2.5.2"
pyarrow = ">=1,<8"
pydocstyle = ">=6.1.1,<7"
Expand Down
2 changes: 1 addition & 1 deletion setup.py

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

5 changes: 3 additions & 2 deletions shell.nix
Original file line number Diff line number Diff line change
Expand Up @@ -62,15 +62,16 @@ let
};
mic = pkgs.writeShellApplication {
name = "mic";
runtimeInputs = [ pythonEnv ];
runtimeInputs = [ pythonEnv pkgs.coreutils ];
text = ''
# The immediate reason this is necessary is to allow the subprocess
# invocations of `mkdocs` by `mike` to see Python dependencies.
#
# This shouldn't be necessary, but I think the nix wrappers may be
# indavertently preventing this.
export PYTHONPATH
export PYTHONPATH TEMPDIR
PYTHONPATH="$(python -c 'import os, sys; print(os.pathsep.join(sys.path))')"
TEMPDIR="$(mktemp -d)"
mike "$@"
'';
Expand Down