feat: add support for Ibis 9.x (#1034)

- Adds support for Ibis 9.x and drops all previous versions. - Adds `test_parity.py` checking execution parity between duckdb (with ibis), duckdb (with substrait) and acero. * feat: support ibis >=9.0 * fix(format): run black on translate.py * fix: revert to TypeAlias conditional import for py39 * fix: update flake.lock * fix: duckdb to_pyarrow doesn't work with arrow nightly * fix: update poetry.lock * chore(deps): remove sqlalchemy and duckdb-engine We no longer require sqlalchemy upstream so we can drop these dependencies that were used for downstream testing here. * chore(ci): bump conventional commits to 8.0.0 * chore(nix): bump flake.lock * chore(parity tests): xfail nix+linux tests that need extensions Linux+Nix have sandboxing that prevents duckdb extensions from downloading. * fix: ignore temp directory cleanup errors on windows * fix: disable duckdb_substrait, close duckdb connection * fix: remove nix sandbox markers * fix: apply suggestions from code review Co-authored-by: Gil Forsyth <[email protected]> * fix: remove Sum translation rule * fix: add correct countDistinct handling --------- Co-authored-by: Gil Forsyth <[email protected]> Co-authored-by: Gil Forsyth <[email protected]>
ibis-project · Jun 28, 2024 · 0c4aee8 · 0c4aee8
1 parent 190496e
commit 0c4aee8
Show file tree

Hide file tree

Showing 37 changed files with 10,091 additions and 5,505 deletions.
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -92,14 +92,11 @@ jobs:
           - "3.9"
           - "3.11"
         ibis-version:
-          - "4.1"
-          - "7.0"
-          - "7.1"
-          - "7.2"
+          - "9.0"
         include:
          - os: windows-latest
            python-version: "3.11"
-           ibis-version: "7.0"
+           ibis-version: "9.0"
 
     steps:
       - uses: actions/checkout@v4
@@ -132,8 +129,7 @@ jobs:
           - "3.9"
           - "3.11"
         ibis-version:
-          - "4.1"
-          - "7.0"
+          - "9.0"
 
     steps:
       - uses: actions/checkout@v4

diff --git a/ci/release/dry_run.sh b/ci/release/dry_run.sh
@@ -42,7 +42,7 @@ nix develop '.#release' -c npx --yes \
   -p "@semantic-release/exec" \
   -p "@semantic-release/git" \
   -p "@google/[email protected]" \
-  -p "conventional-changelog-conventionalcommits@6.1.0" \
+  -p "conventional-changelog-conventionalcommits@8.0.0" \
   semantic-release \
   --ci \
   --dry-run \

diff --git a/ci/release/run.sh b/ci/release/run.sh
@@ -11,5 +11,5 @@ nix develop '.#release' -c npx --yes \
   -p "@semantic-release/exec" \
   -p "@semantic-release/git" \
   -p "@google/[email protected]" \
-  -p "conventional-changelog-conventionalcommits@6.1.0" \
+  -p "conventional-changelog-conventionalcommits@8.0.0" \
   semantic-release --ci
diff --git a/ibis_substrait/compiler/core.py b/ibis_substrait/compiler/core.py
@@ -116,7 +116,7 @@ def get_signature(self, op: ops.Node) -> tuple[str, ...]:
         if function_extension is None:
             sigkey = tuple(
                 [
-                    IBIS_SUBSTRAIT_TYPE_MAPPING[arg.output_dtype.name]  # type: ignore
+                    IBIS_SUBSTRAIT_TYPE_MAPPING[arg.dtype.name]  # type: ignore
                     for arg in op.args
                     if arg is not None and isinstance(arg, ops.Node)
                 ]

diff --git a/ibis_substrait/compiler/mapping.py b/ibis_substrait/compiler/mapping.py
@@ -30,7 +30,7 @@
     "Cos": "cos",
     "Count": "count",
     "CountStar": "count",
-    "CountDistinct": "countdistinct",
+    "CountDistinct": "count",
     "Divide": "divide",
     "EndsWith": "ends_with",
     "Equals": "equal",