Sync with base

.github/workflows/lint-global.yml

@@ -15,4 +15,4 @@ jobs:
       - name: Lint Markdown and TOML
         uses: dprint/[email protected]
       - name: Spell Check with Typos
-        uses: crate-ci/typos@v1.20.10
+        uses: crate-ci/typos@v1.21.0

.github/workflows/release-python.yml

@@ -86,14 +86,14 @@ jobs:
       fail-fast: false
       matrix:
         package: [polars, polars-lts-cpu, polars-u64-idx]
-        os: [ubuntu-latest, macos-latest, windows-32gb-ram]
+        os: [ubuntu-latest, macos-13, windows-32gb-ram]
         architecture: [x86-64, aarch64]
         exclude:
           - os: windows-32gb-ram
             architecture: aarch64
 
     env:
-      SED_INPLACE: ${{ matrix.os == 'macos-latest' && '-i ''''' || '-i'}}
+      SED_INPLACE: ${{ matrix.os == 'macos-13' && '-i ''''' || '-i'}}
       CPU_CHECK_MODULE: py-polars/polars/_cpu_check.py
 
     steps:
@@ -128,7 +128,7 @@ jobs:
         if: matrix.architecture == 'x86-64'
         env:
           IS_LTS_CPU: ${{ matrix.package == 'polars-lts-cpu' }}
-          IS_MACOS: ${{ matrix.os == 'macos-latest' }}
+          IS_MACOS: ${{ matrix.os == 'macos-13' }}
         # IMPORTANT: All features enabled here should also be included in py-polars/polars/_cpu_check.py
         run: |
           if [[ "$IS_LTS_CPU" = true ]]; then
@@ -144,6 +144,7 @@ jobs:
         if: matrix.architecture == 'x86-64'
         env:
           FEATURES: ${{ steps.features.outputs.features }}
+          CFG: ${{ matrix.package == 'polars-lts-cpu' && '--cfg default_allocator' || '' }}
         run: echo "RUSTFLAGS=-C target-feature=${{ steps.features.outputs.features }} $CFG" >> $GITHUB_ENV
 
       - name: Set variables in CPU check module
@@ -159,7 +160,7 @@ jobs:
         if: matrix.architecture == 'aarch64'
         id: target
         run: |
-          TARGET=${{ matrix.os == 'macos-latest' && 'aarch64-apple-darwin' || 'aarch64-unknown-linux-gnu'}}
+          TARGET=${{ matrix.os == 'macos-13' && 'aarch64-apple-darwin' || 'aarch64-unknown-linux-gnu'}}
           echo "target=$TARGET" >> $GITHUB_OUTPUT
 
       - name: Set jemalloc for aarch64 Linux

.github/workflows/test-coverage.yml

@@ -34,7 +34,9 @@ jobs:
   coverage-rust:
     # Running under ubuntu doesn't seem to work:
     # https://github.com/pola-rs/polars/issues/14255
-    runs-on: macos-latest
+    # Pinned on macos-13 because latest does not work:
+    # https://github.com/pola-rs/polars/issues/15917
+    runs-on: macos-13
     steps:
       - uses: actions/checkout@v4
 
@@ -85,7 +87,7 @@ jobs:
   coverage-python:
     # Running under ubuntu doesn't seem to work:
     # https://github.com/pola-rs/polars/issues/14255
-    runs-on: macos-latest
+    runs-on: macos-13
     steps:
       - uses: actions/checkout@v4
 

.gitignore

@@ -8,7 +8,6 @@
 .yarn/
 coverage.lcov
 coverage.xml
-data/
 polars/vendor
 
 # OS
@@ -32,6 +31,12 @@ __pycache__/
 .cargo/
 target/
 
+# Data
+*.csv
+*.parquet
+*.feather
+*.tbl
+
 # Project
 /docs/data/
 /docs/images/

LICENSE

@@ -1,4 +1,5 @@
 Copyright (c) 2020 Ritchie Vink
+Some portions Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal

README.md

@@ -29,7 +29,7 @@
   -
   <a href="https://pola-rs.github.io/nodejs-polars/index.html">Node.js</a>
   -
-  <a href="https://rpolars.github.io/index.html">R</a>
+  <a href="https://pola-rs.github.io/r-polars/index.html">R</a>
   |
   <b>StackOverflow</b>:
   <a href="https://stackoverflow.com/questions/tagged/python-polars">Python</a>

_typos.toml

@@ -29,4 +29,4 @@ extend-glob = ["*.gz"]
 check-file = false
 
 [files]
-extend-exclude = ["_typos.toml"]
+extend-exclude = ["_typos.toml", "dists.dss"]

crates/polars-arrow/src/array/dictionary/mod.rs

@@ -25,7 +25,9 @@ use polars_error::{polars_bail, PolarsResult};
 use super::primitive::PrimitiveArray;
 use super::specification::check_indexes;
 use super::{new_empty_array, new_null_array, Array};
-use crate::array::dictionary::typed_iterator::{DictValue, DictionaryValuesIterTyped};
+use crate::array::dictionary::typed_iterator::{
+    DictValue, DictionaryIterTyped, DictionaryValuesIterTyped,
+};
 
 /// Trait denoting [`NativeType`]s that can be used as keys of a dictionary.
 /// # Safety
@@ -241,30 +243,22 @@ impl<K: DictionaryKey> DictionaryArray<K> {
     ///
     /// # Panics
     ///
-    /// Panics if the keys of this [`DictionaryArray`] have any null types.
-    /// If they do [`DictionaryArray::iter_typed`] should be called
+    /// Panics if the keys of this [`DictionaryArray`] has any nulls.
+    /// If they do [`DictionaryArray::iter_typed`] should be used.
     pub fn values_iter_typed<V: DictValue>(&self) -> PolarsResult<DictionaryValuesIterTyped<K, V>> {
         let keys = &self.keys;
         assert_eq!(keys.null_count(), 0);
         let values = self.values.as_ref();
         let values = V::downcast_values(values)?;
-        Ok(unsafe { DictionaryValuesIterTyped::new(keys, values) })
+        Ok(DictionaryValuesIterTyped::new(keys, values))
     }
 
     /// Returns an iterator over the optional values of  [`Option<V::IterValue>`].
-    ///
-    /// # Panics
-    ///
-    /// This function panics if the `values` array
-    pub fn iter_typed<V: DictValue>(
-        &self,
-    ) -> PolarsResult<ZipValidity<V::IterValue<'_>, DictionaryValuesIterTyped<K, V>, BitmapIter>>
-    {
+    pub fn iter_typed<V: DictValue>(&self) -> PolarsResult<DictionaryIterTyped<K, V>> {
         let keys = &self.keys;
         let values = self.values.as_ref();
         let values = V::downcast_values(values)?;
-        let values_iter = unsafe { DictionaryValuesIterTyped::new(keys, values) };
-        Ok(ZipValidity::new_with_validity(values_iter, self.validity()))
+        Ok(DictionaryIterTyped::new(keys, values))
     }
 
     /// Returns the [`ArrowDataType`] of this [`DictionaryArray`]

crates/polars-arrow/src/array/dictionary/typed_iterator.rs

@@ -1,7 +1,7 @@
 use polars_error::{polars_err, PolarsResult};
 
 use super::DictionaryKey;
-use crate::array::{Array, PrimitiveArray, Utf8Array, Utf8ViewArray};
+use crate::array::{Array, PrimitiveArray, StaticArray, Utf8Array, Utf8ViewArray};
 use crate::trusted_len::TrustedLen;
 use crate::types::Offset;
 
@@ -85,7 +85,8 @@ pub struct DictionaryValuesIterTyped<'a, K: DictionaryKey, V: DictValue> {
 }
 
 impl<'a, K: DictionaryKey, V: DictValue> DictionaryValuesIterTyped<'a, K, V> {
-    pub(super) unsafe fn new(keys: &'a PrimitiveArray<K>, values: &'a V) -> Self {
+    pub(super) fn new(keys: &'a PrimitiveArray<K>, values: &'a V) -> Self {
+        assert_eq!(keys.null_count(), 0);
         Self {
             keys,
             values,
@@ -137,3 +138,68 @@ impl<'a, K: DictionaryKey, V: DictValue> DoubleEndedIterator
         }
     }
 }
+
+pub struct DictionaryIterTyped<'a, K: DictionaryKey, V: DictValue> {
+    keys: &'a PrimitiveArray<K>,
+    values: &'a V,
+    index: usize,
+    end: usize,
+}
+
+impl<'a, K: DictionaryKey, V: DictValue> DictionaryIterTyped<'a, K, V> {
+    pub(super) fn new(keys: &'a PrimitiveArray<K>, values: &'a V) -> Self {
+        Self {
+            keys,
+            values,
+            index: 0,
+            end: keys.len(),
+        }
+    }
+}
+
+impl<'a, K: DictionaryKey, V: DictValue> Iterator for DictionaryIterTyped<'a, K, V> {
+    type Item = Option<V::IterValue<'a>>;
+
+    #[inline]
+    fn next(&mut self) -> Option<Self::Item> {
+        if self.index == self.end {
+            return None;
+        }
+        let old = self.index;
+        self.index += 1;
+        unsafe {
+            if let Some(key) = self.keys.get_unchecked(old) {
+                let idx = key.as_usize();
+                Some(Some(self.values.get_unchecked(idx)))
+            } else {
+                Some(None)
+            }
+        }
+    }
+
+    #[inline]
+    fn size_hint(&self) -> (usize, Option<usize>) {
+        (self.end - self.index, Some(self.end - self.index))
+    }
+}
+
+unsafe impl<'a, K: DictionaryKey, V: DictValue> TrustedLen for DictionaryIterTyped<'a, K, V> {}
+
+impl<'a, K: DictionaryKey, V: DictValue> DoubleEndedIterator for DictionaryIterTyped<'a, K, V> {
+    #[inline]
+    fn next_back(&mut self) -> Option<Self::Item> {
+        if self.index == self.end {
+            None
+        } else {
+            self.end -= 1;
+            unsafe {
+                if let Some(key) = self.keys.get_unchecked(self.end) {
+                    let idx = key.as_usize();
+                    Some(Some(self.values.get_unchecked(idx)))
+                } else {
+                    Some(None)
+                }
+            }
+        }
+    }
+}

crates/polars-arrow/src/array/mod.rs

@@ -266,7 +266,7 @@ impl std::fmt::Debug for dyn Array + '_ {
         match self.data_type().to_physical_type() {
             Null => fmt_dyn!(self, NullArray, f),
             Boolean => fmt_dyn!(self, BooleanArray, f),
-            Primitive(primitive) => with_match_primitive_type!(primitive, |$T| {
+            Primitive(primitive) => with_match_primitive_type_full!(primitive, |$T| {
                 fmt_dyn!(self, PrimitiveArray<$T>, f)
             }),
             BinaryView => fmt_dyn!(self, BinaryViewArray, f),

crates/polars-arrow/src/legacy/kernels/ewm/mod.rs

@@ -57,7 +57,7 @@ impl EWMOptions {
     }
     pub fn and_half_life(mut self, half_life: f64) -> Self {
         assert!(half_life > 0.0);
-        self.alpha = 1.0 - ((-2.0f64).ln() / half_life).exp();
+        self.alpha = 1.0 - (-(2.0f64.ln()) / half_life).exp();
         self
     }
     pub fn and_com(mut self, com: f64) -> Self {