Skip to content

feat!: (WIP) Implement TryFrom for arrow-rs types #293

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
dhruv9vats wants to merge 1 commit into
base: main
Choose a base branch
from
Draft

feat!: (WIP) Implement TryFrom for arrow-rs types #293

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions examples/parquet.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -78,7 +78,7 @@ fn main() {
let output = narrow_array.clone().into_iter().collect::<Vec<_>>();
assert_eq!(input.as_slice(), output);

let record_batch = RecordBatch::from(narrow_array);
let record_batch: RecordBatch = narrow_array.into();
println!("From narrow StructArray to Arrow RecordBatch");
pretty::print_batches(&[record_batch.clone()]).unwrap();

Expand All @@ -94,7 +94,7 @@ fn main() {
assert_eq!(record_batch, read.clone());

let round_trip: StructArray<Foo, NonNullable, ScalarBuffer> = read.into();
let arrow_struct_array_round_trip = arrow_array::StructArray::from(round_trip);
let arrow_struct_array_round_trip: arrow_array::StructArray = round_trip.into();
let record_batch_round_trip = arrow_array::RecordBatch::from(arrow_struct_array_round_trip);
println!(
"From Arrow RecordBatch (via Parquet) to narrow StructArray and back to Arrow RecordBatch"
Expand Down
12 changes: 5 additions & 7 deletions narrow-derive/src/enum.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1087,25 +1087,23 @@ impl<'a> Enum<'a> {
let (_, self_ty_generics, _) = self.generics.split_for_impl();
generics.make_where_clause().predicates.extend(
self.variant_indices().map::<WherePredicate, _>(|idx| {
parse_quote!(::std::sync::Arc<dyn ::arrow_array::Array>: From<
<<#self_ident #self_ty_generics as #narrow::array::union::EnumVariant<#idx>>::Data as #narrow::array::ArrayType<<#self_ident #self_ty_generics as #narrow::array::union::EnumVariant<#idx>>::Data>>::Array<
parse_quote!(<<#self_ident #self_ty_generics as #narrow::array::union::EnumVariant<#idx>>::Data as #narrow::array::ArrayType<<#self_ident #self_ty_generics as #narrow::array::union::EnumVariant<#idx>>::Data>>::Array<
Buffer,
OffsetItem,
UnionLayout,
>,
>)
>: Into<::std::sync::Arc<dyn ::arrow_array::Array>>)
}),
);

let (impl_generics, _, where_clause) = generics.split_for_impl();
let (_, ty_generics, _) = generics.split_for_impl();
let idx = self.variant_indices();
let tokens = quote! {
impl #impl_generics ::std::convert::From<#ident #ty_generics> for ::std::vec::Vec<::std::sync::Arc<dyn ::arrow_array::Array>> #where_clause {
fn from(value: #ident #ty_generics) -> Self {
impl #impl_generics ::std::convert::Into<::std::vec::Vec<::std::sync::Arc<dyn ::arrow_array::Array>>> for #ident #ty_generics #where_clause {
fn into(self) -> ::std::vec::Vec<::std::sync::Arc<dyn ::arrow_array::Array>> {
vec![
#(
value.#idx.into(),
self.#idx.into(),
)*
]
}
Expand Down
11 changes: 6 additions & 5 deletions narrow-derive/src/struct.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -332,7 +332,7 @@ impl Struct<'_> {
let field_ident = self.field_idents();
quote!(
#(
value.#field_ident.into(),
self.#field_ident.into(),
)*
)
}
Expand All @@ -344,19 +344,20 @@ impl Struct<'_> {
.map(|(idx, _)| Index::from(idx));
quote!(
#(
value.#field_idx.into(),
self.#field_idx.into(),
)*
)
}
Fields::Unit => {
quote!(value.0.into())
quote!(self.0.into())
}
};

let ident = self.array_struct_ident();
let tokens = quote! {
impl #impl_generics ::std::convert::From<#ident #ty_generics> for ::std::vec::Vec<::std::sync::Arc<dyn ::arrow_array::Array>> #where_clause {
fn from(value: #ident #ty_generics) -> Self {
#[allow(clippy::from_over_into)]
impl #impl_generics std::convert::Into<::std::vec::Vec<::std::sync::Arc<dyn ::arrow_array::Array>>> for #ident #ty_generics #where_clause {
fn into(self) -> ::std::vec::Vec<::std::sync::Arc<dyn ::arrow_array::Array>> {
vec![
#field_arrays
]
Expand Down
28 changes: 14 additions & 14 deletions src/arrow/array/boolean.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,32 +23,32 @@ impl<Nullable: Nullability, Buffer: BufferType> crate::arrow::Array
}
}

impl<Buffer: BufferType> From<BooleanArray<NonNullable, Buffer>> for arrow_array::BooleanArray
impl<Buffer: BufferType> Into<arrow_array::BooleanArray> for BooleanArray<NonNullable, Buffer>
where
arrow_buffer::BooleanBuffer: From<Bitmap<Buffer>>,
Bitmap<Buffer>: Into<arrow_buffer::BooleanBuffer>,
{
fn from(value: BooleanArray<NonNullable, Buffer>) -> Self {
arrow_array::BooleanArray::new(value.0.into(), None)
fn into(self) -> arrow_array::BooleanArray {
arrow_array::BooleanArray::new(self.0.into(), None)
}
}

impl<Buffer: BufferType> From<BooleanArray<Nullable, Buffer>> for arrow_array::BooleanArray
impl<Buffer: BufferType> Into<arrow_array::BooleanArray> for BooleanArray<Nullable, Buffer>
where
arrow_buffer::BooleanBuffer: From<Bitmap<Buffer>>,
arrow_buffer::NullBuffer: From<Bitmap<Buffer>>,
Bitmap<Buffer>: Into<arrow_buffer::BooleanBuffer>,
Bitmap<Buffer>: Into<arrow_buffer::NullBuffer>,
{
fn from(value: BooleanArray<Nullable, Buffer>) -> Self {
arrow_array::BooleanArray::new(value.0.data.into(), Some(value.0.validity.into()))
fn into(self) -> arrow_array::BooleanArray {
arrow_array::BooleanArray::new(self.0.data.into(), Some(self.0.validity.into()))
}
}

impl<Nullable: Nullability, Buffer: BufferType> From<BooleanArray<Nullable, Buffer>>
for Arc<dyn arrow_array::Array>
impl<Nullable: Nullability, Buffer: BufferType> Into<Arc<dyn arrow_array::Array>>
for BooleanArray<Nullable, Buffer>
where
arrow_array::BooleanArray: From<BooleanArray<Nullable, Buffer>>,
BooleanArray<Nullable, Buffer>: Into<arrow_array::BooleanArray>,
{
fn from(value: BooleanArray<Nullable, Buffer>) -> Self {
Arc::new(arrow_array::BooleanArray::from(value))
fn into(self) -> Arc<dyn arrow_array::Array> {
Arc::new(self.into())
}
}

Expand Down
41 changes: 22 additions & 19 deletions src/arrow/array/fixed_size_binary.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -44,12 +44,13 @@ where
}
}

impl<const N: usize, Buffer: BufferType> From<FixedSizeBinaryArray<N, NonNullable, Buffer>>
for arrow_array::FixedSizeBinaryArray

impl<const N: usize, Buffer: BufferType> Into<arrow_array::FixedSizeBinaryArray>
for FixedSizeBinaryArray<N, NonNullable, Buffer>
where
arrow_buffer::Buffer: From<FixedSizePrimitiveArray<u8, NonNullable, Buffer>>,
FixedSizePrimitiveArray<u8, NonNullable, Buffer>: Into<arrow_buffer::Buffer>,
{
fn from(value: FixedSizeBinaryArray<N, NonNullable, Buffer>) -> Self {
fn into(self) -> arrow_array::FixedSizeBinaryArray {
// todo(mbrobbel): const_assert
assert!(N <= 0x7FFF_FFFF); // i32::MAX
#[allow(
Expand All @@ -59,29 +60,31 @@ where
)]
arrow_array::FixedSizeBinaryArray::new(
i32::try_from(N).expect("overflow"),
value.0 .0.into(),
self.0 .0.into(),
None,
)
}
}

impl<Nullable: Nullability, const N: usize, Buffer: BufferType>
From<FixedSizeBinaryArray<N, Nullable, Buffer>> for Arc<dyn arrow_array::Array>

impl<Nullable: Nullability, const N: usize, Buffer: BufferType> Into<Arc<dyn arrow_array::Array>>
for FixedSizeBinaryArray<N, Nullable, Buffer>
where
arrow_array::FixedSizeBinaryArray: From<FixedSizeBinaryArray<N, Nullable, Buffer>>,
FixedSizeBinaryArray<N, Nullable, Buffer>: Into<arrow_array::FixedSizeBinaryArray>,
{
fn from(value: FixedSizeBinaryArray<N, Nullable, Buffer>) -> Self {
Arc::new(arrow_array::FixedSizeBinaryArray::from(value))
fn into(self) -> Arc<dyn arrow_array::Array> {
Arc::new(self.into())
}
}

impl<const N: usize, Buffer: BufferType> From<FixedSizeBinaryArray<N, Nullable, Buffer>>
for arrow_array::FixedSizeBinaryArray

impl<const N: usize, Buffer: BufferType> Into<arrow_array::FixedSizeBinaryArray>
for FixedSizeBinaryArray<N, Nullable, Buffer>
where
arrow_buffer::Buffer: From<FixedSizePrimitiveArray<u8, NonNullable, Buffer>>,
FixedSizePrimitiveArray<u8, NonNullable, Buffer>: Into<arrow_buffer::Buffer>,
Bitmap<Buffer>: Into<NullBuffer>,
{
fn from(value: FixedSizeBinaryArray<N, Nullable, Buffer>) -> Self {
fn into(self) -> arrow_array::FixedSizeBinaryArray {
// todo(mbrobbel): const_assert
assert!(N <= 0x7FFF_FFFF); // i32::MAX
#[allow(
Expand All @@ -91,8 +94,8 @@ where
)]
arrow_array::FixedSizeBinaryArray::new(
i32::try_from(N).expect("overflow"),
value.0 .0.data.into(),
Some(value.0 .0.validity.into()),
self.0 .0.data.into(),
Some(self.0 .0.validity.into()),
)
}
}
Expand Down Expand Up @@ -148,7 +151,7 @@ mod tests {
fn from() {
let fixed_size_binary_array = INPUT.into_iter().collect::<FixedSizeBinaryArray<2>>();
assert_eq!(
arrow_array::FixedSizeBinaryArray::from(fixed_size_binary_array)
Into::<arrow_array::FixedSizeBinaryArray>::into(fixed_size_binary_array)
.iter()
.flatten()
.flatten()
Expand All @@ -161,7 +164,7 @@ mod tests {
.into_iter()
.collect::<FixedSizeBinaryArray<2, Nullable>>();
assert_eq!(
arrow_array::FixedSizeBinaryArray::from(fixed_size_binary_array_nullable)
Into::<arrow_array::FixedSizeBinaryArray>::into(fixed_size_binary_array_nullable)
.iter()
.flatten()
.flatten()
Expand Down Expand Up @@ -206,7 +209,7 @@ mod tests {
.into_iter()
.collect::<FixedSizeBinaryArray<2, Nullable>>();
let fixed_size_binary_array_nullable =
arrow_array::FixedSizeBinaryArray::from(fixed_size_binary_array_nullable_input);
Into::<arrow_array::FixedSizeBinaryArray>::into(fixed_size_binary_array_nullable_input);
assert_eq!(
FixedSizeBinaryArray::<2, Nullable>::from(fixed_size_binary_array_nullable)
.into_iter()
Expand Down
48 changes: 26 additions & 22 deletions src/arrow/array/fixed_size_list.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -48,43 +48,46 @@ where
}
}

impl<const N: usize, T: Array, Buffer: BufferType>
From<FixedSizeListArray<N, T, NonNullable, Buffer>> for Arc<dyn arrow_array::Array>

impl<const N: usize, T: Array, Buffer: BufferType> Into<Arc<dyn arrow_array::Array>>
for FixedSizeListArray<N, T, NonNullable, Buffer>
where
T: crate::arrow::Array + Into<Arc<dyn arrow_array::Array>>,
{
fn from(value: FixedSizeListArray<N, T, NonNullable, Buffer>) -> Self {
fn into(self) -> Arc<dyn arrow_array::Array> {
Arc::new(arrow_array::FixedSizeListArray::new(
Arc::new(T::as_field("item")),
i32::try_from(N).expect("overflow"),
value.0.into(),
self.0.into(),
None,
))
}
}

impl<const N: usize, T: Array, Buffer: BufferType> From<FixedSizeListArray<N, T, Nullable, Buffer>>
for Arc<dyn arrow_array::Array>

impl<const N: usize, T: Array, Buffer: BufferType> Into<Arc<dyn arrow_array::Array>>
for FixedSizeListArray<N, T, Nullable, Buffer>
where
T: crate::arrow::Array + Into<Arc<dyn arrow_array::Array>>,
Bitmap<Buffer>: Into<NullBuffer>,
{
fn from(value: FixedSizeListArray<N, T, Nullable, Buffer>) -> Self {
fn into(self) -> Arc<dyn arrow_array::Array> {
Arc::new(arrow_array::FixedSizeListArray::new(
Arc::new(T::as_field("item")),
i32::try_from(N).expect("overflow"),
value.0.data.into(),
Some(value.0.validity.into()),
self.0.data.into(),
Some(self.0.validity.into()),
))
}
}


impl<const N: usize, T: crate::arrow::Array, Buffer: BufferType>
From<FixedSizeListArray<N, T, NonNullable, Buffer>> for arrow_array::FixedSizeListArray
Into<arrow_array::FixedSizeListArray> for FixedSizeListArray<N, T, NonNullable, Buffer>
where
<T as crate::arrow::Array>::Array: From<T> + 'static,
T: Into<<T as crate::arrow::Array>::Array> + 'static,
{
fn from(value: FixedSizeListArray<N, T, NonNullable, Buffer>) -> Self {
fn into(self) -> arrow_array::FixedSizeListArray {
// todo(mbrobbel): const_assert
assert!(N <= 0x7FFF_FFFF); // i32::MAX
#[allow(
Expand All @@ -95,19 +98,20 @@ where
arrow_array::FixedSizeListArray::new(
Arc::new(T::as_field("item")),
i32::try_from(N).expect("overflow"),
Arc::<<T as crate::arrow::Array>::Array>::new(value.0.into()),
Arc::<<T as crate::arrow::Array>::Array>::new(self.0.into()),
None,
)
}
}


impl<const N: usize, T: crate::arrow::Array, Buffer: BufferType>
From<FixedSizeListArray<N, T, Nullable, Buffer>> for arrow_array::FixedSizeListArray
Into<arrow_array::FixedSizeListArray> for FixedSizeListArray<N, T, Nullable, Buffer>
where
<T as crate::arrow::Array>::Array: From<T> + 'static,
T: Into<<T as crate::arrow::Array>::Array> + 'static,
Bitmap<Buffer>: Into<NullBuffer>,
{
fn from(value: FixedSizeListArray<N, T, Nullable, Buffer>) -> Self {
fn into(self) -> arrow_array::FixedSizeListArray {
// todo(mbrobbel): const_assert
assert!(N <= 0x7FFF_FFFF); // i32::MAX
#[allow(
Expand All @@ -118,8 +122,8 @@ where
arrow_array::FixedSizeListArray::new(
Arc::new(T::as_field("item")),
i32::try_from(N).expect("overflow"),
Arc::<<T as crate::arrow::Array>::Array>::new(value.0.data.into()),
Some(value.0.validity.into()),
Arc::<<T as crate::arrow::Array>::Array>::new(self.0.data.into()),
Some(self.0.validity.into()),
)
}
}
Expand Down Expand Up @@ -183,7 +187,7 @@ mod tests {
.into_iter()
.collect::<FixedSizeListArray<2, Uint32Array>>();
assert_eq!(
arrow_array::FixedSizeListArray::from(fixed_size_list_array)
Into::<arrow_array::FixedSizeListArray>::into(fixed_size_list_array)
.iter()
.flatten()
.flat_map(|dyn_array| {
Expand All @@ -199,7 +203,7 @@ mod tests {
.into_iter()
.collect::<FixedSizeListArray<2, StringArray, Nullable>>();
assert_eq!(
arrow_array::FixedSizeListArray::from(fixed_size_list_array_nullable)
Into::<arrow_array::FixedSizeListArray>::into(fixed_size_list_array_nullable)
.iter()
.flatten()
.flat_map(|dyn_array| {
Expand Down Expand Up @@ -281,8 +285,8 @@ mod tests {
let fixed_size_list_array_nullable_input = INPUT_NULLABLE
.into_iter()
.collect::<FixedSizeListArray<2, StringArray, Nullable>>();
let fixed_size_list_array_nullable =
arrow_array::FixedSizeListArray::from(fixed_size_list_array_nullable_input);
let fixed_size_list_array_nullable: arrow_array::FixedSizeListArray =
fixed_size_list_array_nullable_input.into();

assert_eq!(
FixedSizeListArray::<
Expand Down
Loading