From 218c50c9f55bf8dd6b9b692907f6545aa82676d9 Mon Sep 17 00:00:00 2001 From: liyixin <601947961@qq.com> Date: Thu, 11 Apr 2024 16:43:08 +0800 Subject: [PATCH 1/3] fix duration conversion error --- parquet/src/arrow/schema/primitive.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/parquet/src/arrow/schema/primitive.rs b/parquet/src/arrow/schema/primitive.rs index f1fed8f2a557..cc276eb611b0 100644 --- a/parquet/src/arrow/schema/primitive.rs +++ b/parquet/src/arrow/schema/primitive.rs @@ -43,6 +43,7 @@ fn apply_hint(parquet: DataType, hint: DataType) -> DataType { (DataType::Int32 | DataType::Int64, DataType::Timestamp(_, _)) => hint, (DataType::Int32, DataType::Time32(_)) => hint, (DataType::Int64, DataType::Time64(_)) => hint, + (DataType::Int64, DataType::Duration(_)) => hint, // Date64 doesn't have a corresponding LogicalType / ConvertedType (DataType::Int64, DataType::Date64) => hint, From bb9351b4ff8caf6a85553b16435996b6345d3862 Mon Sep 17 00:00:00 2001 From: liyixin Date: Thu, 8 May 2025 09:43:40 +0800 Subject: [PATCH 2/3] implement durtion type convert in parquet writer --- parquet/src/arrow/arrow_writer/mod.rs | 5 +++++ parquet/src/arrow/schema/mod.rs | 5 ++++- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/parquet/src/arrow/arrow_writer/mod.rs b/parquet/src/arrow/arrow_writer/mod.rs index 1e1054c9a063..57a4aedbc1ce 100644 --- a/parquet/src/arrow/arrow_writer/mod.rs +++ b/parquet/src/arrow/arrow_writer/mod.rs @@ -2437,6 +2437,11 @@ mod tests { one_column_roundtrip(values, false); } + #[test] + fn duration_single_column() { + required_and_optional::(0..SMALL_SIZE as i64); + } + #[test] fn date32_single_column() { required_and_optional::(0..SMALL_SIZE as i32); diff --git a/parquet/src/arrow/schema/mod.rs b/parquet/src/arrow/schema/mod.rs index 89c42f5eaf92..d264e0cce897 100644 --- a/parquet/src/arrow/schema/mod.rs +++ b/parquet/src/arrow/schema/mod.rs @@ -586,7 +586,10 @@ fn arrow_to_parquet_type(field: &Field, coerce_types: bool) -> Result { .with_repetition(repetition) .with_id(id) .build(), - DataType::Duration(_) => Err(arrow_err!("Converting Duration to parquet not supported",)), + DataType::Duration(_) => Type::primitive_type_builder(name, PhysicalType::INT64) + .with_repetition(repetition) + .with_id(id) + .build(), DataType::Interval(_) => { Type::primitive_type_builder(name, PhysicalType::FIXED_LEN_BYTE_ARRAY) .with_converted_type(ConvertedType::INTERVAL) From 6ad3f20fc47c4cac3c803ddebe32995d60615bde Mon Sep 17 00:00:00 2001 From: Andrew Lamb Date: Thu, 8 May 2025 13:43:20 -0400 Subject: [PATCH 3/3] Use existing tests, remove new test --- parquet/src/arrow/arrow_writer/mod.rs | 9 --------- 1 file changed, 9 deletions(-) diff --git a/parquet/src/arrow/arrow_writer/mod.rs b/parquet/src/arrow/arrow_writer/mod.rs index 57a4aedbc1ce..67987baebc61 100644 --- a/parquet/src/arrow/arrow_writer/mod.rs +++ b/parquet/src/arrow/arrow_writer/mod.rs @@ -2437,11 +2437,6 @@ mod tests { one_column_roundtrip(values, false); } - #[test] - fn duration_single_column() { - required_and_optional::(0..SMALL_SIZE as i64); - } - #[test] fn date32_single_column() { required_and_optional::(0..SMALL_SIZE as i32); @@ -2476,25 +2471,21 @@ mod tests { } #[test] - #[should_panic(expected = "Converting Duration to parquet not supported")] fn duration_second_single_column() { required_and_optional::(0..SMALL_SIZE as i64); } #[test] - #[should_panic(expected = "Converting Duration to parquet not supported")] fn duration_millisecond_single_column() { required_and_optional::(0..SMALL_SIZE as i64); } #[test] - #[should_panic(expected = "Converting Duration to parquet not supported")] fn duration_microsecond_single_column() { required_and_optional::(0..SMALL_SIZE as i64); } #[test] - #[should_panic(expected = "Converting Duration to parquet not supported")] fn duration_nanosecond_single_column() { required_and_optional::(0..SMALL_SIZE as i64); }