Skip to content

Commit

Permalink
pqarrow/arrowutils: Add support for sorting Timestamps (#934)
Browse files Browse the repository at this point in the history
  • Loading branch information
metalmatze authored Nov 19, 2024
1 parent ecd6b80 commit 3139e12
Show file tree
Hide file tree
Showing 2 changed files with 40 additions and 6 deletions.
4 changes: 3 additions & 1 deletion pqarrow/arrowutils/sort.go
Original file line number Diff line number Diff line change
Expand Up @@ -220,7 +220,7 @@ func newMultiColSorter(
}
ms.Reserve(int(r.NumRows()), len(columns))
for i := range columns {
ms.directions[i] = int(columns[i].Direction.comparison())
ms.directions[i] = columns[i].Direction.comparison()
ms.nullsFirst[i] = columns[i].NullsFirst
}
for i, col := range columns {
Expand All @@ -243,6 +243,8 @@ func newMultiColSorter(
ms.comparisons[i] = newOrderedSorter[string](e, cmp.Compare)
case *array.Binary:
ms.comparisons[i] = newOrderedSorter[[]byte](e, bytes.Compare)
case *array.Timestamp:
ms.comparisons[i] = newOrderedSorter[arrow.Timestamp](e, cmp.Compare)
case *array.Dictionary:
switch elem := e.Dictionary().(type) {
case *array.String:
Expand Down
42 changes: 37 additions & 5 deletions pqarrow/arrowutils/sort_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -109,6 +109,26 @@ func TestSortRecord(t *testing.T) {
Columns: []SortingColumn{{Index: 2, Direction: Descending}},
Indices: []int32{2, 1, 0},
},
{
Name: "By Timestamp column ascending",
Samples: Samples{
{Timestamp: 3},
{Timestamp: 2},
{Timestamp: 1},
},
Columns: []SortingColumn{{Index: 5}},
Indices: []int32{2, 1, 0},
},
{
Name: "By Timestamp column descending",
Samples: Samples{
{Timestamp: 1},
{Timestamp: 2},
{Timestamp: 3},
},
Columns: []SortingColumn{{Index: 5, Direction: Descending}},
Indices: []int32{2, 1, 0},
},
{
Name: "By Dict column ascending",
Samples: Samples{
Expand Down Expand Up @@ -365,11 +385,12 @@ func TestReorderRecord(t *testing.T) {

// Use all supported sort field.
type Sample struct {
Int int64
Double float64
String string
Dict string
Nullable *int64
Int int64
Double float64
String string
Dict string
Nullable *int64
Timestamp arrow.Timestamp
}

type Samples []Sample
Expand Down Expand Up @@ -401,6 +422,11 @@ func (s Samples) Record() arrow.Record {
Type: arrow.PrimitiveTypes.Int64,
Nullable: true,
},
{
Name: "timestamp",
Type: &arrow.TimestampType{},
Nullable: true,
},
}, nil),
)

Expand All @@ -409,11 +435,17 @@ func (s Samples) Record() arrow.Record {
fString := b.Field(2).(*array.StringBuilder)
fDict := b.Field(3).(*array.BinaryDictionaryBuilder)
fNullable := b.Field(4).(*array.Int64Builder)
fTimestamp := b.Field(5).(*array.TimestampBuilder)

for _, v := range s {
fInt.Append(v.Int)
fDouble.Append(v.Double)
fString.Append(v.String)
if v.Timestamp == 0 {
fTimestamp.AppendNull()
} else {
fTimestamp.Append(v.Timestamp)
}
_ = fDict.AppendString(v.Dict)
if v.Nullable != nil {
fNullable.Append(*v.Nullable)
Expand Down

0 comments on commit 3139e12

Please sign in to comment.