Merge "Unify the behaviors of Shard*(...) utility functions" into main

This commit is contained in:
Jihoon Kang
2024-04-17 17:21:02 +00:00
committed by Gerrit Code Review

View File

@@ -524,22 +524,27 @@ func SplitFileExt(name string) (string, string, string) {
return root, suffix, ext return root, suffix, ext
} }
// ShardPaths takes a Paths, and returns a slice of Paths where each one has at most shardSize paths. func shard[T ~[]E, E any](toShard T, shardSize int) []T {
func ShardPaths(paths Paths, shardSize int) []Paths { if len(toShard) == 0 {
if len(paths) == 0 {
return nil return nil
} }
ret := make([]Paths, 0, (len(paths)+shardSize-1)/shardSize)
for len(paths) > shardSize { ret := make([]T, 0, (len(toShard)+shardSize-1)/shardSize)
ret = append(ret, paths[0:shardSize]) for len(toShard) > shardSize {
paths = paths[shardSize:] ret = append(ret, toShard[0:shardSize])
toShard = toShard[shardSize:]
} }
if len(paths) > 0 { if len(toShard) > 0 {
ret = append(ret, paths) ret = append(ret, toShard)
} }
return ret return ret
} }
// ShardPaths takes a Paths, and returns a slice of Paths where each one has at most shardSize paths.
func ShardPaths(paths Paths, shardSize int) []Paths {
return shard(paths, shardSize)
}
// ShardString takes a string and returns a slice of strings where the length of each one is // ShardString takes a string and returns a slice of strings where the length of each one is
// at most shardSize. // at most shardSize.
func ShardString(s string, shardSize int) []string { func ShardString(s string, shardSize int) []string {
@@ -560,18 +565,7 @@ func ShardString(s string, shardSize int) []string {
// ShardStrings takes a slice of strings, and returns a slice of slices of strings where each one has at most shardSize // ShardStrings takes a slice of strings, and returns a slice of slices of strings where each one has at most shardSize
// elements. // elements.
func ShardStrings(s []string, shardSize int) [][]string { func ShardStrings(s []string, shardSize int) [][]string {
if len(s) == 0 { return shard(s, shardSize)
return nil
}
ret := make([][]string, 0, (len(s)+shardSize-1)/shardSize)
for len(s) > shardSize {
ret = append(ret, s[0:shardSize])
s = s[shardSize:]
}
if len(s) > 0 {
ret = append(ret, s)
}
return ret
} }
// CheckDuplicate checks if there are duplicates in given string list. // CheckDuplicate checks if there are duplicates in given string list.