Skip to content

Commit

Permalink
fix(bulk): Batch list in bulk loader to avoid panic (#6446)
Browse files Browse the repository at this point in the history
  • Loading branch information
harshil-goel authored Sep 16, 2020
1 parent 1e179c1 commit 9c8c994
Showing 1 changed file with 10 additions and 1 deletion.
11 changes: 10 additions & 1 deletion dgraph/cmd/bulk/reduce.go
Original file line number Diff line number Diff line change
Expand Up @@ -402,7 +402,16 @@ func (r *reducer) startWriting(ci *countIndexer, writerCh chan *encodeRequest, c
// Wait for it to be encoded.
start := time.Now()

x.Check(ci.writer.Write(req.list))
for len(req.list.GetKv()) > 0 {
batchSize := 100
if len(req.list.Kv) < batchSize {
batchSize = len(req.list.Kv)
}
batch := &bpb.KVList{Kv: req.list.Kv[:batchSize]}
req.list.Kv = req.list.Kv[batchSize:]
x.Check(ci.writer.Write(batch))
}

if req.splitList != nil && len(req.splitList.Kv) > 0 {
splitCh <- req.splitList
}
Expand Down

0 comments on commit 9c8c994

Please sign in to comment.