When requests repeatedly fail, indexer may eventually panic with `fatal error: runtime: out of memory`
When requests fail, the indexer will continue to retry them. In cases where the requests always fail, such as the max_bulk_size being too large, the indexer may go into a loop where the process is unable to allocate memory and panics.
Previously this scenario did not occur because the indexer would panic after the first failed request, but !50 (merged) fixed that issue.
Stacktrace:
goroutine 14 [running]:
runtime.systemstack_switch()
/usr/local/go/src/runtime/asm_amd64.s:311 fp=0xc000d71630 sp=0xc000d71628 pc=0x457d20
runtime.mallocgc(0xa261ae, 0xbfd240, 0x1, 0x1)
/usr/local/go/src/runtime/malloc.go:949 +0x872 fp=0xc000d716d0 sp=0xc000d71630 pc=0x40dcb2
runtime.makeslice(0xbfd240, 0x0, 0xa261ae, 0x0)
/usr/local/go/src/runtime/slice.go:49 +0x6c fp=0xc000d71700 sp=0xc000d716d0 pc=0x44329c
github.com/olivere/elastic.(*BulkService).bodyAsString(0xc0001ba2d0, 0x0, 0x1, 0x0, 0x1)
/root/go/pkg/mod/github.com/olivere/elastic@v6.2.24+incompatible/bulk.go:214 +0x68 fp=0xc000d717a0 sp=0xc000d71700 pc=0x829808
github.com/olivere/elastic.(*BulkService).Do(0xc0001ba2d0, 0xef96a0, 0xc00003e018, 0x0, 0x0, 0x1)
/root/go/pkg/mod/github.com/olivere/elastic@v6.2.24+incompatible/bulk.go:240 +0xbd fp=0xc000d71c40 sp=0xc000d717a0 pc=0x829abd
github.com/olivere/elastic.(*bulkWorker).commit.func1(0xa0d03680, 0xc000c82340)
/root/go/pkg/mod/github.com/olivere/elastic@v6.2.24+incompatible/bulk_processor.go:535 +0x7e fp=0xc000d71d80 sp=0xc000d71c40 pc=0x90493e
github.com/olivere/elastic.RetryNotify(0xc00c9aa000, 0xee5a80, 0xc000f4d580, 0xc000d71e60, 0x2, 0x0)
/root/go/pkg/mod/github.com/olivere/elastic@v6.2.24+incompatible/retry.go:40 +0x42 fp=0xc000d71dd0 sp=0xc000d71d80 pc=0x8af6f2
github.com/olivere/elastic.(*bulkWorker).commit(0xc0010eeb80, 0xef96a0, 0xc00003e018, 0x157, 0xc00006df48)
/root/go/pkg/mod/github.com/olivere/elastic@v6.2.24+incompatible/bulk_processor.go:580 +0x1d9 fp=0xc000d71e80 sp=0xc000d71dd0 pc=0x831f69
github.com/olivere/elastic.(*bulkWorker).work(0xc0010eeb80, 0xef96a0, 0xc00003e018)
/root/go/pkg/mod/github.com/olivere/elastic@v6.2.24+incompatible/bulk_processor.go:487 +0x5ff fp=0xc000d71fc8 sp=0xc000d71e80 pc=0x831c2f
runtime.goexit()
/usr/local/go/src/runtime/asm_amd64.s:1337 +0x1 fp=0xc000d71fd0 sp=0xc000d71fc8 pc=0x459df1
created by github.com/olivere/elastic.(*BulkProcessor).Start
/root/go/pkg/mod/github.com/olivere/elastic@v6.2.24+incompatible/bulk_processor.go:336 +0x1af
runtime stack:
runtime.throw(0xd8823b, 0x16)
/usr/local/go/src/runtime/panic.go:617 +0x72
runtime.sysMap(0xc020000000, 0x4000000, 0x17eb338)
/usr/local/go/src/runtime/mem_linux.go:170 +0xc7
runtime.(*mheap).sysAlloc(0x17d2c60, 0xa28000, 0x17d2c70, 0x514)
/usr/local/go/src/r untime/malloc.go:633 +0x1cd
runtime.(*mheap).grow(0x17d2c60, 0x514, 0x0)
/usr/local/go/src/runtime/mheap.go:1222 +0x42
runtime.(*mheap).allocSpanLocked(0x17d2c60, 0x514, 0x17eb348, 0x0)
/usr/local/go/src/runtime/mheap.go:1150 +0x37f
runtime.(*mheap).alloc_m(0x17d2c60, 0x514, 0x101, 0x7f2500000001)
/usr/local/go/src/runtime/mheap.go:977 +0xc2
runtime.(*mheap).alloc.func1()
/usr/local/go/src/runtime/mheap.go:1048 +0x4c
runtime.(*mheap).alloc(0x17d2c60, 0x514, 0x202010101, 0xc000ae0780)
/usr/local/go/src/runtime/mheap.go:1047 +0x8a
runtime.largeAlloc(0xa261ae, 0x17d0101, 0x1)
/usr/local/go/src/runtime/malloc.go:1055 +0x99
runtime.mallocgc.func1()
/usr/local/go/src/runtime/malloc.go:950 +0x46
runtime.systemstack(0x7f2502600000)
/usr/local/go/src/runtime/asm_amd64.s:351 +0x66
runtime.mstart()
/usr/local/go/src/runtime/proc.go:1153
/cc @jrreid @nick.thomas
Edited by Will Chandler (ex-GitLab)