Skip to content

Consume HTTP request body and only then write response

Mikhail Mazurskiy requested to merge ash2k/http-server-read-then-write into master

Found while investigating gitlab-org/gitlab#343148 (closed), but it's not the cause of it.

==================
WARNING: DATA RACE
Read at 0x00c000482088 by goroutine 25:
  bufio.(*Reader).Read()
      /usr/local/opt/go/libexec/src/bufio/bufio.go:206 +0x175
  io.(*LimitedReader).Read()
      /usr/local/opt/go/libexec/src/io/io.go:473 +0xc5
  net/http.(*body).readLocked()
      /usr/local/opt/go/libexec/src/net/http/transfer.go:843 +0xb7
  net/http.(*body).Read()
      /usr/local/opt/go/libexec/src/net/http/transfer.go:835 +0x17d
  io.(*LimitedReader).Read()
      /usr/local/opt/go/libexec/src/io/io.go:473 +0xc5
  io.discard.ReadFrom()
      /usr/local/opt/go/libexec/src/io/io.go:598 +0x91
  io.(*discard).ReadFrom()
      <autogenerated>:1 +0x5b
  io.copyBuffer()
      /usr/local/opt/go/libexec/src/io/io.go:409 +0x1c2
  io.Copy()
      /usr/local/opt/go/libexec/src/io/io.go:382 +0xcb
  io.CopyN()
      /usr/local/opt/go/libexec/src/io/io.go:358 +0xad
  net/http.(*chunkWriter).writeHeader()
      /usr/local/opt/go/libexec/src/net/http/server.go:1348 +0xeea
  net/http.(*chunkWriter).flush()
      /usr/local/opt/go/libexec/src/net/http/server.go:395 +0x4d
  net/http.(*response).Flush()
      /usr/local/opt/go/libexec/src/net/http/server.go:1659 +0x78
  gitlab.com/gitlab-org/cluster-integration/gitlab-agent/v14/internal/tool/grpctool.(*InboundHttpToOutboundGrpc).pipeOutboundToInbound.func1()
      /Users/mikhail/src/gitlab-agent/internal/tool/grpctool/inbound_http_to_outbound_grpc.go:110 +0x136
  runtime.call16()
      /usr/local/opt/go/libexec/src/runtime/asm_amd64.s:625 +0x48
  reflect.Value.Call()
      /usr/local/opt/go/libexec/src/reflect/value.go:339 +0xd7
  gitlab.com/gitlab-org/cluster-integration/gitlab-agent/v14/internal/tool/grpctool.(*StreamVisitor).Visit()
      /Users/mikhail/src/gitlab-agent/internal/tool/grpctool/stream_visitor.go:117 +0xbc4
  gitlab.com/gitlab-org/cluster-integration/gitlab-agent/v14/internal/tool/grpctool.(*InboundHttpToOutboundGrpc).pipeOutboundToInbound()
      /Users/mikhail/src/gitlab-agent/internal/tool/grpctool/inbound_http_to_outbound_grpc.go:103 +0x567
  gitlab.com/gitlab-org/cluster-integration/gitlab-agent/v14/internal/tool/grpctool.(*InboundHttpToOutboundGrpc).pipe.func1()
      /Users/mikhail/src/gitlab-agent/internal/tool/grpctool/inbound_http_to_outbound_grpc.go:85 +0x84
  k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1()
      /Users/mikhail/go/pkg/mod/k8s.io/apimachinery@v0.22.2/pkg/util/wait/wait.go:73 +0x7e

Previous write at 0x00c000482088 by goroutine 17:
  bufio.(*Reader).reset()
      /usr/local/opt/go/libexec/src/bufio/bufio.go:76 +0x9d
  bufio.(*Reader).Reset()
      /usr/local/opt/go/libexec/src/bufio/bufio.go:72 +0x2a
  net/http.putBufioReader()
      /usr/local/opt/go/libexec/src/net/http/server.go:829 +0x34
  net/http.(*conn).finalFlush()
      /usr/local/opt/go/libexec/src/net/http/server.go:1666 +0x64
  net/http.(*conn).close()
      /usr/local/opt/go/libexec/src/net/http/server.go:1681 +0x2e
  net/http.(*conn).serve.func1()
      /usr/local/opt/go/libexec/src/net/http/server.go:1805 +0x1ed
  runtime.deferCallSave()
      /usr/local/opt/go/libexec/src/runtime/panic.go:950 +0x81
  testing.(*common).Fatalf()
      /usr/local/opt/go/libexec/src/testing/testing.go:830 +0x84
  testing.(*T).Fatalf()
      <autogenerated>:1 +0x75
  github.com/golang/mock/gomock.(*Controller).Call.func1()
      /Users/mikhail/go/pkg/mod/github.com/golang/mock@v1.6.0/gomock/controller.go:231 +0x44c
  github.com/golang/mock/gomock.(*Controller).Call()
      /Users/mikhail/go/pkg/mod/github.com/golang/mock@v1.6.0/gomock/controller.go:247 +0xcd
  gitlab.com/gitlab-org/cluster-integration/gitlab-agent/v14/internal/tool/testing/mock_kubernetes_api.(*MockKubernetesApi_MakeRequestClient).Send()
      /Users/mikhail/src/gitlab-agent/internal/tool/testing/mock_kubernetes_api/rpc.go:159 +0xe6
  gitlab.com/gitlab-org/cluster-integration/gitlab-agent/v14/internal/tool/grpctool.(*InboundHttpToOutboundGrpc).pipeInboundToOutbound()
      /Users/mikhail/src/gitlab-agent/internal/tool/grpctool/inbound_http_to_outbound_grpc.go:146 +0x512
  gitlab.com/gitlab-org/cluster-integration/gitlab-agent/v14/internal/tool/grpctool.(*InboundHttpToOutboundGrpc).pipe()
      /Users/mikhail/src/gitlab-agent/internal/tool/grpctool/inbound_http_to_outbound_grpc.go:88 +0x2ba
  gitlab.com/gitlab-org/cluster-integration/gitlab-agent/v14/internal/tool/grpctool.(*InboundHttpToOutboundGrpc).Pipe()
      /Users/mikhail/src/gitlab-agent/internal/tool/grpctool/inbound_http_to_outbound_grpc.go:62 +0x94
  gitlab.com/gitlab-org/cluster-integration/gitlab-agent/v14/internal/module/kubernetes_api/server.(*kubernetesApiProxy).pipeStreams()
      /Users/mikhail/src/gitlab-agent/internal/module/kubernetes_api/server/proxy.go:169 +0x6fe
  gitlab.com/gitlab-org/cluster-integration/gitlab-agent/v14/internal/module/kubernetes_api/server.(*kubernetesApiProxy).proxy()
      /Users/mikhail/src/gitlab-agent/internal/module/kubernetes_api/server/proxy.go:133 +0xc34
  gitlab.com/gitlab-org/cluster-integration/gitlab-agent/v14/internal/module/kubernetes_api/server.(*kubernetesApiProxy).proxy-fm()
      /Users/mikhail/src/gitlab-agent/internal/module/kubernetes_api/server/proxy.go:70 +0x57
  net/http.HandlerFunc.ServeHTTP()
      /usr/local/opt/go/libexec/src/net/http/server.go:2046 +0x4d
  gitlab.com/gitlab-org/labkit/correlation.InjectCorrelationID.func1()
      /Users/mikhail/go/pkg/mod/gitlab.com/gitlab-org/labkit@v1.10.0/correlation/inbound_http.go:43 +0x848
  net/http.HandlerFunc.ServeHTTP()
      /usr/local/opt/go/libexec/src/net/http/server.go:2046 +0x4d
  net/http.serverHandler.ServeHTTP()
      /usr/local/opt/go/libexec/src/net/http/server.go:2878 +0x89a
  net/http.(*conn).serve()
      /usr/local/opt/go/libexec/src/net/http/server.go:1929 +0x12e4
  net/http.(*Server).Serve·dwrap·82()
      /usr/local/opt/go/libexec/src/net/http/server.go:3033 +0x58

Goroutine 25 (running) created at:
  k8s.io/apimachinery/pkg/util/wait.(*Group).Start()
      /Users/mikhail/go/pkg/mod/k8s.io/apimachinery@v0.22.2/pkg/util/wait/wait.go:71 +0xde
  gitlab.com/gitlab-org/cluster-integration/gitlab-agent/v14/internal/tool/grpctool.(*InboundHttpToOutboundGrpc).pipe()
      /Users/mikhail/src/gitlab-agent/internal/tool/grpctool/inbound_http_to_outbound_grpc.go:84 +0x28b
  gitlab.com/gitlab-org/cluster-integration/gitlab-agent/v14/internal/tool/grpctool.(*InboundHttpToOutboundGrpc).Pipe()
      /Users/mikhail/src/gitlab-agent/internal/tool/grpctool/inbound_http_to_outbound_grpc.go:62 +0x94
  gitlab.com/gitlab-org/cluster-integration/gitlab-agent/v14/internal/module/kubernetes_api/server.(*kubernetesApiProxy).pipeStreams()
      /Users/mikhail/src/gitlab-agent/internal/module/kubernetes_api/server/proxy.go:169 +0x6fe
  gitlab.com/gitlab-org/cluster-integration/gitlab-agent/v14/internal/module/kubernetes_api/server.(*kubernetesApiProxy).proxy()
      /Users/mikhail/src/gitlab-agent/internal/module/kubernetes_api/server/proxy.go:133 +0xc34
  gitlab.com/gitlab-org/cluster-integration/gitlab-agent/v14/internal/module/kubernetes_api/server.(*kubernetesApiProxy).proxy-fm()
      /Users/mikhail/src/gitlab-agent/internal/module/kubernetes_api/server/proxy.go:70 +0x57
  net/http.HandlerFunc.ServeHTTP()
      /usr/local/opt/go/libexec/src/net/http/server.go:2046 +0x4d
  gitlab.com/gitlab-org/labkit/correlation.InjectCorrelationID.func1()
      /Users/mikhail/go/pkg/mod/gitlab.com/gitlab-org/labkit@v1.10.0/correlation/inbound_http.go:43 +0x848
  net/http.HandlerFunc.ServeHTTP()
      /usr/local/opt/go/libexec/src/net/http/server.go:2046 +0x4d
  net/http.serverHandler.ServeHTTP()
      /usr/local/opt/go/libexec/src/net/http/server.go:2878 +0x89a
  net/http.(*conn).serve()
      /usr/local/opt/go/libexec/src/net/http/server.go:1929 +0x12e4
  net/http.(*Server).Serve·dwrap·82()
      /usr/local/opt/go/libexec/src/net/http/server.go:3033 +0x58

Goroutine 17 (finished) created at:
  net/http.(*Server).Serve()
      /usr/local/opt/go/libexec/src/net/http/server.go:3033 +0x847
  gitlab.com/gitlab-org/cluster-integration/gitlab-agent/v14/internal/tool/httpz.RunServer()
      /Users/mikhail/src/gitlab-agent/internal/tool/httpz/run_server.go:29 +0x293
  gitlab.com/gitlab-org/cluster-integration/gitlab-agent/v14/internal/module/kubernetes_api/server.(*kubernetesApiProxy).Run()
      /Users/mikhail/src/gitlab-agent/internal/module/kubernetes_api/server/proxy.go:67 +0x249
  gitlab.com/gitlab-org/cluster-integration/gitlab-agent/v14/internal/module/kubernetes_api/server.setupProxyWithHandler.func4()
      /Users/mikhail/src/gitlab-agent/internal/module/kubernetes_api/server/proxy_test.go:497 +0x6e
  k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1()
      /Users/mikhail/go/pkg/mod/k8s.io/apimachinery@v0.22.2/pkg/util/wait/wait.go:73 +0x7e
==================
Edited by Mikhail Mazurskiy

Merge request reports