agentk getting stuck if kas is not accessible
Run agentk without a kas to connect to. Agentk gets stuck and doesn't exit on sigterm. Stacktrace dump is below. It looks like Goroutine 15
is the offender.
Goroutine 1
runtime.gopark (/opt/homebrew/Cellar/go/1.20.6/libexec/src/runtime/proc.go:382)
runtime.chanrecv (/opt/homebrew/Cellar/go/1.20.6/libexec/src/runtime/chan.go:583)
runtime.chanrecv1 (/opt/homebrew/Cellar/go/1.20.6/libexec/src/runtime/chan.go:442)
github.com/ash2k/stager.(*stager).Run (/Users/mike/go/pkg/mod/github.com/ash2k/stager@v0.3.0/stager.go:58)
github.com/ash2k/stager.RunStages (/Users/mike/go/pkg/mod/github.com/ash2k/stager@v0.3.0/stager.go:77)
gitlab.com/gitlab-org/cluster-integration/gitlab-agent/v16/cmd/agentk/agentkapp.(*App).Run (/Users/mike/src/gitlab-agent/cmd/agentk/agentkapp/app.go:187)
gitlab.com/gitlab-org/cluster-integration/gitlab-agent/v16/cmd/agentk/agentkapp.NewCommand.func1 (/Users/mike/src/gitlab-agent/cmd/agentk/agentkapp/app.go:451)
github.com/spf13/cobra.(*Command).execute (/Users/mike/go/pkg/mod/github.com/spf13/cobra@v1.7.0/command.go:940)
github.com/spf13/cobra.(*Command).ExecuteC (/Users/mike/go/pkg/mod/github.com/spf13/cobra@v1.7.0/command.go:1068)
github.com/spf13/cobra.(*Command).Execute (/Users/mike/go/pkg/mod/github.com/spf13/cobra@v1.7.0/command.go:992)
github.com/spf13/cobra.(*Command).ExecuteContext (/Users/mike/go/pkg/mod/github.com/spf13/cobra@v1.7.0/command.go:985)
gitlab.com/gitlab-org/cluster-integration/gitlab-agent/v16/cmd.run (/Users/mike/src/gitlab-agent/cmd/utils.go:30)
gitlab.com/gitlab-org/cluster-integration/gitlab-agent/v16/cmd.Run (/Users/mike/src/gitlab-agent/cmd/utils.go:16)
main.main (/Users/mike/src/gitlab-agent/cmd/agentk/main.go:9)
runtime.main (/opt/homebrew/Cellar/go/1.20.6/libexec/src/runtime/proc.go:250)
runtime.goexit (/opt/homebrew/Cellar/go/1.20.6/libexec/src/runtime/asm_arm64.s:1172)
created at: runtime.newproc (<autogenerated>:1)
Goroutine 2
runtime.gopark (/opt/homebrew/Cellar/go/1.20.6/libexec/src/runtime/proc.go:382)
runtime.goparkunlock (/opt/homebrew/Cellar/go/1.20.6/libexec/src/runtime/proc.go:387)
runtime.forcegchelper (/opt/homebrew/Cellar/go/1.20.6/libexec/src/runtime/proc.go:305)
runtime.goexit (/opt/homebrew/Cellar/go/1.20.6/libexec/src/runtime/asm_arm64.s:1172)
created at: runtime.init.6 (/opt/homebrew/Cellar/go/1.20.6/libexec/src/runtime/proc.go:293)
Goroutine 3
runtime.gopark (/opt/homebrew/Cellar/go/1.20.6/libexec/src/runtime/proc.go:382)
runtime.runfinq (/opt/homebrew/Cellar/go/1.20.6/libexec/src/runtime/mfinal.go:193)
runtime.goexit (/opt/homebrew/Cellar/go/1.20.6/libexec/src/runtime/asm_arm64.s:1172)
created at: runtime.createfing (/opt/homebrew/Cellar/go/1.20.6/libexec/src/runtime/mfinal.go:163)
Goroutine 4
runtime.gopark (/opt/homebrew/Cellar/go/1.20.6/libexec/src/runtime/proc.go:382)
runtime.gcBgMarkWorker (/opt/homebrew/Cellar/go/1.20.6/libexec/src/runtime/mgc.go:1275)
runtime.goexit (/opt/homebrew/Cellar/go/1.20.6/libexec/src/runtime/asm_arm64.s:1172)
created at: runtime.gcBgMarkStartWorkers (/opt/homebrew/Cellar/go/1.20.6/libexec/src/runtime/mgc.go:1199)
Goroutine 5
runtime.gopark (/opt/homebrew/Cellar/go/1.20.6/libexec/src/runtime/proc.go:382)
runtime.gcBgMarkWorker (/opt/homebrew/Cellar/go/1.20.6/libexec/src/runtime/mgc.go:1275)
runtime.goexit (/opt/homebrew/Cellar/go/1.20.6/libexec/src/runtime/asm_arm64.s:1172)
created at: runtime.gcBgMarkStartWorkers (/opt/homebrew/Cellar/go/1.20.6/libexec/src/runtime/mgc.go:1199)
Goroutine 6
runtime.gopark (/opt/homebrew/Cellar/go/1.20.6/libexec/src/runtime/proc.go:382)
runtime.gcBgMarkWorker (/opt/homebrew/Cellar/go/1.20.6/libexec/src/runtime/mgc.go:1275)
runtime.goexit (/opt/homebrew/Cellar/go/1.20.6/libexec/src/runtime/asm_arm64.s:1172)
created at: runtime.gcBgMarkStartWorkers (/opt/homebrew/Cellar/go/1.20.6/libexec/src/runtime/mgc.go:1199)
Goroutine 7
runtime.gopark (/opt/homebrew/Cellar/go/1.20.6/libexec/src/runtime/proc.go:382)
runtime.gcBgMarkWorker (/opt/homebrew/Cellar/go/1.20.6/libexec/src/runtime/mgc.go:1275)
runtime.goexit (/opt/homebrew/Cellar/go/1.20.6/libexec/src/runtime/asm_arm64.s:1172)
created at: runtime.gcBgMarkStartWorkers (/opt/homebrew/Cellar/go/1.20.6/libexec/src/runtime/mgc.go:1199)
Goroutine 8
runtime.gopark (/opt/homebrew/Cellar/go/1.20.6/libexec/src/runtime/proc.go:382)
runtime.gcBgMarkWorker (/opt/homebrew/Cellar/go/1.20.6/libexec/src/runtime/mgc.go:1275)
runtime.goexit (/opt/homebrew/Cellar/go/1.20.6/libexec/src/runtime/asm_arm64.s:1172)
created at: runtime.gcBgMarkStartWorkers (/opt/homebrew/Cellar/go/1.20.6/libexec/src/runtime/mgc.go:1199)
Goroutine 10
runtime.gopark (/opt/homebrew/Cellar/go/1.20.6/libexec/src/runtime/proc.go:382)
runtime.selectgo (/opt/homebrew/Cellar/go/1.20.6/libexec/src/runtime/select.go:327)
runtime.ensureSigM.func1 (/opt/homebrew/Cellar/go/1.20.6/libexec/src/runtime/signal_unix.go:1004)
runtime.goexit (/opt/homebrew/Cellar/go/1.20.6/libexec/src/runtime/asm_arm64.s:1172)
created at: runtime.ensureSigM (/opt/homebrew/Cellar/go/1.20.6/libexec/src/runtime/signal_unix.go:987)
Goroutine 12
runtime.gopark (/opt/homebrew/Cellar/go/1.20.6/libexec/src/runtime/proc.go:382)
runtime.chanrecv (/opt/homebrew/Cellar/go/1.20.6/libexec/src/runtime/chan.go:583)
runtime.chanrecv2 (/opt/homebrew/Cellar/go/1.20.6/libexec/src/runtime/chan.go:447)
k8s.io/apimachinery/pkg/watch.(*Broadcaster).loop (/Users/mike/go/pkg/mod/k8s.io/apimachinery@v0.26.5/pkg/watch/mux.go:268)
k8s.io/apimachinery/pkg/watch.NewLongQueueBroadcaster.func1 (/Users/mike/go/pkg/mod/k8s.io/apimachinery@v0.26.5/pkg/watch/mux.go:93)
runtime.goexit (/opt/homebrew/Cellar/go/1.20.6/libexec/src/runtime/asm_arm64.s:1172)
created at: k8s.io/apimachinery/pkg/watch.NewLongQueueBroadcaster (/Users/mike/go/pkg/mod/k8s.io/apimachinery@v0.26.5/pkg/watch/mux.go:93)
Goroutine 13
runtime.gopark (/opt/homebrew/Cellar/go/1.20.6/libexec/src/runtime/proc.go:382)
runtime.chanrecv (/opt/homebrew/Cellar/go/1.20.6/libexec/src/runtime/chan.go:583)
runtime.chanrecv2 (/opt/homebrew/Cellar/go/1.20.6/libexec/src/runtime/chan.go:447)
k8s.io/client-go/tools/record.(*eventBroadcasterImpl).StartEventWatcher.func1 (/Users/mike/go/pkg/mod/k8s.io/client-go@v0.26.5/tools/record/event.go:307)
runtime.goexit (/opt/homebrew/Cellar/go/1.20.6/libexec/src/runtime/asm_arm64.s:1172)
created at: k8s.io/client-go/tools/record.(*eventBroadcasterImpl).StartEventWatcher (/Users/mike/go/pkg/mod/k8s.io/client-go@v0.26.5/tools/record/event.go:305)
Goroutine 14
runtime.gopark (/opt/homebrew/Cellar/go/1.20.6/libexec/src/runtime/proc.go:382)
runtime.chanrecv (/opt/homebrew/Cellar/go/1.20.6/libexec/src/runtime/chan.go:583)
runtime.chanrecv2 (/opt/homebrew/Cellar/go/1.20.6/libexec/src/runtime/chan.go:447)
k8s.io/client-go/tools/record.(*eventBroadcasterImpl).StartEventWatcher.func1 (/Users/mike/go/pkg/mod/k8s.io/client-go@v0.26.5/tools/record/event.go:307)
runtime.goexit (/opt/homebrew/Cellar/go/1.20.6/libexec/src/runtime/asm_arm64.s:1172)
created at: k8s.io/client-go/tools/record.(*eventBroadcasterImpl).StartEventWatcher (/Users/mike/go/pkg/mod/k8s.io/client-go@v0.26.5/tools/record/event.go:305)
Goroutine 15
runtime.gopark (/opt/homebrew/Cellar/go/1.20.6/libexec/src/runtime/proc.go:382)
runtime.chanrecv (/opt/homebrew/Cellar/go/1.20.6/libexec/src/runtime/chan.go:583)
runtime.chanrecv1 (/opt/homebrew/Cellar/go/1.20.6/libexec/src/runtime/chan.go:442)
gitlab.com/gitlab-org/cluster-integration/gitlab-agent/v16/cmd/agentk/agentkapp.(*leaderRunner).Run (/Users/mike/src/gitlab-agent/cmd/agentk/agentkapp/leader_runner.go:155)
gitlab.com/gitlab-org/cluster-integration/gitlab-agent/v16/cmd/agentk/agentkapp.(*App).Run.func2.1 (/Users/mike/src/gitlab-agent/cmd/agentk/agentkapp/app.go:191)
github.com/ash2k/stager.(*stage).Go.func1 (/Users/mike/go/pkg/mod/github.com/ash2k/stager@v0.3.0/stage.go:25)
runtime.goexit (/opt/homebrew/Cellar/go/1.20.6/libexec/src/runtime/asm_arm64.s:1172)
created at: github.com/ash2k/stager.(*stage).Go (/Users/mike/go/pkg/mod/github.com/ash2k/stager@v0.3.0/stage.go:24)
Goroutine 18
runtime.gopark (/opt/homebrew/Cellar/go/1.20.6/libexec/src/runtime/proc.go:382)
runtime.goparkunlock (/opt/homebrew/Cellar/go/1.20.6/libexec/src/runtime/proc.go:387)
runtime.bgsweep (/opt/homebrew/Cellar/go/1.20.6/libexec/src/runtime/mgcsweep.go:319)
runtime.gcenable.func1 (/opt/homebrew/Cellar/go/1.20.6/libexec/src/runtime/mgc.go:178)
runtime.goexit (/opt/homebrew/Cellar/go/1.20.6/libexec/src/runtime/asm_arm64.s:1172)
created at: runtime.gcenable (/opt/homebrew/Cellar/go/1.20.6/libexec/src/runtime/mgc.go:178)
Goroutine 19
runtime.gopark (/opt/homebrew/Cellar/go/1.20.6/libexec/src/runtime/proc.go:382)
runtime.goparkunlock (/opt/homebrew/Cellar/go/1.20.6/libexec/src/runtime/proc.go:387)
runtime.(*scavengerState).park (/opt/homebrew/Cellar/go/1.20.6/libexec/src/runtime/mgcscavenge.go:400)
runtime.bgscavenge (/opt/homebrew/Cellar/go/1.20.6/libexec/src/runtime/mgcscavenge.go:633)
runtime.gcenable.func2 (/opt/homebrew/Cellar/go/1.20.6/libexec/src/runtime/mgc.go:179)
runtime.goexit (/opt/homebrew/Cellar/go/1.20.6/libexec/src/runtime/asm_arm64.s:1172)
created at: runtime.gcenable (/opt/homebrew/Cellar/go/1.20.6/libexec/src/runtime/mgc.go:179)
Goroutine 20
runtime.gopark (/opt/homebrew/Cellar/go/1.20.6/libexec/src/runtime/proc.go:382)
runtime.gcBgMarkWorker (/opt/homebrew/Cellar/go/1.20.6/libexec/src/runtime/mgc.go:1275)
runtime.goexit (/opt/homebrew/Cellar/go/1.20.6/libexec/src/runtime/asm_arm64.s:1172)
created at: runtime.gcBgMarkStartWorkers (/opt/homebrew/Cellar/go/1.20.6/libexec/src/runtime/mgc.go:1199)
Goroutine 21
runtime.gopark (/opt/homebrew/Cellar/go/1.20.6/libexec/src/runtime/proc.go:382)
runtime.gcBgMarkWorker (/opt/homebrew/Cellar/go/1.20.6/libexec/src/runtime/mgc.go:1275)
runtime.goexit (/opt/homebrew/Cellar/go/1.20.6/libexec/src/runtime/asm_arm64.s:1172)
created at: runtime.gcBgMarkStartWorkers (/opt/homebrew/Cellar/go/1.20.6/libexec/src/runtime/mgc.go:1199)
Goroutine 22
runtime.gopark (/opt/homebrew/Cellar/go/1.20.6/libexec/src/runtime/proc.go:382)
runtime.selectgo (/opt/homebrew/Cellar/go/1.20.6/libexec/src/runtime/select.go:327)
os/signal.NotifyContext.func1 (/opt/homebrew/Cellar/go/1.20.6/libexec/src/os/signal/signal.go:288)
runtime.goexit (/opt/homebrew/Cellar/go/1.20.6/libexec/src/runtime/asm_arm64.s:1172)
created at: os/signal.NotifyContext (/opt/homebrew/Cellar/go/1.20.6/libexec/src/os/signal/signal.go:287)
Goroutine 27
runtime.gopark (/opt/homebrew/Cellar/go/1.20.6/libexec/src/runtime/proc.go:382)
runtime.selectgo (/opt/homebrew/Cellar/go/1.20.6/libexec/src/runtime/select.go:327)
google.golang.org/grpc/internal/grpcsync.(*CallbackSerializer).run (/Users/mike/go/pkg/mod/google.golang.org/grpc@v1.56.2/internal/grpcsync/callback_serializer.go:83)
google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer.func1 (/Users/mike/go/pkg/mod/google.golang.org/grpc@v1.56.2/internal/grpcsync/callback_serializer.go:55)
runtime.goexit (/opt/homebrew/Cellar/go/1.20.6/libexec/src/runtime/asm_arm64.s:1172)
created at: google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer (/Users/mike/go/pkg/mod/google.golang.org/grpc@v1.56.2/internal/grpcsync/callback_serializer.go:55)
Goroutine 28
runtime.gopark (/opt/homebrew/Cellar/go/1.20.6/libexec/src/runtime/proc.go:382)
runtime.selectgo (/opt/homebrew/Cellar/go/1.20.6/libexec/src/runtime/select.go:327)
google.golang.org/grpc/internal/grpcsync.(*CallbackSerializer).run (/Users/mike/go/pkg/mod/google.golang.org/grpc@v1.56.2/internal/grpcsync/callback_serializer.go:83)
google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer.func1 (/Users/mike/go/pkg/mod/google.golang.org/grpc@v1.56.2/internal/grpcsync/callback_serializer.go:55)
runtime.goexit (/opt/homebrew/Cellar/go/1.20.6/libexec/src/runtime/asm_arm64.s:1172)
created at: google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer (/Users/mike/go/pkg/mod/google.golang.org/grpc@v1.56.2/internal/grpcsync/callback_serializer.go:55)
Goroutine 30
runtime.gopark (/opt/homebrew/Cellar/go/1.20.6/libexec/src/runtime/proc.go:382)
runtime.selectgo (/opt/homebrew/Cellar/go/1.20.6/libexec/src/runtime/select.go:327)
google.golang.org/grpc/internal/grpcsync.(*CallbackSerializer).run (/Users/mike/go/pkg/mod/google.golang.org/grpc@v1.56.2/internal/grpcsync/callback_serializer.go:83)
google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer.func1 (/Users/mike/go/pkg/mod/google.golang.org/grpc@v1.56.2/internal/grpcsync/callback_serializer.go:55)
runtime.goexit (/opt/homebrew/Cellar/go/1.20.6/libexec/src/runtime/asm_arm64.s:1172)
created at: google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer (/Users/mike/go/pkg/mod/google.golang.org/grpc@v1.56.2/internal/grpcsync/callback_serializer.go:55)
Goroutine 31
runtime.gopark (/opt/homebrew/Cellar/go/1.20.6/libexec/src/runtime/proc.go:382)
runtime.selectgo (/opt/homebrew/Cellar/go/1.20.6/libexec/src/runtime/select.go:327)
google.golang.org/grpc/internal/grpcsync.(*CallbackSerializer).run (/Users/mike/go/pkg/mod/google.golang.org/grpc@v1.56.2/internal/grpcsync/callback_serializer.go:83)
google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer.func1 (/Users/mike/go/pkg/mod/google.golang.org/grpc@v1.56.2/internal/grpcsync/callback_serializer.go:55)
runtime.goexit (/opt/homebrew/Cellar/go/1.20.6/libexec/src/runtime/asm_arm64.s:1172)
created at: google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer (/Users/mike/go/pkg/mod/google.golang.org/grpc@v1.56.2/internal/grpcsync/callback_serializer.go:55)
Goroutine 34
runtime.gopark (/opt/homebrew/Cellar/go/1.20.6/libexec/src/runtime/proc.go:382)
runtime.gcBgMarkWorker (/opt/homebrew/Cellar/go/1.20.6/libexec/src/runtime/mgc.go:1275)
runtime.goexit (/opt/homebrew/Cellar/go/1.20.6/libexec/src/runtime/asm_arm64.s:1172)
created at: runtime.gcBgMarkStartWorkers (/opt/homebrew/Cellar/go/1.20.6/libexec/src/runtime/mgc.go:1199)
Goroutine 35
runtime.gopark (/opt/homebrew/Cellar/go/1.20.6/libexec/src/runtime/proc.go:382)
runtime.gcBgMarkWorker (/opt/homebrew/Cellar/go/1.20.6/libexec/src/runtime/mgc.go:1275)
runtime.goexit (/opt/homebrew/Cellar/go/1.20.6/libexec/src/runtime/asm_arm64.s:1172)
created at: runtime.gcBgMarkStartWorkers (/opt/homebrew/Cellar/go/1.20.6/libexec/src/runtime/mgc.go:1199)
Goroutine 36
runtime.gopark (/opt/homebrew/Cellar/go/1.20.6/libexec/src/runtime/proc.go:382)
runtime.gcBgMarkWorker (/opt/homebrew/Cellar/go/1.20.6/libexec/src/runtime/mgc.go:1275)
runtime.goexit (/opt/homebrew/Cellar/go/1.20.6/libexec/src/runtime/asm_arm64.s:1172)
created at: runtime.gcBgMarkStartWorkers (/opt/homebrew/Cellar/go/1.20.6/libexec/src/runtime/mgc.go:1199)
Goroutine 66
runtime.gopark (/opt/homebrew/Cellar/go/1.20.6/libexec/src/runtime/proc.go:382)
time.Sleep (/opt/homebrew/Cellar/go/1.20.6/libexec/src/runtime/time.go:195)
sigs.k8s.io/controller-runtime/pkg/log.init.0.func1 (/Users/mike/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.14.6/pkg/log/log.go:63)
runtime.goexit (/opt/homebrew/Cellar/go/1.20.6/libexec/src/runtime/asm_arm64.s:1172)
created at: sigs.k8s.io/controller-runtime/pkg/log.init.0 (/Users/mike/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.14.6/pkg/log/log.go:62)
Goroutine 82
<unknown>
runtime.systemstack_switch (/opt/homebrew/Cellar/go/1.20.6/libexec/src/runtime/asm_arm64.s:200)
runtime.libcCall (/opt/homebrew/Cellar/go/1.20.6/libexec/src/runtime/sys_libc.go:49)
runtime.read (/opt/homebrew/Cellar/go/1.20.6/libexec/src/runtime/sys_darwin.go:269)
runtime.sigNoteSleep (/opt/homebrew/Cellar/go/1.20.6/libexec/src/runtime/os_darwin.go:124)
os/signal.signal_recv (/opt/homebrew/Cellar/go/1.20.6/libexec/src/runtime/sigqueue.go:149)
os/signal.loop (/opt/homebrew/Cellar/go/1.20.6/libexec/src/os/signal/signal_unix.go:23)
runtime.goexit (/opt/homebrew/Cellar/go/1.20.6/libexec/src/runtime/asm_arm64.s:1172)
created at: os/signal.Notify.func1.1 (/opt/homebrew/Cellar/go/1.20.6/libexec/src/os/signal/signal.go:151)
Goroutine 151
runtime.gopark (/opt/homebrew/Cellar/go/1.20.6/libexec/src/runtime/proc.go:382)
runtime.selectgo (/opt/homebrew/Cellar/go/1.20.6/libexec/src/runtime/select.go:327)
google.golang.org/grpc.(*addrConn).resetTransport (/Users/mike/go/pkg/mod/google.golang.org/grpc@v1.56.2/clientconn.go:1352)
google.golang.org/grpc.(*addrConn).connect (/Users/mike/go/pkg/mod/google.golang.org/grpc@v1.56.2/clientconn.go:978)
google.golang.org/grpc.(*acBalancerWrapper).Connect.func1 (/Users/mike/go/pkg/mod/google.golang.org/grpc@v1.56.2/balancer_conn_wrappers.go:398)
runtime.goexit (/opt/homebrew/Cellar/go/1.20.6/libexec/src/runtime/asm_arm64.s:1172)
created at: google.golang.org/grpc.(*acBalancerWrapper).Connect (/Users/mike/go/pkg/mod/google.golang.org/grpc@v1.56.2/balancer_conn_wrappers.go:398)