Machine creation failed ssh command error
Summary
Today I started experiencing a problem while creating a new runner EC2 machine using AWS spot instances. This setup was working fine for more than a year
When any of my GitLab jobs are starting gitlab-runner provisioning a new EC2 spot instance. After installing Docker, gitlab-runner tries to copy some certs to remote machine through ssh and here's the error Error creating machine: Error running provisioning: ssh command error: driver=amazonec2 name=runner-8x-zjxha-runner-1675346104-2d7914c4 operation=create
Relevant logs and/or screenshots
Feb 2 13:55:05 ip-10-255-3-240 gitlab-runner: Creating machine... #033[0;m driver#033[0;m=amazonec2 name#033[0;m=runner-8x-zjxha-runner-1675346104-2d7914c4 operation#033[0;m=create
Feb 2 13:55:05 ip-10-255-3-240 gitlab-runner: (runner-8x-zjxha-runner-1675346104-2d7914c4) Launching instance...#033[0;m driver#033[0;m=amazonec2 name#033[0;m=runner-8x-zjxha-runner-1675346104-2d7914c4 operation#033[0;m=create
Feb 2 13:55:06 ip-10-255-3-240 gitlab-runner: (runner-8x-zjxha-runner-1675346104-2d7914c4) Waiting for spot instance...#033[0;m driver#033[0;m=amazonec2 name#033[0;m=runner-8x-zjxha-runner-1675346104-2d7914c4 operation#033[0;m=create
Feb 2 13:55:11 ip-10-255-3-240 gitlab-runner: (runner-8x-zjxha-runner-1675346104-2d7914c4) Created spot instance request sir-65tehfhj#033[0;m driver#033[0;m=amazonec2 name#033[0;m=runner-8x-zjxha-runner-1675346104-2d7914c4 operation#033[0;m=create
Feb 2 13:55:11 ip-10-255-3-240 gitlab-runner: (runner-8x-zjxha-runner-1675346104-2d7914c4) Tags are not key value in pairs. 11 elements found#033[0;m driver#033[0;m=amazonec2 name#033[0;m=runner-8x-zjxha-runner-1675346104-2d7914c4 operation#033[0;m=create
Feb 2 13:55:11 ip-10-255-3-240 gitlab-runner: Waiting for machine to be running, this may take a few minutes...#033[0;m driver#033[0;m=amazonec2 name#033[0;m=runner-8x-zjxha-runner-1675346104-2d7914c4 operation#033[0;m=create
Feb 2 13:55:11 ip-10-255-3-240 gitlab-runner: Detecting operating system of created instance... #033[0;m driver#033[0;m=amazonec2 name#033[0;m=runner-8x-zjxha-runner-1675346104-2d7914c4 operation#033[0;m=create
Feb 2 13:55:11 ip-10-255-3-240 gitlab-runner: Waiting for SSH to be available... #033[0;m driver#033[0;m=amazonec2 name#033[0;m=runner-8x-zjxha-runner-1675346104-2d7914c4 operation#033[0;m=create
Feb 2 13:55:30 ip-10-255-3-240 gitlab-runner: Detecting the provisioner... #033[0;m driver#033[0;m=amazonec2 name#033[0;m=runner-8x-zjxha-runner-1675346104-2d7914c4 operation#033[0;m=create
Feb 2 13:55:30 ip-10-255-3-240 gitlab-runner: Provisioning with ubuntu(systemd)... #033[0;m driver#033[0;m=amazonec2 name#033[0;m=runner-8x-zjxha-runner-1675346104-2d7914c4 operation#033[0;m=create
Feb 2 13:55:40 ip-10-255-3-240 dhclient[1982]: XMT: Solicit on eth0, interval 121330ms.
Feb 2 13:55:40 ip-10-255-3-240 gitlab-runner: Installing Docker... #033[0;m driver#033[0;m=amazonec2 name#033[0;m=runner-8x-zjxha-runner-1675346104-2d7914c4 operation#033[0;m=create
Feb 2 13:56:04 ip-10-255-3-240 gitlab-runner: Copying certs to the local machine directory... #033[0;m driver#033[0;m=amazonec2 name#033[0;m=runner-8x-zjxha-runner-1675346104-2d7914c4 operation#033[0;m=create
Feb 2 13:56:05 ip-10-255-3-240 gitlab-runner: Copying certs to the remote machine... #033[0;m driver#033[0;m=amazonec2 name#033[0;m=runner-8x-zjxha-runner-1675346104-2d7914c4 operation#033[0;m=create
Feb 2 13:56:07 ip-10-255-3-240 gitlab-runner: #033[31;1mERROR: Error creating machine: Error running provisioning: ssh command error:#033[0;m #033[31;1mdriver#033[0;m=amazonec2 #033[31;1mname#033[0;m=runner-8x-zjxha-runner-1675346104-2d7914c4 #033[31;1moperation#033[0;m=create
Feb 2 13:56:07 ip-10-255-3-240 gitlab-runner: #033[31;1mERROR: command : printf '%s' '-----BEGIN CERTIFICATE-----#033[0;m #033[31;1mdriver#033[0;m=amazonec2 #033[31;1mname#033[0;m=runner-8x-zjxha-runner-1675346104-2d7914c4 #033[31;1moperation#033[0;m=create
Feb 2 13:56:07 ip-10-255-3-240 gitlab-runner: #033[31;1mERROR: -----END CERTIFICATE----- #033[0;m #033[31;1mdriver#033[0;m=amazonec2 #033[31;1mname#033[0;m=runner-8x-zjxha-runner-1675346104-2d7914c4 #033[31;1moperation#033[0;m=create
Feb 2 13:56:07 ip-10-255-3-240 gitlab-runner: #033[31;1mERROR: ' | sudo tee /etc/docker/ca.pem #033[0;m #033[31;1mdriver#033[0;m=amazonec2 #033[31;1mname#033[0;m=runner-8x-zjxha-runner-1675346104-2d7914c4 #033[31;1moperation#033[0;m=create
Feb 2 13:56:07 ip-10-255-3-240 gitlab-runner: #033[31;1mERROR: err : exit status 1 #033[0;m #033[31;1mdriver#033[0;m=amazonec2 #033[31;1mname#033[0;m=runner-8x-zjxha-runner-1675346104-2d7914c4 #033[31;1moperation#033[0;m=create
Feb 2 13:56:07 ip-10-255-3-240 gitlab-runner: #033[31;1mERROR: output : -----BEGIN CERTIFICATE----- #033[0;m #033[31;1mdriver#033[0;m=amazonec2 #033[31;1mname#033[0;m=runner-8x-zjxha-runner-1675346104-2d7914c4 #033[31;1moperation#033[0;m=create
Feb 2 13:56:07 ip-10-255-3-240 gitlab-runner: #033[31;1mERROR: -----END CERTIFICATE----- #033[0;m #033[31;1mdriver#033[0;m=amazonec2 #033[31;1mname#033[0;m=runner-8x-zjxha-runner-1675346104-2d7914c4 #033[31;1moperation#033[0;m=create
Feb 2 13:56:07 ip-10-255-3-240 gitlab-runner: #033[31;1mERROR: tee: /etc/docker/ca.pem: No such file or directory#033[0;m #033[31;1mdriver#033[0;m=amazonec2 #033[31;1mname#033[0;m=runner-8x-zjxha-runner-1675346104-2d7914c4 #033[31;1moperation#033[0;m=create
Feb 2 13:56:07 ip-10-255-3-240 gitlab-runner: #033[0;33mWARNING: Problem while reading command output #033[0;m #033[0;33merror#033[0;m=read |0: file already closed
Feb 2 13:56:07 ip-10-255-3-240 gitlab-runner: #033[0;33mWARNING: Problem while reading command output #033[0;m #033[0;33merror#033[0;m=read |0: file already closed
Environment description
concurrent = 10
check_interval = 0
sentry_dsn = ""
[[runners]]
name = "gitlab-runner"
url = "https://gitlab.com"
token = "<token>"
executor = "docker+machine"
pre_build_script = ""
post_build_script = ""
pre_clone_script = ""
request_concurrency = 1
output_limit = 10000
limit = 0
[runners.docker]
tls_verify = false
image = "docker:18.03.1-ce"
privileged = true
disable_cache = false
volumes = ["/cache","/certs/client"]
shm_size = 1073741824
pull_policy = "if-not-present"
runtime = ""
helper_image = ""
[runners.docker.tmpfs]
"/var/opt/cache" = "rw,noexec"
[runners.docker.services_tmpfs]
[runners.cache]
Type = "s3"
Shared = true
[runners.cache.s3]
ServerAddress = "s3.amazonaws.com"
BucketName = "<bucket>"
BucketLocation = "eu-west-1"
Insecure = false
[runners.machine]
IdleCount = 0
IdleTime = 1200
MachineDriver = "amazonec2"
MachineName = "runner-%s"
MachineOptions = [
"amazonec2-instance-type=c6i.xlarge",
"amazonec2-region=eu-west-1",
"amazonec2-zone=a",
"amazonec2-vpc-id=<vpc>",
"amazonec2-subnet-id=<subnet>",
"amazonec2-private-address-only=true",
"amazonec2-use-private-address=false",
"amazonec2-request-spot-instance=true",
"amazonec2-spot-price=0.10",
"amazonec2-security-group=gitlab-runner-docker-machine20210924082328686300000002",
"amazonec2-tags=Name,gitlab-runner-docker-machine,Environment,gitlab-runner,Name,gitlab-runner,tf-aws-gitlab-runner:instancelifecycle,spot:yes,tf-aws-gitlab-runner:main,runner-main,",
"amazonec2-use-ebs-optimized-instance=true",
"amazonec2-monitoring=false",
"amazonec2-iam-instance-profile=gitlab-runner-docker-machine",
"amazonec2-root-size=50",
"amazonec2-ami=ami-0c68b55d1c875067e"
]
Used GitLab Runner version
# gitlab-runner --version
Version: 14.0.1
Git revision: c1edb478
Git branch: refs/pipelines/326100216
GO version: go1.13.8
Built: 2021-06-23T16:35:23+0000
OS/Arch: linux/amd64
I tried to update amazonec2-ami to the latest, no luck