We’re experiencing an issue where our Golang services, running in private images, occasionally get stuck in a pending state.
There doesn’t seem to be any consistency to when it happens. Restarting the build will sometimes help and sometimes not.
We’re using a self-hosted instance of Drone. I’ve looked into the logs and there don’t seem to be any notable entries there.
Any clue what might cause this issue?
---
name: integration-tests
kind: pipeline
type: docker
workspace:
base: /go
path: src/src.dualinventive.com/railcloud/warning-integration-tests
steps:
- name: sql-files
image: alpine:latest
commands:
- cp ./database/* /tmp/database
volumes:
- name: sql-files
path: /tmp/database
- name: database
detach: true
image: mariadb:latest
environment:
MARIADB_ROOT_PASSWORD: guest
volumes:
- name: sql-files
path: /docker-entrypoint-initdb.d
depends_on:
- sql-files
- name: dependencies
image: golang:latest
commands:
- go mod download
- go list -json -deps *.go > deps.txt
- name: dependency-updates
image: sonatypecommunity/nancy:alpine
environment:
OSSI_USERNAME:
from_secret: sona-type-user
OSSI_TOKEN:
from_secret: sona-type-token
commands:
- nancy sleuth --loud < deps.txt
depends_on:
- dependencies
failure: ignore
- name: dependency-audit
image: sonatypecommunity/nancy:alpine
environment:
OSSI_USERNAME:
from_secret: sona-type-user
OSSI_TOKEN:
from_secret: sona-type-token
commands:
- nancy sleuth < deps.txt
depends_on:
- dependencies
- name: lint
image: golangci/golangci-lint:latest
commands:
- ./scripts/lint.sh
volumes:
- name: golangci-lint-cache
path: /root/.cache/golangci-lint
depends_on:
- dependencies
- name: test
image: golang:1.19
environment:
RELEASE_DB: root:guest@(database:3306)/warning_release
RELEASE_API: http://release-api:9999
CONTROL_DB: root:guest@(database:3306)/warning_control
CONTROL_API: http://control-api:9999
SYSTEM_DB: root:guest@(database:3306)/warning_system
commands:
- apt update && apt install -y mariadb-client
- while (! mysqladmin ping -hdatabase --silent); do sleep 1; done
- while [ $(curl -so /dev/null -w "%{http_code}" http://release-api:9998/health) -ne 200 ]; do sleep 1; done
- while [ $(curl -so /dev/null -w "%{http_code}" http://control-api:9998/health) -ne 200 ]; do sleep 1; done
- while [ $(curl -so /dev/null -w "%{http_code}" http://warning-system:9999/health) -ne 200 ]; do sleep 1; done
- while [ $(curl -so /dev/null -w "%{http_code}" http://duu-gateway:9999/health) -ne 200 ]; do sleep 1; done
- while [ $(curl -so /dev/null -w "%{http_code}" http://dum-gateway:9999/health) -ne 200 ]; do sleep 1; done
- while [ $(curl -so /dev/null -w "%{http_code}" http://wum-gateway:9999/health) -ne 200 ]; do sleep 1; done
- go test ./...
depends_on:
- dependencies
volumes:
- name: sql-files
temp: { }
- name: golangci-lint-cache
host:
path: /tmp/golangci-lint
services:
- name: release-api
image: dualinventivedocker/di-warning-release-api:latest
environment:
PULSAR_CA:
from_secret: pulsar_dev_ca
PULSAR_CERT:
from_secret: pulsar_dev_cert
PULSAR_KEY:
from_secret: pulsar_dev_key
commands:
- ./scripts/add_certs.sh
- apt update && apt install -y mariadb-client
- while (! mysqladmin ping -hdatabase --silent); do sleep 1; done
- /bin/di-warning-release-api -config configs/warning-release-api.toml
- name: control-api
image: dualinventivedocker/di-warning-control-api:latest
environment:
PULSAR_CA:
from_secret: pulsar_dev_ca
PULSAR_CERT:
from_secret: pulsar_dev_cert
PULSAR_KEY:
from_secret: pulsar_dev_key
commands:
- ./scripts/add_certs.sh
- apt update && apt install -y mariadb-client
- while (! mysqladmin ping -hdatabase --silent); do sleep 1; done
- /bin/di-warning-control-api -config configs/warning-control-api.toml
- name: warning-system
image: dualinventivedocker/di-warning-system:latest
environment:
PULSAR_CA:
from_secret: pulsar_dev_ca
PULSAR_CERT:
from_secret: pulsar_dev_cert
PULSAR_KEY:
from_secret: pulsar_dev_key
commands:
- ./scripts/add_certs.sh
- apt update && apt install -y mariadb-client
- while (! mysqladmin ping -hdatabase --silent); do sleep 1; done
- /bin/di-warning-system -config configs/warning-system.toml
- name: duu-gateway
image: dualinventivedocker/di-duu-gateway:latest
environment:
PULSAR_CA:
from_secret: pulsar_dev_ca
PULSAR_CERT:
from_secret: pulsar_dev_cert
PULSAR_KEY:
from_secret: pulsar_dev_key
commands:
- ./scripts/add_certs.sh
- /bin/di-duu-gateway -config configs/duu-gateway.toml
- name: dum-gateway
image: dualinventivedocker/di-dum-gateway:latest
environment:
PULSAR_CA:
from_secret: pulsar_dev_ca
PULSAR_CERT:
from_secret: pulsar_dev_cert
PULSAR_KEY:
from_secret: pulsar_dev_key
commands:
- ./scripts/add_certs.sh
- /bin/di-dum-gateway -config configs/dum-gateway.toml
- name: wum-gateway
image: dualinventivedocker/di-wum-gateway:latest
environment:
PULSAR_CA:
from_secret: pulsar_dev_ca
PULSAR_CERT:
from_secret: pulsar_dev_cert
PULSAR_KEY:
from_secret: pulsar_dev_key
commands:
- ./scripts/add_certs.sh
- /bin/di-wum-gateway -config configs/wum-gateway.toml
- name: who-is
image: dualinventivedocker/di-who-is:latest
environment:
PULSAR_CA:
from_secret: pulsar_dev_ca
PULSAR_CERT:
from_secret: pulsar_dev_cert
PULSAR_KEY:
from_secret: pulsar_dev_key
commands:
- ./scripts/add_certs.sh
- apt update && apt install -y mariadb-client
- while (! mysqladmin ping -hdatabase --silent); do sleep 1; done
- /bin/di-who-is -config configs/who-is.toml
- name: simulators
image: dualinventivedocker/di-dinet-simulator:latest
environment:
PULSAR_CA:
from_secret: pulsar_dev_ca
PULSAR_CERT:
from_secret: pulsar_dev_cert
PULSAR_KEY:
from_secret: pulsar_dev_key
commands:
- ./scripts/add_certs.sh
- apt update && apt install -y curl
- while [ $(curl -so /dev/null -w "%{http_code}" http://who-is:9999/health) -ne 200 ]; do sleep 1; done
- /bin/simulator-linux json --file configs/atws.json
{
"id": 71642,
"repo_id": 12137,
"trigger": "svujinovic",
"number": 187,
"parent": 186,
"status": "killed",
"event": "push",
"action": "",
"link": "https://src.dualinventive.com/railcloud/warning-integration-tests/compare/08032c152b71ca3de41a1a75c2ff0c34987e7363...902d0b5717828af0f5b76c9fe29458f247d515bb",
"timestamp": 0,
"message": "added more zone tests\n",
"before": "08032c152b71ca3de41a1a75c2ff0c34987e7363",
"after": "902d0b5717828af0f5b76c9fe29458f247d515bb",
"ref": "refs/heads/master",
"source_repo": "",
"source": "master",
"target": "master",
"author_login": "svujinovic",
"author_name": "Stefan Vujinovic",
"author_email": "stefan.vujinovic@dualinventive.com",
"author_avatar": "https://src.dualinventive.com/avatar/ac2ff83df3b837c4c4d84f8dc018db2d",
"sender": "svujinovic",
"started": 1673879101,
"finished": 1673879464,
"created": 1673879100,
"updated": 1673879101,
"version": 3,
"stages": [
{
"id": 71851,
"repo_id": 12137,
"build_id": 71642,
"number": 1,
"name": "integration-tests",
"kind": "pipeline",
"type": "docker",
"status": "killed",
"errignore": false,
"exit_code": 0,
"machine": "my-first-runner",
"os": "linux",
"arch": "amd64",
"started": 1673879101,
"stopped": 1673879464,
"created": 1673879100,
"updated": 1673879101,
"version": 4,
"on_success": true,
"on_failure": false,
"steps": [
{
"id": 494872,
"step_id": 71851,
"number": 1,
"name": "clone",
"status": "success",
"exit_code": 0,
"started": 1673879101,
"stopped": 1673879103,
"version": 4,
"image": "drone/git:latest"
},
{
"id": 494873,
"step_id": 71851,
"number": 2,
"name": "release-api",
"status": "killed",
"exit_code": 130,
"started": 1673879103,
"stopped": 1673879464,
"version": 3,
"depends_on": [
"clone"
],
"image": "docker.io/dualinventivedocker/di-warning-release-api:latest",
"detached": true
},
{
"id": 494874,
"step_id": 71851,
"number": 3,
"name": "control-api",
"status": "killed",
"exit_code": 130,
"started": 1673879103,
"stopped": 1673879464,
"version": 3,
"depends_on": [
"clone"
],
"image": "docker.io/dualinventivedocker/di-warning-control-api:latest",
"detached": true
},
{
"id": 494875,
"step_id": 71851,
"number": 4,
"name": "warning-system",
"status": "killed",
"exit_code": 130,
"started": 1673879103,
"stopped": 1673879464,
"version": 3,
"depends_on": [
"clone"
],
"image": "docker.io/dualinventivedocker/di-warning-system:latest",
"detached": true
},
{
"id": 494876,
"step_id": 71851,
"number": 5,
"name": "duu-gateway",
"status": "killed",
"exit_code": 130,
"started": 1673879103,
"stopped": 1673879464,
"version": 3,
"depends_on": [
"clone"
],
"image": "docker.io/dualinventivedocker/di-duu-gateway:latest",
"detached": true
},
{
"id": 494877,
"step_id": 71851,
"number": 6,
"name": "dum-gateway",
"status": "killed",
"exit_code": 130,
"started": 1673879103,
"stopped": 1673879464,
"version": 3,
"depends_on": [
"clone"
],
"image": "docker.io/dualinventivedocker/di-dum-gateway:latest",
"detached": true
},
{
"id": 494878,
"step_id": 71851,
"number": 7,
"name": "wum-gateway",
"status": "killed",
"exit_code": 130,
"started": 1673879103,
"stopped": 1673879464,
"version": 3,
"depends_on": [
"clone"
],
"image": "docker.io/dualinventivedocker/di-wum-gateway:latest",
"detached": true
},
{
"id": 494879,
"step_id": 71851,
"number": 8,
"name": "simulators",
"status": "killed",
"exit_code": 130,
"started": 1673879103,
"stopped": 1673879464,
"version": 3,
"depends_on": [
"clone"
],
"image": "docker.io/dualinventivedocker/di-dinet-simulator:latest",
"detached": true
},
{
"id": 494880,
"step_id": 71851,
"number": 9,
"name": "sql-files",
"status": "success",
"exit_code": 0,
"started": 1673879103,
"stopped": 1673879106,
"version": 3,
"depends_on": [
"clone"
],
"image": "docker.io/library/alpine:latest"
},
{
"id": 494881,
"step_id": 71851,
"number": 10,
"name": "database",
"status": "killed",
"exit_code": 130,
"started": 1673879106,
"stopped": 1673879464,
"version": 3,
"depends_on": [
"sql-files"
],
"image": "docker.io/library/mariadb:latest",
"detached": true
},
{
"id": 494882,
"step_id": 71851,
"number": 11,
"name": "dependencies",
"status": "success",
"exit_code": 0,
"started": 1673879103,
"stopped": 1673879106,
"version": 3,
"depends_on": [
"clone"
],
"image": "docker.io/library/golang:latest"
},
{
"id": 494883,
"step_id": 71851,
"number": 12,
"name": "dependency-updates",
"status": "success",
"errignore": true,
"exit_code": 0,
"started": 1673879106,
"stopped": 1673879108,
"version": 3,
"depends_on": [
"dependencies"
],
"image": "docker.io/sonatypecommunity/nancy:alpine"
},
{
"id": 494884,
"step_id": 71851,
"number": 13,
"name": "dependency-audit",
"status": "success",
"exit_code": 0,
"started": 1673879106,
"stopped": 1673879108,
"version": 3,
"depends_on": [
"dependencies"
],
"image": "docker.io/sonatypecommunity/nancy:alpine"
},
{
"id": 494885,
"step_id": 71851,
"number": 14,
"name": "lint",
"status": "success",
"exit_code": 0,
"started": 1673879106,
"stopped": 1673879111,
"version": 3,
"depends_on": [
"dependencies"
],
"image": "docker.io/golangci/golangci-lint:latest"
},
{
"id": 494886,
"step_id": 71851,
"number": 15,
"name": "test",
"status": "killed",
"exit_code": 130,
"started": 1673879106,
"stopped": 1673879464,
"version": 3,
"depends_on": [
"dependencies"
],
"image": "docker.io/library/golang:1.19"
},
{
"id": 494887,
"step_id": 71851,
"number": 16,
"name": "notify",
"status": "skipped",
"errignore": true,
"exit_code": 130,
"started": 1673879464,
"stopped": 1673879464,
"version": 2,
"depends_on": [
"test"
],
"image": "docker.io/plugins/slack:latest"
}
]
}
]
}