[solved] Drone builds stuck in pending after server move

Drone Support
1

Hi,

I recently moved my Drone instance from one server to another, but after the move all my builds are stuck in pending mode. As per Builds are Stuck in Pending Status, my logs (following a restart and “Test Delivery” from my Gitea instance) and my configuration will be posted in the comments.

Any ideas?

All configuration is the same as on the previous server, where everything worked perfectly. I can access the web interface, and my Test Deliveries arrive - the builds just don’t start.

2

Logs:

license: ""
authn:
  endpoint: ""
  secret: ""
  skipverify: false
agent:
  disabled: false
azureblob:
  containername: ""
  storageaccountname: ""
  storageaccesskey: ""
convert:
  extension: ""
  endpoint: ""
  secret: ""
  skipverify: false
cron:
  disabled: false
  interval: 30m0s
cloning:
  alwaysauth: true
  username: ""
  password: ""
  image: ""
  pull: IfNotExists
database:
  driver: sqlite3
  datasource: /var/lib/drone/drone.sqlite
  secret: ""
  legacybatch: false
datadog:
  enabled: true
  endpoint: https://stats.drone.ci/api/v1/series
  token: ""
docker:
  config: ""
http:
  allowedhosts: []
  hostsproxyheaders: []
  sslredirect: false
  ssltemporaryredirect: false
  sslhost: ""
  sslproxyheaders: {}
  stsseconds: 0
  stsincludesubdomains: false
  stspreload: false
  forcestsheader: false
  browserxssfilter: true
  framedeny: true
  contenttypenosniff: false
  contentsecuritypolicy: ""
  referrerpolicy: ""
jsonnet:
  enabled: false
logging:
  debug: false
  trace: true
  color: false
  pretty: false
  text: false
prometheus:
  enableanonymousaccess: false
proxy:
  addr: https://drone.domain.com
  host: drone.domain.com
  proto: https
registration:
  closed: false
registries:
  endpoint: ""
  password: ""
  skipverify: false
repository:
  filter: []
  visibility: ""
  trusted: false
runner:
  local: false
  image: drone/controller:1.0.0
  platform: linux/amd64
  os: linux
  arch: amd64
  kernel: ""
  variant: ""
  machine: b52afcf6bb7a
  capacity: 2
  labels: {}
  volumes: []
  networks: []
  devices: []
  privileged: []
  environ: {}
  limits:
    memswaplimit: 0
    memlimit: 0
    shmsize: 0
    cpuquota: 0
    cpushares: 0
    cpuset: ""
nomad:
  enabled: false
  datacenters:
  - dc1
  namespace: ""
  region: ""
  prefix: drone-job-
  image: ""
  imagepull: false
  memory: 1024
  cpu: 500
kube:
  enabled: false
  namespace: ""
  path: ""
  url: ""
  ttl: 300
  serviceaccountname: ""
  pullpolicy: Always
  image: ""
rpc:
  server: ""
  secret: ""
  debug: false
  host: drone.domain.com
  proto: https
s3:
  bucket: ""
  prefix: ""
  endpoint: ""
  pathstyle: false
secrets:
  endpoint: ""
  password: ""
  skipverify: false
server:
  addr: https://drone.domain.com
  host: drone.domain.com
  port: :80
  proto: https
  acme: false
  email: ""
  cert: ""
  key: ""
session:
  timeout: 720h0m0s
  secret: wJepjGbqVP5TZ2iPGlKz2Srn72f1QNRj
  secure: false
  mappingfile: ""
status:
  disabled: false
  name: ""
users:
  create:
    username: user-name
    machine: false
    admin: true
    token: ""
  filter: []
  minage: 0s
validate:
  endpoint: ""
  secret: ""
  skipverify: false
webhook:
  events: []
  endpoint: []
  secret: ""
  skipverify: false
yaml:
  endpoint: ""
  secret: ""
  skipverify: false
bitbucket:
  clientid: ""
  clientsecret: ""
  skipverify: false
  debug: false
gitea:
  server: https://gitea-host
  clientid: 3269ee1f-be22-447a-b164-1fc3c5b6c075
  clientsecret: tG83YcJ5s3BtdfbWqVPWJ_dNVwq5SKM-2NmJZX24wM8=
  skipverify: false
  scope:
  - repo
  - repo:status
  - user:email
  - read:org
  debug: false
github:
  server: https://github.com
  apiserver: https://api.github.com
  clientid: ""
  clientsecret: ""
  skipverify: false
  scope:
  - repo
  - repo:status
  - user:email
  - read:org
  ratelimit: 0
  debug: false
gitlab:
  server: https://gitlab.com
  clientid: ""
  clientsecret: ""
  skipverify: false
  debug: false
gogs:
  server: ""
  skipverify: false
  debug: false
stash:
  server: ""
  consumerkey: ""
  consumersecret: ""
  privatekey: ""
  skipverify: false
  debug: false

{"level":"info","msg":"main: internal scheduler enabled","time":"2019-10-14T09:51:37Z"}
{"build.limit":0,"expires":"0001-01-01T00:00:00Z","kind":"trial","level":"debug","msg":"main: license loaded","repo.limit":0,"time":"2019-10-14T09:51:37Z","user.limit":0}
{"admin":true,"level":"debug","login":"user-name","machine":false,"msg":"bootstrap: create account","time":"2019-10-14T09:51:37Z","token":""}
{"admin":true,"level":"debug","login":"user-name","machine":false,"msg":"bootstrap: updating account","time":"2019-10-14T09:51:37Z","token":""}
{"admin":true,"level":"debug","login":"user-name","machine":false,"msg":"bootstrap: account already up-to-date","time":"2019-10-14T09:51:37Z","token":""}
{"acme":false,"host":"drone.domain.com","level":"info","msg":"starting the http server","port":":80","proto":"https","time":"2019-10-14T09:51:37Z","url":"https://drone.domain.com"}
{"interval":"30m0s","level":"info","msg":"starting the cron scheduler","time":"2019-10-14T09:51:37Z"}
{"level":"debug","msg":"events: stream opened","request-id":"1SBpAQ78e3BSYVtmwO59P2WSK2B","time":"2019-10-14T09:51:38Z"}
...
{"fields.time":"2019-10-14T09:53:45Z","latency":1687910,"level":"debug","method":"GET","msg":"","remote":"172.27.0.2:40210","request":"/api/repos/user-name/sandbox/builds?page=1","request-id":"1SBpQNoLihyMbApnmPbFP1Amvgw","time":"2019-10-14T09:53:45Z"}
{"commit":"992a0926b37ce9b663a93effcac117561d0d538a","event":"push","level":"debug","msg":"webhook parsed","name":"sandbox","namespace":"user-name","time":"2019-10-14T09:53:55Z"}
{"commit":"992a0926b37ce9b663a93effcac117561d0d538a","event":"push","level":"debug","msg":"trigger: received","ref":"refs/heads/hello-world-yaml","repo":"user-name/sandbox","time":"2019-10-14T09:53:55Z"}
{"fields.time":"2019-10-14T09:53:55Z","latency":40060952,"level":"debug","method":"POST","msg":"","remote":"172.27.0.2:40420","request":"/hook?secret=0IYsZR2X5XczWN4Bwqk1Z6KfWQRWeiIL","request-id":"1SBpRXiWsJeUb5xgx3hJZSgpFxy","time":"2019-10-14T09:53:55Z"}
{"level":"debug","msg":"events: stream error","request-id":"1SBpAQ78e3BSYVtmwO59P2WSK2B","time":"2019-10-14T09:54:08Z"}
{"level":"debug","msg":"events: stream closed","request-id":"1SBpAQ78e3BSYVtmwO59P2WSK2B","time":"2019-10-14T09:54:08Z"}
{"level":"debug","msg":"api: guest access","request-id":"1SBpAQ78e3BSYVtmwO59P2WSK2B","time":"2019-10-14T09:54:08Z"}
{"fields.time":"2019-10-14T09:54:08Z","latency":150002518708,"level":"debug","method":"GET","msg":"","remote":"172.27.0.2:39672","request":"/api/stream","request-id":"1SBpAQ78e3BSYVtmwO59P2WSK2B","time":"2019-10-14T09:54:08Z"}
{"level":"debug","msg":"events: stream cancelled","request-id":"1SBpLrnWWJDVXmpCaiWaWFeBQPH","time":"2019-10-14T09:54:10Z"}
{"level":"debug","msg":"events: stream closed","request-id":"1SBpLrnWWJDVXmpCaiWaWFeBQPH","time":"2019-10-14T09:54:10Z"}
{"level":"debug","msg":"api: guest access","request-id":"1SBpLrnWWJDVXmpCaiWaWFeBQPH","time":"2019-10-14T09:54:10Z"}
{"fields.time":"2019-10-14T09:54:10Z","latency":60001395528,"level":"debug","method":"GET","msg":"","remote":"172.27.0.2:40044","request":"/api/stream","request-id":"1SBpLrnWWJDVXmpCaiWaWFeBQPH","time":"2019-10-14T09:54:10Z"}
{"level":"debug","msg":"events: stream cancelled","request-id":"1SBpMQzVHP61kxW0IinXokrgFhU","time":"2019-10-14T09:54:14Z"}
{"level":"debug","msg":"events: stream closed","request-id":"1SBpMQzVHP61kxW0IinXokrgFhU","time":"2019-10-14T09:54:14Z"}
{"level":"debug","msg":"api: guest access","request-id":"1SBpMQzVHP61kxW0IinXokrgFhU","time":"2019-10-14T09:54:14Z"}
{"fields.time":"2019-10-14T09:54:14Z","latency":60001326806,"level":"debug","method":"GET","msg":"","remote":"172.27.0.2:40048","request":"/api/stream","request-id":"1SBpMQzVHP61kxW0IinXokrgFhU","time":"2019-10-14T09:54:14Z"}
{"level":"debug","msg":"api: guest access","request-id":"1SBpaBfVC4ZKFEH6WtD14C3t7MB","time":"2019-10-14T09:55:03Z"}
{"fields.time":"2019-10-14T09:55:45Z","latency":150003162241,"level":"debug","method":"GET","msg":"","remote":"172.27.0.2:40208","request":"/api/stream","request-id":"1SBpMWlv9vIPXsfctXStyg0K6pC","time":"2019-10-14T09:55:45Z"}
3

Deployment config

version: '2'

services:
...
  drone:
    image: drone/drone:1
    networks:
      - backend
    volumes:
      - drone-data:/var/lib/drone/
      - /var/run/docker.sock:/var/run/docker.sock
    env_file:
      - /.../drone.env
    environment:
      DRONE_SERVER_HOST: "drone.domain.com"
      DRONE_RUNNER_CAPACITY: 2
      DRONE_SERVER_PROTO: "https"
      DRONE_DATABASE_DRIVER: sqlite3
      DRONE_DATABASE_DATASOURCE: /var/lib/drone/drone.sqlite
      DRONE_TLS_AUTOCERT: "false"
      # DRONE_ORGS: ""
      DRONE_ADMIN_ALL: "false"
      # GITEA params
      DRONE_GITEA_SERVER: "https://gitea-host"
      DRONE_GITEA_SKIP_VERIFY: "false"
      DRONE_GIT_ALWAYS_AUTH : "true"
      DRONE_LOGS_TRACE: "true"
      VIRTUAL_HOST: https://drone.domain.com drone.domain.com
      FORCE_SSL: "true"
    expose:
      - 80
      - 443
    restart: always
  haproxy:
    image: dockercloud/haproxy:1.6.7
    ports:
      - 80:80
      - 443:443
    restart: always
    environment:
      ...
    links:
      ...
      - drone
    networks:
      - backend
    depends_on:
      ...
      - drone
    volumes:
      - /var/run/docker.sock:/var/run/docker.sock

volumes:
    ...
    drone-data:

networks:
    backend:

drone.env:

DRONE_SECRET=*******
DRONE_GITEA_CLIENT_ID=*******
DRONE_GITEA_CLIENT_SECRET=*******
DRONE_USER_CREATE=username:user-name,admin:true
4

.drone.yml:

kind: pipeline
name: drone-demo

clone:
  disable: true

steps:
  - name: clone-merge-push
    image: appleboy/drone-git-push
    commands:
      ...
5

You probably have a new version, from the recent change logs https://github.com/drone/drone/blob/master/CHANGELOG.md

DRONE_AGENTS_DISABLED=true

caught me out as well

2 Likes
6

Brilliant, this was the solution. Thanks a bunch!

7

This fixed it for the latest version. It was weird that some builds were getting through too.