CI Pipeline Hangs

Drone Support
1

Hi all,

I’m looking for a bit of help diagnosing an issue I’m having with a CI pipeline. I’m using this jsonnet template to simultaneously run checks and install_docker_cross, then once those are complete, the builds should start. When trying to run this as-is, the checks pipeline runs and succeeded, but the install_docker_cross never starts. This is on push and on tag. On push, install_docker_cross and the builds shouldn’t run, but as it is right now, it tries to. On tag, it behaves the same and just hangs forever.

image
image1242×2511 122 KB

Here’s my configuration:
.drone.jsonnet

local checks = {
  kind: "pipeline",
  type: "docker",
  name: "check",
  steps: [
    {
      name: "check",
      image: "rust",
      commands: [
        "cargo check",
      ]
    }
  ]
};

local install_docker_cross = {
  kind: "pipeline",
  type: "docker",
  name: "install_docker_cross",
  when: {
    event: "tag"
  },
  steps: [
    {
      name: "install_docker_cross",
      image: "rust",
      commands: [
        "curl -fsSL https://download.docker.com/linux/static/stable/x86_64/docker-18.03.1-ce.tgz | tar zxvf - --strip 1 -C /usr/bin docker/docker",
        "cargo install cross"
      ]
    }
  ]
};

local build(arch) = {
  kind: "pipeline",
  type: "docker",
  name: "rust-stable-" + arch,
  depends_on: [
    "check",
    "install_docker_cross"
  ],
  when: {
    event: "tag"
  },
  steps: [
    {
      name: "build",
      image: "rust",
      volumes: [
        {
          name: "dockersock",
          path: "/var/run/docker.sock",
          readonly: true
        },
        {
          name: "builds",
          path: "/builds",
        }
      ],
      commands: [
        "CROSS_DOCKER_IN_DOCKER=true cross build --release --target " + arch,
        "tar -czvf /builds/rust_rsa-" + arch + ".tar.gz -C target/" + arch + "/release ."
      ],
    },
    {
      name: "publish",
      image: "plugins/github-release",
      volumes: [
        {
          name: "builds",
          path: "/builds",
        }
      ],
      settings: {
        "api_key": { from_secret: "github_token" },
        "files": "/builds/rust_rsa-" + arch + ".tar.gz"
      },
    }
  ],
  volumes: [
    {
      name: "dockersock",
      host: {
        path: "/var/run/docker.sock"
      }
    },
    {
      name: "builds",
      temp: {}
    },
  ]
};

[
  checks,
  install_docker_cross,
  build("aarch64-unknown-linux-gnu"),
  build("aarch64-unknown-linux-musl"),
  build("arm-unknown-linux-gnueabi"),
  build("arm-unknown-linux-gnueabihf"),
  build("arm-unknown-linux-musleabi"),
  build("arm-unknown-linux-musleabihf"),
  build("armv7-unknown-linux-gnueabihf"),
  build("armv7-unknown-linux-musleabihf"),
  build("x86_64-pc-windows-gnu"),
  build("x86_64-unknown-linux-gnu"),
  build("x86_64-unknown-linux-musl")
]
2

Please see this topic which can help diagnose reasons why stages may be stuck in a pending state https://discuss.harness.io/t/builds-are-stuck-in-pending-status/11275

3

Hi @brad,

Thanks for providing the troubleshooting link. I took a look at each step and provided some information here and checked my configurations against some of the recommended troubleshooting steps.

Here’s the latest CI file (I updated the cargo home directory, but the same issue still stands)

local checks = {
  kind: "pipeline",
  type: "docker",
  name: "check",
  steps: [
    {
      name: "check",
      image: "rust",
      volumes: [
        {
          name: "cargo",
          path: "/cargo-cache",
        }
      ],
      environment: {
        CARGO_HOME: "/cargo-cache"
      },
      commands: [
        "cargo check",
      ]
    }
  ],
  volumes: [
    {
      name: "cargo",
      temp: {}
    }
  ]
};

local install_docker_cross = {
  kind: "pipeline",
  type: "docker",
  name: "install_docker_cross",
  when: {
    event: "tag"
  },
  depends_on: [
    "check"
  ],
  steps: [
    {
      name: "install_docker_cross",
      image: "rust",
      volumes: [
        {
          name: "cargo",
          path: "/cargo-cache",
        }
      ],
      environment: {
        CARGO_HOME: "/cargo-cache"
      },
      commands: [
        "curl -fsSL https://download.docker.com/linux/static/stable/x86_64/docker-18.03.1-ce.tgz | tar zxvf - --strip 1 -C /usr/bin docker/docker",
        "cargo install cross"
      ]
    }
  ],
  volumes: [
    {
      name: "cargo",
      temp: {}
    }
  ]
};

local build(arch) = {
  kind: "pipeline",
  type: "docker",
  name: "rust-stable-" + arch,
  depends_on: [
    "install_docker_cross"
  ],
  when: {
    event: "tag"
  },
  steps: [
    {
      name: "build",
      image: "rust",
      volumes: [
        {
          name: "dockersock",
          path: "/var/run/docker.sock",
          readonly: true
        },
        {
          name: "cargo",
          path: "/cargo-cache",
        },
        {
          name: "builds",
          path: "/builds",
        }
      ],
      environment: {
        CARGO_HOME: "/cargo-cache"
      },
      commands: [
        "CROSS_DOCKER_IN_DOCKER=true cross build --release --target " + arch,
        "tar -czvf /builds/rust_rsa-" + arch + ".tar.gz -C target/" + arch + "/release ."
      ],
    },
    {
      name: "publish",
      image: "plugins/github-release",
      volumes: [
        {
          name: "builds",
          path: "/builds",
        }
      ],
      settings: {
        "api_key": { from_secret: "github_token" },
        "files": "/builds/rust_rsa-" + arch + ".tar.gz"
      },
    }
  ],
  volumes: [
    {
      name: "dockersock",
      host: {
        path: "/var/run/docker.sock"
      }
    },
    {
      name: "builds",
      temp: {}
    },
    {
      name: "cargo",
      temp: {}
    }
  ]
};

[
  checks,
  install_docker_cross,
  build("aarch64-unknown-linux-gnu"),
  build("aarch64-unknown-linux-musl"),
  build("arm-unknown-linux-gnueabi"),
  build("arm-unknown-linux-gnueabihf"),
  build("arm-unknown-linux-musleabi"),
  build("arm-unknown-linux-musleabihf"),
  build("armv7-unknown-linux-gnueabihf"),
  build("armv7-unknown-linux-musleabihf"),
  build("x86_64-pc-windows-gnu"),
  build("x86_64-unknown-linux-gnu"),
  build("x86_64-unknown-linux-musl")
]

Here is the server and runner configuration in docker-compose.yml:

version: '3.3'
services:
    server:
        volumes:
            - '/var/lib/drone:/data'
        environment:
            - DRONE_GITHUB_CLIENT_ID=${DRONE_GITHUB_CLIENT_ID}
            - DRONE_GITHUB_CLIENT_SECRET=${DRONE_GITHUB_CLIENT_SECRET}
            - DRONE_RPC_SECRET=${DRONE_RPC_SECRET}
            - DRONE_SERVER_HOST=drone.jeffresc.dev
            - DRONE_SERVER_PROTO=https
            - DRONE_USER_FILTER=JeffResc
            - DRONE_JSONNET_ENABLED=true
            - DRONE_LOGS_TRACE=true
        ports:
            - '80:80'
            - '443:443'
        restart: always
        container_name: drone-server
        image: 'drone/drone:2'
        networks:
          - drone

    runner:
        volumes:
            - '/var/run/docker.sock:/var/run/docker.sock'
        environment:
            - DRONE_RPC_PROTO=http
            - DRONE_RPC_HOST=server
            - DRONE_RPC_SECRET=${DRONE_RPC_SECRET}
            - DRONE_RUNNER_CAPACITY=4
            - DRONE_RUNNER_NAME=runner
            - DRONE_UI_USERNAME=${DRONE_UI_USERNAME}
            - DRONE_UI_PASSWORD=${DRONE_UI_PASSWORD}
            - DRONE_LOGS_TRACE=true
            - DRONE_RPC_DUMP_HTTP=true
            - DRONE_RPC_DUMP_HTTP_BODY=true
        ports:
            - '3000:3000'
        depends_on:
          - server
        restart: always
        container_name: drone-runner
        image: 'drone/drone-runner-docker:1'
        networks:
          - drone

networks:
  drone:
    driver: bridge

Here are the relevant trace logs with sensitive information redacted: Logs

Build information from the API endpoint: Build info

I believe I checked all of the troubleshooting steps correctly, but maybe I missed something. Looking at the logs, I’m not finding anything obvious that’s creating an issue in this build pipeline. Any help here would be greatly appreciated, and let me know if there’s anything else I can provide.

4

I checked the logs you provided (thanks for that) and noticed the install_docker_cross stage has a status of blocked. That might imply you have protected builds enabled which will block the pipeline if the yaml is unsigned. Just checking, did you intend to enable this checkbox? I only ask because some folks enable not really knowing the implications, and unchecking will unblock them.

5

Hi @brad,

Yes, I do have protected builds turned on because the Docker socket is passed into some of the pipeline steps. I approved the pipeline via the web UI and the build starts, but not the install_docker_cross pipeline. Upon checking the tab for that pipeline, it doesn’t seek any approval, it just stays blocked. If I cancel the build and restart it, this fixes the approval process, but only temporarily for that build. This is a public repo where pull requests can build against the CI system, is there a better way to protect the CI host from malicious contributors?