Steps occasionally stuck in pending state

We’re experiencing an issue where our Golang services, running in private images, occasionally get stuck in a pending state.
There doesn’t seem to be any consistency to when it happens. Restarting the build will sometimes help and sometimes not.
We’re using a self-hosted instance of Drone. I’ve looked into the logs and there don’t seem to be any notable entries there.

Any clue what might cause this issue?

---

name: integration-tests
kind: pipeline
type: docker

workspace:
  base: /go
  path: src/src.dualinventive.com/railcloud/warning-integration-tests

steps:
  - name: sql-files
    image: alpine:latest
    commands:
      - cp ./database/* /tmp/database
    volumes:
      - name: sql-files
        path: /tmp/database

  - name: database
    detach: true
    image: mariadb:latest
    environment:
      MARIADB_ROOT_PASSWORD: guest
    volumes:
      - name: sql-files
        path: /docker-entrypoint-initdb.d
    depends_on:
      - sql-files

  - name: dependencies
    image: golang:latest
    commands:
      - go mod download
      - go list -json -deps *.go > deps.txt

  - name: dependency-updates
    image: sonatypecommunity/nancy:alpine
    environment:
      OSSI_USERNAME:
        from_secret: sona-type-user
      OSSI_TOKEN:
        from_secret: sona-type-token
    commands:
      - nancy sleuth --loud < deps.txt
    depends_on:
      - dependencies
    failure: ignore

  - name: dependency-audit
    image: sonatypecommunity/nancy:alpine
    environment:
      OSSI_USERNAME:
        from_secret: sona-type-user
      OSSI_TOKEN:
        from_secret: sona-type-token
    commands:
      - nancy sleuth < deps.txt
    depends_on:
      - dependencies

  - name: lint
    image: golangci/golangci-lint:latest
    commands:
      - ./scripts/lint.sh
    volumes:
      - name: golangci-lint-cache
        path: /root/.cache/golangci-lint
    depends_on:
      - dependencies

  - name: test
    image: golang:1.19
    environment:
      RELEASE_DB: root:[email protected](database:3306)/warning_release
      RELEASE_API: http://release-api:9999
      CONTROL_DB: root:[email protected](database:3306)/warning_control
      CONTROL_API: http://control-api:9999
      SYSTEM_DB: root:[email protected](database:3306)/warning_system
    commands:
      - apt update && apt install -y mariadb-client
      - while (! mysqladmin ping -hdatabase --silent); do sleep 1; done
      - while [ $(curl -so /dev/null -w "%{http_code}" http://release-api:9998/health) -ne 200 ]; do sleep 1; done
      - while [ $(curl -so /dev/null -w "%{http_code}" http://control-api:9998/health) -ne 200 ]; do sleep 1; done
      - while [ $(curl -so /dev/null -w "%{http_code}" http://warning-system:9999/health) -ne 200 ]; do sleep 1; done
      - while [ $(curl -so /dev/null -w "%{http_code}" http://duu-gateway:9999/health) -ne 200 ]; do sleep 1; done
      - while [ $(curl -so /dev/null -w "%{http_code}" http://dum-gateway:9999/health) -ne 200 ]; do sleep 1; done
      - while [ $(curl -so /dev/null -w "%{http_code}" http://wum-gateway:9999/health) -ne 200 ]; do sleep 1; done
      - go test ./...
    depends_on:
      - dependencies

volumes:
  - name: sql-files
    temp: { }
  - name: golangci-lint-cache
    host:
      path: /tmp/golangci-lint

services:
  - name: release-api
    image: dualinventivedocker/di-warning-release-api:latest
    environment:
      PULSAR_CA:
        from_secret: pulsar_dev_ca
      PULSAR_CERT:
        from_secret: pulsar_dev_cert
      PULSAR_KEY:
        from_secret: pulsar_dev_key
    commands:
      - ./scripts/add_certs.sh
      - apt update && apt install -y mariadb-client
      - while (! mysqladmin ping -hdatabase --silent); do sleep 1; done
      - /bin/di-warning-release-api -config configs/warning-release-api.toml

  - name: control-api
    image: dualinventivedocker/di-warning-control-api:latest
    environment:
      PULSAR_CA:
        from_secret: pulsar_dev_ca
      PULSAR_CERT:
        from_secret: pulsar_dev_cert
      PULSAR_KEY:
        from_secret: pulsar_dev_key
    commands:
      - ./scripts/add_certs.sh
      - apt update && apt install -y mariadb-client
      - while (! mysqladmin ping -hdatabase --silent); do sleep 1; done
      - /bin/di-warning-control-api -config configs/warning-control-api.toml

  - name: warning-system
    image: dualinventivedocker/di-warning-system:latest
    environment:
      PULSAR_CA:
        from_secret: pulsar_dev_ca
      PULSAR_CERT:
        from_secret: pulsar_dev_cert
      PULSAR_KEY:
        from_secret: pulsar_dev_key
    commands:
      - ./scripts/add_certs.sh
      - apt update && apt install -y mariadb-client
      - while (! mysqladmin ping -hdatabase --silent); do sleep 1; done
      - /bin/di-warning-system -config configs/warning-system.toml

  - name: duu-gateway
    image: dualinventivedocker/di-duu-gateway:latest
    environment:
      PULSAR_CA:
        from_secret: pulsar_dev_ca
      PULSAR_CERT:
        from_secret: pulsar_dev_cert
      PULSAR_KEY:
        from_secret: pulsar_dev_key
    commands:
      - ./scripts/add_certs.sh
      - /bin/di-duu-gateway -config configs/duu-gateway.toml

  - name: dum-gateway
    image: dualinventivedocker/di-dum-gateway:latest
    environment:
      PULSAR_CA:
        from_secret: pulsar_dev_ca
      PULSAR_CERT:
        from_secret: pulsar_dev_cert
      PULSAR_KEY:
        from_secret: pulsar_dev_key
    commands:
      - ./scripts/add_certs.sh
      - /bin/di-dum-gateway -config configs/dum-gateway.toml

  - name: wum-gateway
    image: dualinventivedocker/di-wum-gateway:latest
    environment:
      PULSAR_CA:
        from_secret: pulsar_dev_ca
      PULSAR_CERT:
        from_secret: pulsar_dev_cert
      PULSAR_KEY:
        from_secret: pulsar_dev_key
    commands:
      - ./scripts/add_certs.sh
      - /bin/di-wum-gateway -config configs/wum-gateway.toml

  - name: who-is
    image: dualinventivedocker/di-who-is:latest
    environment:
      PULSAR_CA:
        from_secret: pulsar_dev_ca
      PULSAR_CERT:
        from_secret: pulsar_dev_cert
      PULSAR_KEY:
        from_secret: pulsar_dev_key
    commands:
      - ./scripts/add_certs.sh
      - apt update && apt install -y mariadb-client
      - while (! mysqladmin ping -hdatabase --silent); do sleep 1; done
      - /bin/di-who-is -config configs/who-is.toml

  - name: simulators
    image: dualinventivedocker/di-dinet-simulator:latest
    environment:
      PULSAR_CA:
        from_secret: pulsar_dev_ca
      PULSAR_CERT:
        from_secret: pulsar_dev_cert
      PULSAR_KEY:
        from_secret: pulsar_dev_key
    commands:
      - ./scripts/add_certs.sh
      - apt update && apt install -y curl
      - while [ $(curl -so /dev/null -w "%{http_code}" http://who-is:9999/health) -ne 200 ]; do sleep 1; done
      - /bin/simulator-linux json --file configs/atws.json
{
  "id": 71642,
  "repo_id": 12137,
  "trigger": "svujinovic",
  "number": 187,
  "parent": 186,
  "status": "killed",
  "event": "push",
  "action": "",
  "link": "https://src.dualinventive.com/railcloud/warning-integration-tests/compare/08032c152b71ca3de41a1a75c2ff0c34987e7363...902d0b5717828af0f5b76c9fe29458f247d515bb",
  "timestamp": 0,
  "message": "added more zone tests\n",
  "before": "08032c152b71ca3de41a1a75c2ff0c34987e7363",
  "after": "902d0b5717828af0f5b76c9fe29458f247d515bb",
  "ref": "refs/heads/master",
  "source_repo": "",
  "source": "master",
  "target": "master",
  "author_login": "svujinovic",
  "author_name": "Stefan Vujinovic",
  "author_email": "[email protected]",
  "author_avatar": "https://src.dualinventive.com/avatar/ac2ff83df3b837c4c4d84f8dc018db2d",
  "sender": "svujinovic",
  "started": 1673879101,
  "finished": 1673879464,
  "created": 1673879100,
  "updated": 1673879101,
  "version": 3,
  "stages": [
    {
      "id": 71851,
      "repo_id": 12137,
      "build_id": 71642,
      "number": 1,
      "name": "integration-tests",
      "kind": "pipeline",
      "type": "docker",
      "status": "killed",
      "errignore": false,
      "exit_code": 0,
      "machine": "my-first-runner",
      "os": "linux",
      "arch": "amd64",
      "started": 1673879101,
      "stopped": 1673879464,
      "created": 1673879100,
      "updated": 1673879101,
      "version": 4,
      "on_success": true,
      "on_failure": false,
      "steps": [
        {
          "id": 494872,
          "step_id": 71851,
          "number": 1,
          "name": "clone",
          "status": "success",
          "exit_code": 0,
          "started": 1673879101,
          "stopped": 1673879103,
          "version": 4,
          "image": "drone/git:latest"
        },
        {
          "id": 494873,
          "step_id": 71851,
          "number": 2,
          "name": "release-api",
          "status": "killed",
          "exit_code": 130,
          "started": 1673879103,
          "stopped": 1673879464,
          "version": 3,
          "depends_on": [
            "clone"
          ],
          "image": "docker.io/dualinventivedocker/di-warning-release-api:latest",
          "detached": true
        },
        {
          "id": 494874,
          "step_id": 71851,
          "number": 3,
          "name": "control-api",
          "status": "killed",
          "exit_code": 130,
          "started": 1673879103,
          "stopped": 1673879464,
          "version": 3,
          "depends_on": [
            "clone"
          ],
          "image": "docker.io/dualinventivedocker/di-warning-control-api:latest",
          "detached": true
        },
        {
          "id": 494875,
          "step_id": 71851,
          "number": 4,
          "name": "warning-system",
          "status": "killed",
          "exit_code": 130,
          "started": 1673879103,
          "stopped": 1673879464,
          "version": 3,
          "depends_on": [
            "clone"
          ],
          "image": "docker.io/dualinventivedocker/di-warning-system:latest",
          "detached": true
        },
        {
          "id": 494876,
          "step_id": 71851,
          "number": 5,
          "name": "duu-gateway",
          "status": "killed",
          "exit_code": 130,
          "started": 1673879103,
          "stopped": 1673879464,
          "version": 3,
          "depends_on": [
            "clone"
          ],
          "image": "docker.io/dualinventivedocker/di-duu-gateway:latest",
          "detached": true
        },
        {
          "id": 494877,
          "step_id": 71851,
          "number": 6,
          "name": "dum-gateway",
          "status": "killed",
          "exit_code": 130,
          "started": 1673879103,
          "stopped": 1673879464,
          "version": 3,
          "depends_on": [
            "clone"
          ],
          "image": "docker.io/dualinventivedocker/di-dum-gateway:latest",
          "detached": true
        },
        {
          "id": 494878,
          "step_id": 71851,
          "number": 7,
          "name": "wum-gateway",
          "status": "killed",
          "exit_code": 130,
          "started": 1673879103,
          "stopped": 1673879464,
          "version": 3,
          "depends_on": [
            "clone"
          ],
          "image": "docker.io/dualinventivedocker/di-wum-gateway:latest",
          "detached": true
        },
        {
          "id": 494879,
          "step_id": 71851,
          "number": 8,
          "name": "simulators",
          "status": "killed",
          "exit_code": 130,
          "started": 1673879103,
          "stopped": 1673879464,
          "version": 3,
          "depends_on": [
            "clone"
          ],
          "image": "docker.io/dualinventivedocker/di-dinet-simulator:latest",
          "detached": true
        },
        {
          "id": 494880,
          "step_id": 71851,
          "number": 9,
          "name": "sql-files",
          "status": "success",
          "exit_code": 0,
          "started": 1673879103,
          "stopped": 1673879106,
          "version": 3,
          "depends_on": [
            "clone"
          ],
          "image": "docker.io/library/alpine:latest"
        },
        {
          "id": 494881,
          "step_id": 71851,
          "number": 10,
          "name": "database",
          "status": "killed",
          "exit_code": 130,
          "started": 1673879106,
          "stopped": 1673879464,
          "version": 3,
          "depends_on": [
            "sql-files"
          ],
          "image": "docker.io/library/mariadb:latest",
          "detached": true
        },
        {
          "id": 494882,
          "step_id": 71851,
          "number": 11,
          "name": "dependencies",
          "status": "success",
          "exit_code": 0,
          "started": 1673879103,
          "stopped": 1673879106,
          "version": 3,
          "depends_on": [
            "clone"
          ],
          "image": "docker.io/library/golang:latest"
        },
        {
          "id": 494883,
          "step_id": 71851,
          "number": 12,
          "name": "dependency-updates",
          "status": "success",
          "errignore": true,
          "exit_code": 0,
          "started": 1673879106,
          "stopped": 1673879108,
          "version": 3,
          "depends_on": [
            "dependencies"
          ],
          "image": "docker.io/sonatypecommunity/nancy:alpine"
        },
        {
          "id": 494884,
          "step_id": 71851,
          "number": 13,
          "name": "dependency-audit",
          "status": "success",
          "exit_code": 0,
          "started": 1673879106,
          "stopped": 1673879108,
          "version": 3,
          "depends_on": [
            "dependencies"
          ],
          "image": "docker.io/sonatypecommunity/nancy:alpine"
        },
        {
          "id": 494885,
          "step_id": 71851,
          "number": 14,
          "name": "lint",
          "status": "success",
          "exit_code": 0,
          "started": 1673879106,
          "stopped": 1673879111,
          "version": 3,
          "depends_on": [
            "dependencies"
          ],
          "image": "docker.io/golangci/golangci-lint:latest"
        },
        {
          "id": 494886,
          "step_id": 71851,
          "number": 15,
          "name": "test",
          "status": "killed",
          "exit_code": 130,
          "started": 1673879106,
          "stopped": 1673879464,
          "version": 3,
          "depends_on": [
            "dependencies"
          ],
          "image": "docker.io/library/golang:1.19"
        },
        {
          "id": 494887,
          "step_id": 71851,
          "number": 16,
          "name": "notify",
          "status": "skipped",
          "errignore": true,
          "exit_code": 130,
          "started": 1673879464,
          "stopped": 1673879464,
          "version": 2,
          "depends_on": [
            "test"
          ],
          "image": "docker.io/plugins/slack:latest"
        }
      ]
    }
  ]
}