Recently we have upgraded our Linux kernel of web server from 3.12.57-60.35 to 3.12.62-60.62.1.
After the upgrade, all docker containers can't be well started.
Each time when I try to start my container, a brand new runc:[2:INIT]
process will be created and hang there, which takes 100% cpu load.
Meanwhile, the container failed to be started.
What does runc:[2:INIT] mean?
How to fix it?
Some background information
> docker -v
Docker version 17.04.0-ce, build 78d1802
> uname -r
3.12.74-60.64.96-default
> containerd -v
containerd version 0.2.3
> docker-runc -v
runc version spec: 1.0.0-rc2-dev
Docker info
docker info
Containers: 1
Running: 0
Paused: 0
Stopped: 1
Images: 28
Server Version: 17.04.0-ce
Storage Driver: overlay
Backing Filesystem: xfs
Supports d_type: true
Logging Driver: json-file
Cgroup Driver: cgroupfs
Plugins:
Volume: local
Network: bridge host macvlan null overlay
Swarm: inactive
Runtimes: oci runc
Default Runtime: runc
Init Binary:
containerd version: (expected: 422e31ce907fd9c3833a38d7b8fdd023e5a76e73)
runc version: N/A (expected: 9c2d8d184e5da67c95d601382adf14862e4f2228)
init version: N/A (expected: 949e6facb77383876aeff8a6944dde66b3089574)
Security Options:
apparmor
seccomp
Profile: default
Kernel Version: 3.12.74-60.64.96-default
Operating System: SUSE Linux Enterprise Server 12 SP1
OSType: linux
Architecture: x86_64
CPUs: 8
Total Memory: 15.19GiB
Name: pc23jamdevops01
ID: 5V4L:E4FT:6FYQ:JCNJ:GPKC:UZBW:O33D:PMKF:Y5X5:7T2X:GFG6
Docker Root Dir: /docker
Debug Mode (client): false
Debug Mode (server): false
Registry: https://index.docker.io/v1/
Experimental: false
Insecure Registries:
xxxxxxx(it's masked by me.)
Live Restore Enabled: false
WARNING: No swap limit support
WARNING: No kernel memory limit support
Trace
- enabled the debug mode of docker
- kill -SIGUSR1 $(pidof dockerd)
the trace shows
goroutine 795 [select, 29 minutes]:
github.com/docker/docker/vendor/google.golang.org/grpc/transport.(*Stream).waitOnHeader(0xc420774ff0, 0x10, 0xc4209caa80)
/home/abuild/rpmbuild/BUILD/docker-18.09.0_ce/components/engine/.gopath/src/github.com/docker/docker/vendor/google.golang.org/grpc/transport/transport.go:222 +0x101
github.com/docker/docker/vendor/google.golang.org/grpc/transport.(*Stream).RecvCompress(0xc420774ff0, 0x7f24a3df3ab0, 0xc4209cab40)
/home/abuild/rpmbuild/BUILD/docker-18.09.0_ce/components/engine/.gopath/src/github.com/docker/docker/vendor/google.golang.org/grpc/transport/transport.go:233 +0x2d
github.com/docker/docker/vendor/google.golang.org/grpc.(*csAttempt).recvMsg(0xc421c98dd0, 0x7f24a3c5e920, 0xc420b80840, 0x0, 0x0)
/home/abuild/rpmbuild/BUILD/docker-18.09.0_ce/components/engine/.gopath/src/github.com/docker/docker/vendor/google.golang.org/grpc/stream.go:515 +0x63b
github.com/docker/docker/vendor/google.golang.org/grpc.(*clientStream).RecvMsg(0xc421954280, 0x7f24a3c5e920, 0xc420b80840, 0x0, 0x0)
/home/abuild/rpmbuild/BUILD/docker-18.09.0_ce/components/engine/.gopath/src/github.com/docker/docker/vendor/google.golang.org/grpc/stream.go:395 +0x45
github.com/docker/docker/vendor/google.golang.org/grpc.invoke(0x7f24a3e29980, 0xc421534090, 0x7f24a33f206d, 0x2a, 0x7f24a3c63380, 0xc42090e0e0, 0x7f24a3c5e920, 0xc420b80840, 0xc4209a0000, 0xc420978420, ...)
/home/abuild/rpmbuild/BUILD/docker-18.09.0_ce/components/engine/.gopath/src/github.com/docker/docker/vendor/google.golang.org/grpc/call.go:83 +0x185
github.com/docker/docker/vendor/github.com/containerd/containerd.namespaceInterceptor.unary(0x7f24a339c737, 0x4, 0x7f24a3e29900, 0xc42003e028, 0x7f24a33f206d, 0x2a, 0x7f24a3c63380, 0xc42090e0e0, 0x7f24a3c5e920, 0xc420b80840, ...)
/home/abuild/rpmbuild/BUILD/docker-18.09.0_ce/components/engine/.gopath/src/github.com/docker/docker/vendor/github.com/containerd/containerd/grpc.go:35 +0xf6
github.com/docker/docker/vendor/github.com/containerd/containerd.(namespaceInterceptor).(github.com/docker/docker/vendor/github.com/containerd/containerd.unary)-fm(0x7f24a3e29900, 0xc42003e028, 0x7f24a33f206d, 0x2a, 0x7f24a3c63380, 0xc42090e0e0, 0x7f24a3c5e920, 0xc420b80840, 0xc4209a0000, 0x7f24a3df3b20, ...)
/home/abuild/rpmbuild/BUILD/docker-18.09.0_ce/components/engine/.gopath/src/github.com/docker/docker/vendor/github.com/containerd/containerd/grpc.go:51 +0xf6
github.com/docker/docker/vendor/google.golang.org/grpc.(*ClientConn).Invoke(0xc4209a0000, 0x7f24a3e29900, 0xc42003e028, 0x7f24a33f206d, 0x2a, 0x7f24a3c63380, 0xc42090e0e0, 0x7f24a3c5e920, 0xc420b80840, 0x0, ...)
/home/abuild/rpmbuild/BUILD/docker-18.09.0_ce/components/engine/.gopath/src/github.com/docker/docker/vendor/google.golang.org/grpc/call.go:35 +0x10b
github.com/docker/docker/vendor/google.golang.org/grpc.Invoke(0x7f24a3e29900, 0xc42003e028, 0x7f24a33f206d, 0x2a, 0x7f24a3c63380, 0xc42090e0e0, 0x7f24a3c5e920, 0xc420b80840, 0xc4209a0000, 0x0, ...)
/home/abuild/rpmbuild/BUILD/docker-18.09.0_ce/components/engine/.gopath/src/github.com/docker/docker/vendor/google.golang.org/grpc/call.go:60 +0xc3
github.com/docker/docker/vendor/github.com/containerd/containerd/api/services/tasks/v1.(*tasksClient).Create(0xc420a5c5b0, 0x7f24a3e29900, 0xc42003e028, 0xc42090e0e0, 0x0, 0x0, 0x0, 0x0, 0xc420ec4a40, 0x1e)
/home/abuild/rpmbuild/BUILD/docker-18.09.0_ce/components/engine/.gopath/src/github.com/docker/docker/vendor/github.com/containerd/containerd/api/services/tasks/v1/tasks.pb.go:412 +0xd4
github.com/docker/docker/vendor/github.com/containerd/containerd.(*container).NewTask(0xc42124f7e0, 0x7f24a3e29900, 0xc42003e028, 0xc4223f5360, 0xc42000e9c8, 0x1, 0x1, 0x0, 0x0, 0x0, ...)
/home/abuild/rpmbuild/BUILD/docker-18.09.0_ce/components/engine/.gopath/src/github.com/docker/docker/vendor/github.com/containerd/containerd/container.go:250 +0x70a
github.com/docker/docker/libcontainerd.(*client).Start(0xc420a125b0, 0x7f24a3e29900, 0xc42003e028, 0xc4215fce40, 0x40, 0x0, 0x0, 0xc4210bd000, 0xc4207fd6f0, 0x0, ...)
/home/abuild/rpmbuild/BUILD/docker-18.09.0_ce/components/engine/.gopath/src/github.com/docker/docker/libcontainerd/client_daemon.go:289 +0x446
github.com/docker/docker/daemon.(*Daemon).containerStart(0xc420948000, 0xc421a458c0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0)
/home/abuild/rpmbuild/BUILD/docker-18.09.0_ce/components/engine/.gopath/src/github.com/docker/docker/daemon/start.go:185 +0x55f
github.com/docker/docker/daemon.(*Daemon).ContainerStart(0xc420948000, 0xc420731517, 0x40, 0x0, 0x0, 0x0, 0x0, 0x0, 0x7f24a3a00640, 0xc421dd7601)
/home/abuild/rpmbuild/BUILD/docker-18.09.0_ce/components/engine/.gopath/src/github.com/docker/docker/daemon/start.go:95 +0x1c4
github.com/docker/docker/api/server/router/container.(*containerRouter).postContainersStart(0xc420ea83c0, 0x7f24a3e29980, 0xc4211cf290, 0x7f24a3e27d80, 0xc420854fc0, 0xc420690400, 0xc4211cf1d0, 0x7f24a339d6d5, 0x5)
/home/abuild/rpmbuild/BUILD/docker-18.09.0_ce/components/engine/.gopath/src/github.com/docker/docker/api/server/router/container/container_routes.go:203 +0x26a
github.com/docker/docker/api/server/router/container.(*containerRouter).(github.com/docker/docker/api/server/router/container.postContainersStart)-fm(0x7f24a3e29980, 0xc4211cf290, 0x7f24a3e27d80, 0xc420854fc0, 0xc420690400, 0xc4211cf1d0, 0x7f24a1ee64cc, 0x7f24a3c663a0)
/home/abuild/rpmbuild/BUILD/docker-18.09.0_ce/components/engine/.gopath/src/github.com/docker/docker/api/server/router/container/container.go:52 +0x6b
github.com/docker/docker/api/server/middleware.ExperimentalMiddleware.WrapHandler.func1(0x7f24a3e29980, 0xc4211cf290, 0x7f24a3e27d80, 0xc420854fc0, 0xc420690400, 0xc4211cf1d0, 0x7f24a3e29980, 0xc4211cf290)
/home/abuild/rpmbuild/BUILD/docker-18.09.0_ce/components/engine/.gopath/src/github.com/docker/docker/api/server/middleware/experimental.go:26 +0xda
github.com/docker/docker/api/server/middleware.VersionMiddleware.WrapHandler.func1(0x7f24a3e29980, 0xc4211cf230, 0x7f24a3e27d80, 0xc420854fc0, 0xc420690400, 0xc4211cf1d0, 0x18, 0xc421dd78f8)
/home/abuild/rpmbuild/BUILD/docker-18.09.0_ce/components/engine/.gopath/src/github.com/docker/docker/api/server/middleware/version.go:62 +0x401
github.com/docker/docker/pkg/authorization.(*Middleware).WrapHandler.func1(0x7f24a3e29980, 0xc4211cf230, 0x7f24a3e27d80, 0xc420854fc0, 0xc420690400, 0xc4211cf1d0, 0x7f24a1daefd7, 0x7f24a3571380)
/home/abuild/rpmbuild/BUILD/docker-18.09.0_ce/components/engine/.gopath/src/github.com/docker/docker/pkg/authorization/middleware.go:59 +0x7ab
github.com/docker/docker/api/server/middleware.DebugRequestMiddleware.func1(0x7f24a3e29980, 0xc4211cf230, 0x7f24a3e27d80, 0xc420854fc0, 0xc420690400, 0xc4211cf1d0, 0x7f24a3e29980, 0xc4211cf230)
/home/abuild/rpmbuild/BUILD/docker-18.09.0_ce/components/engine/.gopath/src/github.com/docker/docker/api/server/middleware/debug.go:25 +0x8db
github.com/docker/docker/api/server.(*Server).makeHTTPHandler.func1(0x7f24a3e27d80, 0xc420854fc0, 0xc420690400)
/home/abuild/rpmbuild/BUILD/docker-18.09.0_ce/components/engine/.gopath/src/github.com/docker/docker/api/server/server.go:141 +0x19a
net/http.HandlerFunc.ServeHTTP(0xc4214f36a0, 0x7f24a3e27d80, 0xc420854fc0, 0xc420690400)
/usr/lib64/go/1.10/src/net/http/server.go:1947 +0x46
github.com/docker/docker/vendor/github.com/gorilla/mux.(*Router).ServeHTTP(0xc421ceba40, 0x7f24a3e27d80, 0xc420854fc0, 0xc420690400)
/home/abuild/rpmbuild/BUILD/docker-18.09.0_ce/components/engine/.gopath/src/github.com/docker/docker/vendor/github.com/gorilla/mux/mux.go:103 +0x228
github.com/docker/docker/api/server.(*routerSwapper).ServeHTTP(0xc421a77360, 0x7f24a3e27d80, 0xc420854fc0, 0xc420690400)
/home/abuild/rpmbuild/BUILD/docker-18.09.0_ce/components/engine/.gopath/src/github.com/docker/docker/api/server/router_swapper.go:29 +0x72
net/http.serverHandler.ServeHTTP(0xc42020c4e0, 0x7f24a3e27d80, 0xc420854fc0, 0xc420690400)
/usr/lib64/go/1.10/src/net/http/server.go:2697 +0xbe
net/http.(*conn).serve(0xc420fe2320, 0x7f24a3e298c0, 0xc421196800)
/usr/lib64/go/1.10/src/net/http/server.go:1830 +0x653
created by net/http.(*Server).Serve
/usr/lib64/go/1.10/src/net/http/server.go:2798 +0x27d