We are using "dse spark-submit" on DSE 4.8.4 to process a Spark/Scala application:
scalaVersion := "2.10.5"
libraryDependencies ++= Seq("org.apache.spark" %% "spark-core" % "1.4.1",
"com.datastax.spark" %% "spark-cassandra-connector" % "1.4.1",
"org.slf4j" % "slf4j-api" % "1.7.12",
"org.slf4j" % "slf4j-simple" % "1.7.12"
)
Since we updated from DSE 4.8.2 to 4.8.4 we are seeing the following error:
WARN 2016-02-11 10:48:23 com.datastax.driver.core.NettyUtil: Found Netty's native epoll transport in the classpath, but epoll is not available. Using NIO instead.
java.lang.UnsatisfiedLinkError: /tmp/libnetty-transport-native-epoll247904978292289355.so: /lib64/libc.so.6: version `GLIBC_2.14' not found (required by /tmp/libnetty-transport-native-epoll247904978292289355.so)
at java.lang.ClassLoader$NativeLibrary.load(Native Method) ~[na:1.8.0_65]
at java.lang.ClassLoader.loadLibrary0(Unknown Source) ~[na:1.8.0_65]
at java.lang.ClassLoader.loadLibrary(Unknown Source) ~[na:1.8.0_65]
at java.lang.Runtime.load0(Unknown Source) ~[na:1.8.0_65]
at java.lang.System.load(Unknown Source) ~[na:1.8.0_65]
at io.netty.util.internal.NativeLibraryLoader.load(NativeLibraryLoader.java:193) ~[netty-all-4.0.33.dse.jar:4.0.33.dse]
at io.netty.channel.epoll.Native.<clinit>(Native.java:48) ~[netty-all-4.0.33.dse.jar:4.0.33.dse]
at io.netty.channel.epoll.Epoll.<clinit>(Epoll.java:32) ~[netty-all-4.0.33.dse.jar:4.0.33.dse]
at java.lang.Class.forName0(Native Method) ~[na:1.8.0_65]
at java.lang.Class.forName(Unknown Source) ~[na:1.8.0_65]
at com.datastax.driver.core.NettyUtil.<clinit>(NettyUtil.java:68) ~[cassandra-driver-core-2.1.7.1.jar:na]
at com.datastax.driver.core.NettyOptions.eventLoopGroup(NettyOptions.java:101) [cassandra-driver-core-2.1.7.1.jar:na]
at com.datastax.driver.core.Connection$Factory.<init>(Connection.java:695) [cassandra-driver-core-2.1.7.1.jar:na]
at com.datastax.driver.core.Cluster$Manager.init(Cluster.java:1286) [cassandra-driver-core-2.1.7.1.jar:na]
at com.datastax.driver.core.Cluster.getMetadata(Cluster.java:339) [cassandra-driver-core-2.1.7.1.jar:na]
at com.datastax.spark.connector.cql.CassandraConnector$.com$datastax$spark$connector$cql$CassandraConnector$$createSession(CassandraConnector.scala:157) [spark-cassandra-connector_2.10-1.4.1.jar:1.4.1]
at com.datastax.spark.connector.cql.CassandraConnector$$anonfun$2.apply(CassandraConnector.scala:150) [spark-cassandra-connector_2.10-1.4.1.jar:1.4.1]
at com.datastax.spark.connector.cql.CassandraConnector$$anonfun$2.apply(CassandraConnector.scala:150) [spark-cassandra-connector_2.10-1.4.1.jar:1.4.1]
at com.datastax.spark.connector.cql.RefCountedCache.createNewValueAndKeys(RefCountedCache.scala:31) [spark-cassandra-connector_2.10-1.4.1.jar:1.4.1]
at com.datastax.spark.connector.cql.RefCountedCache.acquire(RefCountedCache.scala:56) [spark-cassandra-connector_2.10-1.4.1.jar:1.4.1]
at com.datastax.spark.connector.cql.CassandraConnector.openSession(CassandraConnector.scala:81) [spark-cassandra-connector_2.10-1.4.1.jar:1.4.1]
at com.datastax.spark.connector.cql.CassandraConnector.withSessionDo(CassandraConnector.scala:109) [spark-cassandra-connector_2.10-1.4.1.jar:1.4.1]
at com.datastax.spark.connector.cql.CassandraConnector.withClusterDo(CassandraConnector.scala:120) [spark-cassandra-connector_2.10-1.4.1.jar:1.4.1]
at com.datastax.spark.connector.cql.Schema$.fromCassandra(Schema.scala:241) [spark-cassandra-connector_2.10-1.4.1.jar:1.4.1]
at com.datastax.spark.connector.rdd.CassandraTableRowReaderProvider$class.tableDef(CassandraTableRowReaderProvider.scala:51) [spark-cassandra-connector_2.10-1.4.1.jar:1.4.1]
at com.datastax.spark.connector.rdd.CassandraTableScanRDD.tableDef$lzycompute(CassandraTableScanRDD.scala:59) [spark-cassandra-connector_2.10-1.4.1.jar:1.4.1]
at com.datastax.spark.connector.rdd.CassandraTableScanRDD.tableDef(CassandraTableScanRDD.scala:59) [spark-cassandra-connector_2.10-1.4.1.jar:1.4.1]
at com.datastax.spark.connector.rdd.CassandraTableRowReaderProvider$class.verify(CassandraTableRowReaderProvider.scala:150) [spark-cassandra-connector_2.10-1.4.1.jar:1.4.1]
at com.datastax.spark.connector.rdd.CassandraTableScanRDD.verify(CassandraTableScanRDD.scala:59) [spark-cassandra-connector_2.10-1.4.1.jar:1.4.1]
at com.datastax.spark.connector.rdd.CassandraTableScanRDD.getPartitions(CassandraTableScanRDD.scala:143) [spark-cassandra-connector_2.10-1.4.1.jar:1.4.1]
at org.apache.spark.rdd.RDD$$anonfun$partitions$2.apply(RDD.scala:219) [spark-core_2.10-1.4.2.2.jar:1.4.2.2]
at org.apache.spark.rdd.RDD$$anonfun$partitions$2.apply(RDD.scala:217) [spark-core_2.10-1.4.2.2.jar:1.4.2.2]
at scala.Option.getOrElse(Option.scala:120) [scala-library-2.10.5.jar:na]
at org.apache.spark.rdd.RDD.partitions(RDD.scala:217) [spark-core_2.10-1.4.2.2.jar:1.4.2.2]
at org.apache.spark.rdd.MapPartitionsRDD.getPartitions(MapPartitionsRDD.scala:32) [spark-core_2.10-1.4.2.2.jar:1.4.2.2]
at org.apache.spark.rdd.RDD$$anonfun$partitions$2.apply(RDD.scala:219) [spark-core_2.10-1.4.2.2.jar:1.4.2.2]
at org.apache.spark.rdd.RDD$$anonfun$partitions$2.apply(RDD.scala:217) [spark-core_2.10-1.4.2.2.jar:1.4.2.2]
at scala.Option.getOrElse(Option.scala:120) [scala-library-2.10.5.jar:na]
at org.apache.spark.rdd.RDD.partitions(RDD.scala:217) [spark-core_2.10-1.4.2.2.jar:1.4.2.2]
at org.apache.spark.rdd.MapPartitionsRDD.getPartitions(MapPartitionsRDD.scala:32) [spark-core_2.10-1.4.2.2.jar:1.4.2.2]
at org.apache.spark.rdd.RDD$$anonfun$partitions$2.apply(RDD.scala:219) [spark-core_2.10-1.4.2.2.jar:1.4.2.2]
at org.apache.spark.rdd.RDD$$anonfun$partitions$2.apply(RDD.scala:217) [spark-core_2.10-1.4.2.2.jar:1.4.2.2]
at scala.Option.getOrElse(Option.scala:120) [scala-library-2.10.5.jar:na]
at org.apache.spark.rdd.RDD.partitions(RDD.scala:217) [spark-core_2.10-1.4.2.2.jar:1.4.2.2]
at org.apache.spark.rdd.MapPartitionsRDD.getPartitions(MapPartitionsRDD.scala:32) [spark-core_2.10-1.4.2.2.jar:1.4.2.2]
at org.apache.spark.rdd.RDD$$anonfun$partitions$2.apply(RDD.scala:219) [spark-core_2.10-1.4.2.2.jar:1.4.2.2]
at org.apache.spark.rdd.RDD$$anonfun$partitions$2.apply(RDD.scala:217) [spark-core_2.10-1.4.2.2.jar:1.4.2.2]
at scala.Option.getOrElse(Option.scala:120) [scala-library-2.10.5.jar:na]
at org.apache.spark.rdd.RDD.partitions(RDD.scala:217) [spark-core_2.10-1.4.2.2.jar:1.4.2.2]
at org.apache.spark.SparkContext.runJob(SparkContext.scala:1806) [spark-core_2.10-1.4.2.2.jar:1.4.2.2]
at org.apache.spark.rdd.RDD.count(RDD.scala:1099) [spark-core_2.10-1.4.2.2.jar:1.4.2.2]
at com.treefin.stats.UserStatistics$.main(UserStatistics.scala:24) [stats_2.10-1.0.jar:1.0]
at com.treefin.stats.UserStatistics.main(UserStatistics.scala) [stats_2.10-1.0.jar:1.0]
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[na:1.8.0_65]
at sun.reflect.NativeMethodAccessorImpl.invoke(Unknown Source) ~[na:1.8.0_65]
at sun.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source) ~[na:1.8.0_65]
at java.lang.reflect.Method.invoke(Unknown Source) ~[na:1.8.0_65]
at org.apache.spark.deploy.SparkSubmit$.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:665) [spark-core_2.10-1.4.2.2.jar:4.8.4]
at org.apache.spark.deploy.SparkSubmit$.doRunMain$1(SparkSubmit.scala:170) [spark-core_2.10-1.4.2.2.jar:4.8.4]
at org.apache.spark.deploy.SparkSubmit$.submit(SparkSubmit.scala:193) [spark-core_2.10-1.4.2.2.jar:4.8.4]
at org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:112) [spark-core_2.10-1.4.2.2.jar:4.8.4]
at org.apache.spark.deploy.DseSparkSubmitBootstrapper$.main(DseSparkSubmitBootstrapper.scala:45) [dse-spark-4.8.4.jar:4.8.4]
at org.apache.spark.deploy.DseSparkSubmitBootstrapper.main(DseSparkSubmitBootstrapper.scala) [dse-spark-4.8.4.jar:4.8.4]
Our systems are running on CentOS 6.5, which use GLIBC_2.12. It should still be supported though: http://www.datastax.com/products/datastax-enterprise#DSE-Supported-Platforms-Table
Is the any way to fix this and get back to using epoll? Thanks!
dse spark-submit --driver-java-options "-Dcom.datastax.driver.FORCE_NIO=true"
helps, but does not solve the problem with the GLIBC version