无法使用Cassandra Datastax Java驱动程序连接到Cassandra节点之一

Question

Cassandra datastax driver unable to connect to one of the nodes in the data center but still I am able to read and write to database. Datacenter contains two nodes in one rack. I specified one of them as seed node.

Connected to cluster: Test Cluster
Datacenter: datacenter1; Host: /PVT IP1; Rack: rack1
Datacenter: datacenter1; Host: /PUBLIC IP2; Rack: rack1
2017-11-04 02:19:50 WARN  com.datastax.driver.core.HostConnectionPool:184 - Error creating connection to /PVT IP1:9042
com.datastax.driver.core.exceptions.TransportException: [/PVT IP1:9042] Cannot connect
    at com.datastax.driver.core.Connection$1.operationComplete(Connection.java:165)
    at com.datastax.driver.core.Connection$1.operationComplete(Connection.java:148)
    at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:507)
    at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:500)
    at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:479)
    at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:420)
    at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:122)
    at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe$1.run(AbstractNioChannel.java:220)
    at io.netty.util.concurrent.PromiseTask$RunnableAdapter.call(PromiseTask.java:38)
    at io.netty.util.concurrent.ScheduledFutureTask.run(ScheduledFutureTask.java:120)
    at io.netty.util.concurrent.SingleThreadEventExecutor.runAllTasks(SingleThreadEventExecutor.java:399)
    at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:463)
    at io.netty.util.concurrent.SingleThreadEventExecutor$2.run(SingleThreadEventExecutor.java:131)
    at io.netty.util.concurrent.DefaultThreadFactory$DefaultRunnableDecorator.run(DefaultThreadFactory.java:138)
    at java.lang.Thread.run(Thread.java:745)
Caused by: io.netty.channel.ConnectTimeoutException: connection timed out: /PVT IP1:9042
    at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe$1.run(AbstractNioChannel.java:218)
    ... 7 more
2017-11-04 02:19:50 WARN  com.datastax.driver.core.Session:378 - Error creating pool to /PVT IP1:9042
com.datastax.driver.core.exceptions.ConnectionException: [/PVT IP1:9042] Pool was closed during initialization
    at com.datastax.driver.core.HostConnectionPool$2.onSuccess(HostConnectionPool.java:148)
    at com.datastax.driver.core.HostConnectionPool$2.onSuccess(HostConnectionPool.java:134)
    at com.google.common.util.concurrent.Futures$6.run(Futures.java:1773)
    at com.google.common.util.concurrent.MoreExecutors$DirectExecutor.execute(MoreExecutors.java:456)
    at com.google.common.util.concurrent.AbstractFuture.executeListener(AbstractFuture.java:817)
    at com.google.common.util.concurrent.AbstractFuture.complete(AbstractFuture.java:753)
    at com.google.common.util.concurrent.AbstractFuture.set(AbstractFuture.java:613)
    at com.google.common.util.concurrent.CollectionFuture$CollectionFutureRunningState.handleAllCompleted(CollectionFuture.java:76)
    at com.google.common.util.concurrent.AggregateFuture$RunningState.processCompleted(AggregateFuture.java:255)
    at com.google.common.util.concurrent.AggregateFuture$RunningState.decrementCountAndMaybeComplete(AggregateFuture.java:242)
    at com.google.common.util.concurrent.AggregateFuture$RunningState.access$300(AggregateFuture.java:91)
    at com.google.common.util.concurrent.AggregateFuture$RunningState$1.run(AggregateFuture.java:146)
    at com.google.common.util.concurrent.MoreExecutors$DirectExecutor.execute(MoreExecutors.java:456)
    at com.google.common.util.concurrent.AbstractFuture.executeListener(AbstractFuture.java:817)
    at com.google.common.util.concurrent.AbstractFuture.complete(AbstractFuture.java:753)
    at com.google.common.util.concurrent.AbstractFuture.completeWithFuture(AbstractFuture.java:730)
    at com.google.common.util.concurrent.AbstractFuture.setFuture(AbstractFuture.java:666)
    at com.google.common.util.concurrent.Futures$AsyncCatchingFuture.doFallback(Futures.java:826)
    at com.google.common.util.concurrent.Futures$AsyncCatchingFuture.doFallback(Futures.java:813)
    at com.google.common.util.concurrent.Futures$AbstractCatchingFuture.run(Futures.java:789)
    at com.google.common.util.concurrent.MoreExecutors$DirectExecutor.execute(MoreExecutors.java:456)
    at com.google.common.util.concurrent.AbstractFuture.executeListener(AbstractFuture.java:817)
    at com.google.common.util.concurrent.AbstractFuture.complete(AbstractFuture.java:753)
    at com.google.common.util.concurrent.AbstractFuture.completeWithFuture(AbstractFuture.java:730)
    at com.google.common.util.concurrent.AbstractFuture.setFuture(AbstractFuture.java:666)
    at com.google.common.util.concurrent.Futures$AsyncCatchingFuture.doFallback(Futures.java:826)
    at com.google.common.util.concurrent.Futures$AsyncCatchingFuture.doFallback(Futures.java:813)
    at com.google.common.util.concurrent.Futures$AbstractCatchingFuture.run(Futures.java:789)
    at com.google.common.util.concurrent.MoreExecutors$DirectExecutor.execute(MoreExecutors.java:456)
    at com.google.common.util.concurrent.AbstractFuture.executeListener(AbstractFuture.java:817)
    at com.google.common.util.concurrent.AbstractFuture.complete(AbstractFuture.java:753)
    at com.google.common.util.concurrent.AbstractFuture.setException(AbstractFuture.java:634)
    at com.google.common.util.concurrent.Futures$AbstractChainingFuture.run(Futures.java:1405)
    at com.google.common.util.concurrent.MoreExecutors$DirectExecutor.execute(MoreExecutors.java:456)
    at com.google.common.util.concurrent.AbstractFuture.executeListener(AbstractFuture.java:817)
    at com.google.common.util.concurrent.AbstractFuture.complete(AbstractFuture.java:753)
    at com.google.common.util.concurrent.AbstractFuture.setException(AbstractFuture.java:634)
    at com.google.common.util.concurrent.SettableFuture.setException(SettableFuture.java:53)
    at com.datastax.driver.core.Connection$1.operationComplete(Connection.java:165)
    at com.datastax.driver.core.Connection$1.operationComplete(Connection.java:148)
    at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:507)
    at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:500)
    at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:479)
    at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:420)
    at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:122)
    at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe$1.run(AbstractNioChannel.java:220)
    at io.netty.util.concurrent.PromiseTask$RunnableAdapter.call(PromiseTask.java:38)
    at io.netty.util.concurrent.ScheduledFutureTask.run(ScheduledFutureTask.java:120)
    at io.netty.util.concurrent.SingleThreadEventExecutor.runAllTasks(SingleThreadEventExecutor.java:399)
    at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:463)
    at io.netty.util.concurrent.SingleThreadEventExecutor$2.run(SingleThreadEventExecutor.java:131)
    at io.netty.util.concurrent.DefaultThreadFactory$DefaultRunnableDecorator.run(DefaultThreadFactory.java:138)
    at java.lang.Thread.run(Thread.java:745)

docker-compose.yaml file for each node looks like this:

Yaml file Node 1:
version: "3"
services:

  cassandra:
    image: cassandra:3.11
    container_name: cassandra_node
    ports:
      - 7000:7000
      - 9042:9042
    volumes:
      - /home/******/docker/cassandra/cassandra-data:/var/lib/cassandra
    environment:
      CASSANDRA_BROADCAST_ADDRESS: PVT IP1
      CASSANDRA_SEEDS: PVT IP1

For node2:

Yaml File Node 2
version: "3"
services:

  cassandra:
    image: cassandra:3.11
    container_name: cassandra_node
    ports:
      - 7000:7000
      - 9042:9042
    volumes:
      - /home/******/docker/cassandra/cassandra-data:/var/lib/cassandra
    environment:
      CASSANDRA_BROADCAST_ADDRESS: PVT IP2
      CASSANDRA_SEEDS: PVT IP1

Could someone help me in figuring out the issue here. Thanks!!

Answer 1

I think the issue here is with values passed for CASSANDRA_BROADCAST_ADDRESS and CASSANDRA_SEEDS . There is space character in values, hence cassandra driver might be unable to parse.

I have been using below docker-compose.yml for creating cassandra cluster. Please give it a try.

version: '3'

networks:
    cassandra-cluster:
        driver: bridge

volumes:
  data-volume-dc1-n1:
  data-volume-dc1-n2:
  data-volume-dc2-n1:
  data-volume-dc2-n2:

services:
    ###############################################################################################
    # DC1 node 1
    ###############################################################################################
    DC1_N1:
        container_name: DC1_N1
        image: cassandra:latest
        command: bash -c 'if [ -z "$$(ls -A /var/lib/cassandra/)" ] ; then sleep 0; fi && /docker-entrypoint.sh cassandra -f'

        volumes:
            - data-volume-dc1-n1:/var/lib/cassandra

        environment:
            - CASSANDRA_CLUSTER_NAME=cassandra_cluster
            - CASSANDRA_SEEDS=DC1_N1,DC1_N2,DC2_N1,DC2_N2
            - CASSANDRA_ENDPOINT_SNITCH=GossipingPropertyFileSnitch
            - CASSANDRA_DC=DC1
            - CASSANDRA_RACK=rack1

        # Expose ports for cassandra cluster
        expose:
            - 7000
            - 7001
            - 7199
            - 9042
            - 9160

        ports:
            - "9042:9042"

        # Cassandra ulimt recommended settings
        ulimits:
            memlock: -1
            nproc: 32768
            nofile: 100000

        networks:
            - cassandra-cluster

        restart: unless-stopped

    ###############################################################################################
    # DC1 node 2
    ###############################################################################################
    DC1_N2:
        container_name: DC1_N2
        image: cassandra:latest
        command: bash -c 'if [ -z "$$(ls -A /var/lib/cassandra/)" ] ; then sleep 0; fi && /docker-entrypoint.sh cassandra -f'

        volumes:
            - data-volume-dc2-n2:/var/lib/cassandra

        environment:
            - CASSANDRA_CLUSTER_NAME=cassandra_cluster
            - CASSANDRA_SEEDS=DC1_N1,DC1_N2,DC2_N1,DC2_N2
            - CASSANDRA_ENDPOINT_SNITCH=GossipingPropertyFileSnitch
            - CASSANDRA_DC=DC1
            - CASSANDRA_RACK=rack2

        # Expose ports for cassandra cluster
        expose:
            - 7000
            - 7001
            - 7199
            - 9042
            - 9160

        # Cassandra ulimt recommended settings
        ulimits:
            memlock: -1
            nproc: 32768
            nofile: 100000

        networks:
            - cassandra-cluster

        restart: unless-stopped

    ###############################################################################################
    # DC2 node 1
    ###############################################################################################
    DC2_N1:
        container_name: DC2_N1
        image: cassandra:latest
        command: bash -c 'if [ -z "$$(ls -A /var/lib/cassandra/)" ] ; then sleep 0; fi && /docker-entrypoint.sh cassandra -f'

        volumes:
            - data-volume-dc2-n1:/var/lib/cassandra

        environment:
            - CASSANDRA_CLUSTER_NAME=cassandra_cluster
            - CASSANDRA_SEEDS=DC1_N1,DC1_N2,DC2_N1,DC2_N2
            - CASSANDRA_ENDPOINT_SNITCH=GossipingPropertyFileSnitch
            - CASSANDRA_DC=DC2
            - CASSANDRA_RACK=rack1

        # Expose ports for cassandra cluster
        expose:
            - 7000
            - 7001
            - 7199
            - 9042
            - 9160

        # Cassandra ulimt recommended settings
        ulimits:
            memlock: -1
            nproc: 32768
            nofile: 100000

        networks:
            - cassandra-cluster

        restart: unless-stopped