python rdma “Hello World”

Question

This program is my first attempt at Infiniband programming. I am using this library:

https://github.com/jgunthorpe/python-rdma

I am probably doing very funny things wrong. I am just trying to send a sequence of bytes "Hello, World" to a multicast address.

import sys
import errno
import unittest
import mmap

import rdma
import rdma.vmad
import rdma.IBA as IBA
import rdma.ibverbs as ibv
import rdma.satransactor
import rdma.path
import rdma.vtools

from rdma.tools import clock_monotonic
from pyqtgraph.Qt import QtCore, QtGui


class umad_self_test():
    umad = None;
    tid = 0;

    def __init__(self):
        self.umad = None
        self.tid = 0

        self.setUp()

    def setUp(self):
        self.end_port = rdma.get_end_port();
        self.ctx = rdma.get_verbs(self.end_port);

        #print self.end_port, self.ctx

    def tearDown(self):
        self.ctx.close();
        self.ctx = None;    

    def test_basic(self):
        print self.ctx.query_port();
        print self.ctx.query_device();

        pd = self.ctx.pd();
        print pd,repr(pd)

        cq = self.ctx.cq(100);
        print cq,repr(cq)

        try:
            cq.resize(200);
        except rdma.SysError as e:
            if e.errno != errno.ENOSYS:
                raise

        comp = self.ctx.comp_channel()
        print comp,repr(comp)

        qp = pd.qp(ibv.IBV_QPT_UD, 100, cq, 100, cq)
        print qp,repr(qp)
        print qp.query(0xFFFF)
        mpath = rdma.path.IBPath(self.ctx. end_port, DLID=0xC000,
                                 DGID=IBA.GID("ff02::1"))
        qp.attach_mcast(mpath)
        qp.detach_mcast(mpath)

        buf = mmap.mmap(-1,4096);

        mr = pd.mr(buf,ibv.IBV_ACCESS_LOCAL_WRITE|ibv.IBV_ACCESS_REMOTE_WRITE);
        print mr,repr(mr)
        print "MR",mr.addr,mr.length,mr.lkey,mr.rkey

        ah = pd.ah(self.end_port.sa_path);
        print ah,repr(ah)

        srq = pd.srq();
        print srq,repr(srq)
        print srq.query();
        srq.modify(100);

        pool = rdma.vtools.BufferPool(pd,count=100,size=1024);
        pool.post_recvs(qp,50);

        buf_idx = pool.pop();
        pool.copy_to("Hello, World!", buf_idx);
        qp.post_send(pool.make_send_wr(buf_idx,pool.size, mpath));

if __name__ == '__main__':   
    if (sys.flags.interactive != 1) or not hasattr(QtCore, 'PYQT_VERSION'):
        umst = umad_self_test()
        umst.test_basic()
        umst.tearDown()

This is the output I get:

[idf@node2 python]$ python myverbs.py 
port_attr(state=4, max_mtu=5, active_mtu=5, gid_tbl_len=128L, port_cap_flags=39389290L, max_msg_sz=1073741824L, bad_pkey_cntr=0L, qkey_viol_cntr=0L, pkey_tbl_len=128L, lid=1L, sm_lid=1L, lmc=0L, max_vl_num=3L, sm_sl=0L, subnet_timeout=18L, init_type_reply=0L, active_width=2L, active_speed=4L, phys_state=5L)
device_attr(fw_ver='2.7.200', node_guid=GUID('0025:90ff:ff1a:0070'), sys_image_guid=GUID('0025:90ff:ff1a:0073'), max_mr_size=18446744073709551615L, page_size_cap=4294966784L, vendor_id=713L, vendor_part_id=26428L, hw_ver=176L, max_qp=163704L, max_qp_wr=16351L, device_cap_flags=8166518L, max_sge=32L, max_sge_rd=0L, max_cq=65408L, max_cqe=4194303L, max_mr=524272L, max_pd=32764L, max_qp_rd_atom=16L, max_ee_rd_atom=0L, max_res_rd_atom=2619264L, max_qp_init_rd_atom=128L, max_ee_init_rd_atom=0L, atomic_cap=1, max_ee=0L, max_rdd=0L, max_mw=0L, max_raw_ipv6_qp=0L, max_raw_ethy_qp=0L, max_mcast_grp=8192L, max_mcast_qp_attach=248L, max_total_mcast_qp_attach=2031616L, max_ah=0L, max_fmr=0L, max_map_per_fmr=8191L, max_srq=65472L, max_srq_wr=16383L, max_srq_sge=31L, max_pkeys=128L, local_ca_ack_delay=15L, phys_port_cnt=1L)
pd:0:mlx4_0 PD(Context('mlx4_0',fd=3),0x0)
cq:0:mlx4_0 CQ(Context('mlx4_0',fd=3),0x0)
comp_channel:5:mlx4_0 CompChannel(Context('mlx4_0',fd=3),5)
qp:193:pd:0:mlx4_0 QP(PD(Context('mlx4_0',fd=3),0x0),0x0,193,qp_type=IBV_QPT_UD(4))
(<rdma.tools.qp_attr object at 0x7f0d29aa24d0>, <rdma.tools.qp_init_attr object at 0x7f0d29aa2550>)
mr:0:pd:0:mlx4_0 MR(PD(Context('mlx4_0',fd=3),0x0),0x0,0x7f0d34377000,4096,lkey=0xb0042102,rkey=0xb0042102)
MR 139694687350784 4096 2953060610 2953060610
ah:6E616269 AH(0x6e616269)
srq:0:pd:0:mlx4_0 SRQ(PD(Context('mlx4_0',fd=3),0x0),0x0)
srq_attr(max_wr=127L, max_sge=1L, srq_limit=0L)
Traceback (most recent call last):
  File "myverbs.py", line 97, in <module>
    umst.test_basic()
  File "myverbs.py", line 92, in test_basic
    qp.post_send(pool.make_send_wr(buf_idx,pool.size, mpath));
  File "ibverbs.pyx", line 1461, in rdma.ibverbs.QP.post_send
TypeError: an integer is required
[idf@node2 python]$ 

Answer 1

I believe the direct problem is that you are not initializing the dqpn (destination QP number) of the path object. According to the IBPath docstring a destination QP number is required when using unconnected QPs.