Anyone play with the chelsio 10 gigabit cards yet?

Any other parameters that i can play with?



testing using netperf

Start server on the desitnation ip (10.10.10.2)
#netserver
Then run the client connect on other machine
#netperf -H 10.10.10.2

IT will do a 10 second bandwidth test (adjustable by the -l flag)

or use ftp:
login into ftp, connect to server
put "|dd if=/dev/zero bs=1024 count=10000000" /dev/null


first test:

Recv Send Send
Socket Socket Message Elapsed
Size Size Size Time Throughput
bytes bytes bytes secs. 10^6bits/sec

4194304 4194304 4194304 5.04 2853.85




Enable jumbo frames
#ifconfig eth1 mtu 9000

second test:

[root@cricket tmp]# netperf -H 10.10.10.2 -l 5
TCP STREAM TEST from 0.0.0.0 (0.0.0.0) port 0 AF_INET to 10.10.10.2 (10.10.10.2) port 0 AF_INET
Recv Send Send
Socket Socket Message Elapsed
Size Size Size Time Throughput
bytes bytes bytes secs. 10^6bits/sec

4194304 4194304 4194304 5.01 3639.24


txqueue len = 2000
#ifconfig eth1 txqueuelen 2000

Third test:

[root@cricket tmp]# ifconfig eth0 txqueuelen 2000
[root@cricket tmp]# netperf -H 10.10.10.2 -l 5
TCP STREAM TEST from 0.0.0.0 (0.0.0.0) port 0 AF_INET to 10.10.10.2 (10.10.10.2) port 0 AF_INET
Recv Send Send
Socket Socket Message Elapsed
Size Size Size Time Throughput
bytes bytes bytes secs. 10^6bits/sec

4194304 4194304 4194304 5.01 3784.78




Run the following script:
#!/bin/bash
echo "Optimise TCP parameters"
# enable timestamps, window scaling
echo 1 > /proc/sys/net/ipv4/tcp_timestamps
echo 1 > /proc/sys/net/ipv4/tcp_window_scaling

echo 1 > /proc/sys/net/ipv4/tcp_moderate_rcvbuf

# buffers 4k 4M 60M
# (calc example: 1.0 Gbps/8 * 100ms delay = 125 MByte)
echo 60000000 > /proc/sys/net/core/wmem_max
echo 60000000 > /proc/sys/net/core/rmem_max
echo 4194304 > /proc/sys/net/core/rmem_default
echo 4194304 > /proc/sys/net/core/wmem_default
echo "4096 4194304 60000000" > /proc/sys/net/ipv4/tcp_rmem
echo "4096 4194304 60000000" > /proc/sys/net/ipv4/tcp_wmem

# echo "Optimise ethernet queue 040826pb"
/sbin/ifconfig eth0 txqueuelen 20000
/sbin/ifconfig eth1 txqueuelen 20000

# UKL/Janet Jumbo Frames Oct 05
# /sbin/ifconfig eth1 mtu 4470

#--JW
# sysctl -a | grep net

# enable path mtu discovery
sysctl -w net.ipv4.tcp_mtu_probing=1
sysctl -w net.ipv4.ip_no_pmtu_disc=0

# re-enable SACK since this isn't gigabit LAN but internet:
sysctl -w net.ipv4.tcp_sack=1

# increase backlog ("rxqueuelen"):
sysctl -w net.core.netdev_max_backlog=25000

# discard metrics of old connections:
sysctl -w net.ipv4.tcp_no_metrics_save=1

# select htcp congestion control algorithm:
# (or, another interesting one, bic)
sysctl -w net.ipv4.tcp_congestion_control=htcp

[root@cricket tmp]# netperf -H 10.10.10.2
TCP STREAM TEST from 0.0.0.0 (0.0.0.0) port 0 AF_INET to 10.10.10.2 (10.10.10.2) port 0 AF_INET
Recv Send Send
Socket Socket Message Elapsed
Size Size Size Time Throughput
bytes bytes bytes secs. 10^6bits/sec

4194304 4194304 4194304 10.00 3854.09