运维之路

comet模型的主机维持长连接，单台连接接数可以达到百万（现网目前用的最高单台可达到60万左右），在统计网络连接状态时会netstat会长期不响应，而SS命令统计相对较快，不过输出不够美观 。本篇主要总结下comet模型下的连接数常见统计方法。tcp连接主机是通过读取/proc/net/tcp文件进行的统计。具体每项值的意思见下图（点击可查看大图）

一、C++ 代码实现连接数统计

[root@361way netstat]# cat ss++.cc
// code from www.361way.com
#include<iostream>
#include<fstream>
#include<cstdlib>
#include<sstream>
using namespace std;
//struct: Property of a tcp connection
struct ConnectionProperty
{
    string local_addr;
    string remote_addr;
    string state;
};
//Change hexadecimal number to decimal number
int HexToInt(char h)
{
    if(h >= '0' && h <= '9')
    {
        return h - '0';
    }
    else if(h >= 'A' && h <= 'F')
    {
        return h - 'A' + 10;
    }
    else
    {
        cerr << "Error: Illegal Hex Number!" << endl;
        return -1;
    }
}
//Get Ip address and port number from string XXXXXXXX:XXXX
string GetIpAddress(char* str)
{
    int a, b, c, d, e;
    a = HexToInt(str[0]) * 16 + HexToInt(str[1]);
    b = HexToInt(str[2]) * 16 + HexToInt(str[3]);
    c = HexToInt(str[4]) * 16 + HexToInt(str[5]);
    d = HexToInt(str[6]) * 16 + HexToInt(str[7]);
    e = HexToInt(str[9]) * 16 * 16 * 16 +
        HexToInt(str[10]) * 16 * 16 +
        HexToInt(str[11]) * 16 +
        HexToInt(str[12]);
    //change int to string
    string sa, sb, sc, sd, se;
    ostringstream oss;
    oss << a;
    sa = oss.str();
    oss.str(""); //clear the content in oss
    oss << b;
    sb = oss.str();
    oss.str("");
    oss << c;
    sc = oss.str();
    oss.str("");
    oss << d;
    sd = oss.str();
    oss.str("");
    oss << e;
    se = oss.str();
    oss.str("");
    //return by order: d.c.b.a:e
    return sd + '.' + sc + '.' + sb + '.' + sa + ':' + se;
}
//Get tcp connection state
string GetConnectionState(char* str)
{
    if(str[0] == '0' && str[1] == '0') return "ERROR_STATUS";
    if(str[0] == '0' && str[1] == '1') return "TCP_ESTABLISHED";
    if(str[0] == '0' && str[1] == '2') return "TCP_SYN_SENT";
    if(str[0] == '0' && str[1] == '3') return "TCP_SYN_RECV";
    if(str[0] == '0' && str[1] == '4') return "TCP_FIN_WAIT1";
    if(str[0] == '0' && str[1] == '5') return "TCP_FIN_WAIT2";
    if(str[0] == '0' && str[1] == '6') return "TCP_TIME_WAIT";
    if(str[0] == '0' && str[1] == '7') return "TCP_CLOSE";
    if(str[0] == '0' && str[1] == '8') return "TCP_CLOSE_WAIT";
    if(str[0] == '0' && str[1] == '9') return "TCP_LAST_ACK";
    if(str[0] == '0' && str[1] == 'A') return "TCP_LISTEN";
    if(str[0] == '0' && str[1] == 'B') return "TCP_CLOSING";
    return "UNKNOWN_STATE";
}
int main()
{
    //read from file /proc/net/tcp
    ifstream infile("/proc/net/tcp", ios :: in);
    if(!infile)
    {
        cerr << "open error!" << endl;
        exit(1);
    }
    string s;
    char s_local_addr[20], s_remote_addr[20], s_state[20];
    getline(infile, s); //title: every column's name
    while(getline(infile, s))
    {
        sscanf(s.c_str(), "%*s%s%s%s", s_local_addr, s_remote_addr, s_state);
        //printf("%s\t%s\t%s\n", s_local_addr, s_remote_addr, s_state);
        string ip_local = GetIpAddress(s_local_addr);
        string ip_remote = GetIpAddress(s_remote_addr);
        string conn_state = GetConnectionState(s_state);
        cout << ip_local << "\t" << ip_remote << "\t" << conn_state << endl;
    }
    return 0;
}

通过如下代码编译生成可执行文件：

g++ -o netinfo ss++.cc 

执行输出如下：

[root@comethost ~]# ./netinfo
115.28.174.118:5224       121.31.251.8:13351      TCP_ESTABLISHED
115.28.174.118:5229       223.104.23.84:62815     TCP_ESTABLISHED
115.28.174.118:5225       117.136.79.70:45116     TCP_ESTABLISHED
115.28.174.118:5224       112.17.243.208:33236    TCP_ESTABLISHED
115.28.174.118:5224       27.204.14.135:44976     TCP_ESTABLISHED
115.28.174.118:5228       58.18.233.29:60135      TCP_ESTABLISHED
115.28.174.118:5226       112.17.246.84:26055     TCP_ESTABLISHED

二、python实现tcp连接统计

#!/usr/bin/env python
# code from www.361way.com
# coding=utf-8
import pwd
import os
import re
import glob
PROC_TCP = "/proc/net/tcp"
STATE = {
        '01':'ESTABLISHED',
        '02':'SYN_SENT',
        '03':'SYN_RECV',
        '04':'FIN_WAIT1',
        '05':'FIN_WAIT2',
        '06':'TIME_WAIT',
        '07':'CLOSE',
        '08':'CLOSE_WAIT',
        '09':'LAST_ACK',
        '0A':'LISTEN',
        '0B':'CLOSING'
        }
def _load():
    ''' Read the table of tcp connections & remove header  '''
    with open(PROC_TCP,'r') as f:
        content = f.readlines()
        content.pop(0)
    return content
def _hex2dec(s):
    return str(int(s,16))
def _ip(s):
    ip = [(_hex2dec(s[6:8])),(_hex2dec(s[4:6])),(_hex2dec(s[2:4])),(_hex2dec(s[0:2]))]
    return '.'.join(ip)
def _remove_empty(array):
    return [x for x in array if x !='']
def _convert_ip_port(array):
    host,port = array.split(':')
    return _ip(host),_hex2dec(port)
def netstat():
    '''
    Function to return a list with status of tcp connections at linux systems
    To get pid of all network process running on system, you must run this script
    as superuser
    '''
    content=_load()
    result = []
    for line in content:
        line_array = _remove_empty(line.split(' '))     # Split lines and remove empty spaces.
        l_host,l_port = _convert_ip_port(line_array[1]) # Convert ipaddress and port from hex to decimal.
        r_host,r_port = _convert_ip_port(line_array[2])
        state = STATE[line_array[3]]
        nline = [ l_host+':'+l_port, r_host+':'+r_port, state ]
        result.append(nline)
    return result
if __name__ == '__main__':
    for conn in netstat():
        print conn

三、ss统计

默认安装完iproute2包后，里面包含有ss工具，不过想要查看ss的源码可以到https://github.com/shemminger/iproute2/blob/master/misc/ss.c 查看 。

四、统计速率统计

这里只找了一台连接数在20多万的comet服务器进行的测试，从执行效果来看：

ss -nt执行速度最快

c++版的netinfo与ss接近（稍慢于c语言版的ss）

python版的比ss与netinfo慢五到六倍

netstat没做测试，根据之前的了解，估计与python版的接近或者差于python版的

所以后续现网有简单的连接数统计需求，会优先选择c++版的ss（输出美观），而且可以在此基础上简单定制 。

参考页面：

c++版的ss参考oschina

python版的ss参考 voorloopnul

相关代码已上传至我的github ：https://github.com/361way/python/tree/master/netstat