LVS 内核实现分析（2）

5300阅读　0评论2017-01-06　frankzfz
分类：云计算

3 重要数据结构

  这里重点介绍其中用到的三个数据结构，

3.1 ip_vs_conn 

  1. /*
  2.  *    IP_VS structure allocated for each dynamically scheduled connection每一个动态的连接会
  3.  有一个ip_vs_conn结构，它包含客户端，IPVS，RealServer的地址和端口信息*/
  4. struct ip_vs_conn {
  5.     struct list_head c_list; /* hashed list heads */

  7.     /* Protocol, addresses and port numbers */
  8.     u16 af;        /* address family */
  9.     union nf_inet_addr caddr; /* client address */
  10.     union nf_inet_addr vaddr; /* virtual address */
  11.     union nf_inet_addr daddr; /* destination address */
  12.     __be16 cport;             /*客户端的端口*/
  13.     __be16 vport;            /*IPVS的端口*/
  14.     __be16 dport;            /*RS的端口*/
  15.     __u16 protocol; /* Which protocol (TCP/UDP)协议号 */

  17.     /* counter and timer ip_vs_conn对象的使用计数。其初值为1，__ip_vs_conn_in_get/__ip_vs_conn_put成对调用 */
  18.     atomic_t        refcnt;        /* reference count */
  19.     struct timer_list    timer;        /* Expiration timerip_vs_conn对象的生存期，当timer到期时，对象被销毁 */
  20.     volatile unsigned long    timeout;    /* timeoutip_vs_conn对象动态的超时时间，每当对象操作完毕，timeout值用来更新timer，以延长对象的生存期。timeout受连接状态等的影响 */

  22.     /* Flags and state transition */
  23.     spinlock_t lock; /* lock for state transition */
  24.     volatile __u16 flags; /* status flags */
  25.     volatile __u16 state; /* state info */
  26.     volatile __u16 old_state; /* old state, to be used for
  27.                          * state transition triggerd
  28.                          * synchronization
  29.                          */

  31.     /* Control members */
  32.     struct ip_vs_conn *control; /* Master control connection */
  33.     atomic_t n_control; /* Number of controlled ones */
  34.     struct ip_vs_dest *dest; /* real server 指向此连接对象对应的ip_vs_dest对象*/
  35.     atomic_t in_pkts; /* incoming packet counter */

  37.     /* packet transmitter for different forwarding methods. If it
  38.      mangles the packet, it must return NF_DROP or better NF_STOLEN,
  39.      otherwise this must be changed to a sk_buff **.
  40.      */
  41.     int (*packet_xmit)(struct sk_buff *skb, struct ip_vs_conn *cp,
  42.              struct ip_vs_protocol *pp); /*不同的发包函数，三种模式对应三个不同的发包函数*/

  44.     /* Note: we can group the following members into a structure,
  45.      in order to save more space, and the following members are
  46.      only used in VS/NAT anyway */
  47.     struct ip_vs_app *app; /* bound ip_vs_app object */
  48.     void *app_data; /* Application private data */
  49.     struct ip_vs_seq in_seq; /* incoming seq. struct */
  50.     struct ip_vs_seq out_seq; /* outgoing seq. struct */
  51. };

3.2  ip_vs_dest

  1. /*
  2.  *    The real server destination forwarding entry
  3.  *    with ip address, port number, and so on.该结构体主要描述的是real server的相关信息，也是ipvsdm配置的到内核的相关信息
  4.  */
  5. struct ip_vs_dest {
  6.     struct list_head    n_list; /* for the dests in the service */
  7.     struct list_head    d_list; /* for table with all the dests t是全局hash链表ip_vs_rtable的一个节点*/

  9.     u16            af;        /* address family 地址的协议族 AF_INET/AF_INET6 */
  10.     union nf_inet_addr    addr;        /* IP address of the server Real Server 的地址 */
  11.     __be16            port;        /* port number of the server Real Server 端口号 */
  12. /*ip_vs_dest对象的状态标志位，IP_VS_DEST_F_AVAILABLE表示此真实服务器可用，IP_VS_DEST_F_OVERLOAD表示此真实服务器超负荷。*/
  13.     volatile unsigned    flags;        /* dest status flags */
  14. /*ip_vs_dest对象的连接标志位。这些标志位本身不是用来标示 ip_vs_dest对象的，而是由ip_vs_dest对象创建ip_vs_conn对象时，赋给后者的。IP_VS_CONN_F_MASQ、 IP_VS_CONN_F_TUNNEL和IP_VS_CONN_F_DROUTE，分别代表NAT、TUN和DR三种模式*/
  15.     atomic_t        conn_flags;    /* flags to copy to conn */
  16.     atomic_t        weight;        /* server weight 权重用于调度*/
  17. /*ip_vs_dest对象的引用计数，初值为0，当对象被加入链表或从链表删除，或者被ip_vs_conn对象引用时，refcnt相应地增或减1*/
  18.     atomic_t        refcnt;        /* reference counter */
  19.     struct ip_vs_stats stats; /* statistics */

  21.     /* connection counters and thresholds 连接的统计和阈值 */
  22.     atomic_t        activeconns;    /* active connections */
  23.     atomic_t        inactconns;    /* inactive connections */
  24.     atomic_t        persistconns;    /* persistent connections */
  25.     __u32            u_threshold;    /* upper threshold */
  26.     __u32            l_threshold;    /* lower threshold */

  28.     /* for destination cache */
  29.     spinlock_t        dst_lock;    /* lock of dst_cache */
  30.     struct dst_entry    *dst_cache;    /* destination cache entry */
  31.     u32            dst_rtos;    /* RT_TOS(tos) for dst */

  33.     /* for virtual service LVS的相关信息*/
  34.     struct ip_vs_service    *svc;        /* service it belongs to */
  35.     __u16            protocol;    /* which protocol (TCP/UDP) */
  36.     union nf_inet_addr    vaddr;        /* virtual IP address */
  37.     __be16            vport;        /* virtual port number */
  38.     __u32            vfwmark;    /* firewall mark of service 防火墙标志*/
  39. };

3. 3  ip_vs_service_user_kern

下面这个结构体主要是用户空间向内核空间发送IPVS Server的相关配置。

  1. struct ip_vs_service_user_kern {
  2.     /* virtual service addresses */
  3.     u16            af;
  4.     u16            protocol;
  5.     union nf_inet_addr    addr;        /* virtual ip address */
  6.     u16            port;
  7.     u32            fwmark;        /* firwall mark of service */

  9.     /* virtual service options */
  10.     char            *sched_name;
  11.     unsigned        flags;        /* virtual service flags */
  12.     unsigned        timeout;    /* persistent timeout in sec */
  13.     u32            netmask;    /* persistent netmask */
  14. };

3.4  ip_vs_dest_user_kern

下面这个结构主要是用户空间向内核空间发送Real Server的相关数据使用。

点击(此处)折叠或打开

  1. struct ip_vs_dest_user_kern {
  2.     /* destination server address */
  3.     union nf_inet_addr    addr;
  4.     u16            port;

  6.     /* real server options */
  7.     unsigned        conn_flags;    /* connection flags */
  8.     int            weight;        /* destination weight */

  10.     /* thresholds for active connections */
  11.     u32            u_threshold;    /* upper threshold */
  12.     u32            l_threshold;    /* lower threshold */
  13. };

3.5  ip_vs_service

点击(此处)折叠或打开

  1. /*
  2.  *    The information about the virtual service offered to the net
  3.  *    and the forwarding entries
  4.  */
  5. struct ip_vs_service {
  6. /*s_list是全局hash链表ip_vs_svc_table的一个节点 static struct list_head ip_vs_svc_table[IP_VS_SVC_TAB_SIZE]; 它是一个数组，每个成员是一个链表头。将ip_vs_service对象的协议类型、地址和端口进行hash，hash值作为数组下标，然后将此对象置入数组成员对应的链表中。*/
  7.     struct list_head    s_list; /* for normal service table */
  8.     struct list_head    f_list; /* for fwmark-based service table */
  9. /*refcnt和usecnt分别是ip_vs_service对象的引用计数和使用计数。它们是atomic_t类型的变量。refcnt在对象新建时为0，当ip_vs_service对象被加入链表或从链表删除，或者被 ip_vs_dest对象引用时，refcnt相应地增或减1。usecnt初始化为1*/
  10.     atomic_t        refcnt; /* reference counter */
  11.     atomic_t        usecnt; /* use counter */

  13.     u16            af; /* address family 地址族*/
  14.     __u16            protocol; /* which protocol (TCP/UDP) */
  15.     union nf_inet_addr    addr;     /* IP address for virtual service 虚拟服务器的地址 */
  16.     __be16            port;     /* port number for the service 虚拟服务器的端口号*/
  17.     __u32 fwmark; /* firewall mark of the service */
  18.     /*ip_vs_service对象的状态标志位，可以取IP_VS_SVC_F_PERSISTENT和 IP_VS_SVC_F_HASHED。前者表示IPVS服务使用了基于IP地址的会话保持，即同一IP地址发起的连接将被负载到同一台真实服务器上。后 者表示ip_vs_service对象已被加入到ip_vs_svc_table链表中*/
  19. unsigned        flags;     /* service status flags */
  20. /*timeout和netmask只有在IP_VS_SVC_F_PERSISTENT标志位被设置时才有效, timeout是会话的超时时间，超过 此时间后，会话将不再有效。同一IP地址发起的两个连接，如果间隔超过此时间，则未必会被负载到同一台真实服务器上*/
  21.     unsigned        timeout; /* persistent timeout in ticks */
  22. /*netmask可以将会话保持设置成 基于IP网段的，即同一网段发起的连接将被负载到同一台真实服务器上。*/
  23.     __be32            netmask; /* grouping granularity */
  24. /*destinations是ip_vs_dest对象链表，它代指了IPVS服务对应的真实服务器列表。num_dests是服务器个数*/
  25.     struct list_head    destinations; /* real server d-linked list */
  26.     __u32            num_dests; /* number of servers */
  27.     struct ip_vs_stats stats; /* statistics for the service */
  28.     struct ip_vs_app    *inc;     /* bind conns to this app inc */

  30.     /* for scheduling 指向了一个ip_vs_scheduler对象，它代指一种调度算法*/
  31.     struct ip_vs_scheduler    *scheduler; /* bound scheduler object */
  32.     rwlock_t        sched_lock; /* lock sched_data */
  33.     void            *sched_data; /* scheduler application data */
  34. };

3.6  ip_vs_dest_user_kern

用户空间真实服务器信息

点击(此处)折叠或打开

  1. struct ip_vs_dest_user_kern {
  2.     /* destination server address */
  3.     union nf_inet_addr    addr;
  4.     __be16            port;

  6.     /* real server options */
  7.     unsigned int        conn_flags;    /* connection flags */
  8.     int            weight;        /* destination weight */

  10.     /* thresholds for active connections */
  11.     u32            u_threshold;    /* upper threshold */
  12.     u32            l_threshold;    /* lower threshold */

  14.     /* Address family of addr */
  15.     u16            af;
  16. };

3.7  ip_vs_service_user_kern

用户空间虚拟服务器信息

点击(此处)折叠或打开

  1. struct ip_vs_service_user_kern {
  2.     /* virtual service addresses */
  3.     u16            af;
  4.     u16            protocol;
  5.     union nf_inet_addr    addr;        /* virtual ip address */
  6.     __be16            port;
  7.     u32            fwmark;        /* firwall mark of service */

  9.     /* virtual service options */
  10.     char            *sched_name;
  11.     char            *pe_name;
  12.     unsigned int        flags;        /* virtual service flags */
  13.     unsigned int        timeout;    /* persistent timeout in sec */
  14.     __be32            netmask;    /* persistent netmask or plen */
  15. };

3.8  ip_vs_protocol


点击(此处)折叠或打开

  1. struct ip_vs_protocol {
  2.     struct ip_vs_protocol    *next;
  3.  /*协议的名字*/
  4.     char            *name;
  5. /*协议号*/
  6.     u16            protocol;
  7.     u16            num_states;
  8.     int            dont_defrag;

  10.     void (*init)(struct ip_vs_protocol *pp);

  12.     void (*exit)(struct ip_vs_protocol *pp);

  14.     int (*init_netns)(struct net *net, struct ip_vs_proto_data *pd);

  16.     void (*exit_netns)(struct net *net, struct ip_vs_proto_data *pd);

  18. /*协议调度函数*/    
  19. int (*conn_schedule)(int af, struct sk_buff *skb,
  20.              struct ip_vs_proto_data *pd,
  21.              int *verdict, struct ip_vs_conn **cpp,
  22.              struct ip_vs_iphdr *iph);
  23.  /*查in方向的IPVS*/
  24.     struct ip_vs_conn *
  25.     (*conn_in_get)(int af,
  26.          const struct sk_buff *skb,
  27.          const struct ip_vs_iphdr *iph,
  28.          int inverse);
  29.  /*查out方向的IPVS*/
  30.     struct ip_vs_conn *
  31.     (*conn_out_get)(int af,
  32.             const struct sk_buff *skb,
  33.             const struct ip_vs_iphdr *iph,
  34.             int inverse);
  35.  /*SNAT处理函数*/
  36.     int (*snat_handler)(struct sk_buff *skb, struct ip_vs_protocol *pp,
  37.              struct ip_vs_conn *cp, struct ip_vs_iphdr *iph);
  38.  /*DNAT处理函数*/
  39.     int (*dnat_handler)(struct sk_buff *skb, struct ip_vs_protocol *pp,
  40.              struct ip_vs_conn *cp, struct ip_vs_iphdr *iph);
  41.  /*校验和处理函数*/
  42.     int (*csum_check)(int af, struct sk_buff *skb,
  43.              struct ip_vs_protocol *pp);

  45.     const char *(*state_name)(int state);
  46.  /*状态转换函数*/
  47.     void (*state_transition)(struct ip_vs_conn *cp, int direction,
  48.                  const struct sk_buff *skb,
  49.                  struct ip_vs_proto_data *pd);

  51.     int (*register_app)(struct net *net, struct ip_vs_app *inc);

  53.     void (*unregister_app)(struct net *net, struct ip_vs_app *inc);
  54.  /*多连接的应用绑定函数*/
  55.     int (*app_conn_bind)(struct ip_vs_conn *cp);

  57.     void (*debug_packet)(int af, struct ip_vs_protocol *pp,
  58.              const struct sk_buff *skb,
  59.              int offset,
  60.              const char *msg);

  62.     void (*timeout_change)(struct ip_vs_proto_data *pd, int flags);
  63. };
上一篇：关于TCP MSS的内核设置
下一篇：LVS 内核实现分析（3）