拓扑

TOPO

配置gre

1
2
3
4
5
6
#routerB
remoteip=30.1.1.2
localip=30.1.1.1
greip=10.10.10.1
grepeerip=10.10.10.2
grename=gre1
1
2
3
4
5
6
#routerA
remoteip=30.1.1.1
localip=30.1.1.2
greip=10.10.10.2
grepeerip=10.10.10.1
grename=gre1
1
2
3
4
5
6
7
8
9
#routerB and routerA
lsmod|grep ip_gre || modprobe ip_gre
ping -c1 ${remoteip} || { echo "make sure the network is ok plz"; exit 1; }
ip addr show ${grename} && { echo "${grename} is already exist"; exit 1; }

ip tunnel add ${grename} mode gre remote ${remoteip} local ${localip} ikey 1 okey 1 ttl 255;
ip addr add ${greip} dev ${grename} peer ${grepeerip};
ip link set ${grename} up; #mtu 1400
ip addr show ${grename};

测试隧道连接

routerB ping -I 10.10.10.1 10.10.10.2 -c2 通路
routerA ping -I 10.10.10.2 10.10.10.1 -c2 通路

配置路由表

routeB ip route add 10.2.1.0/24 via 10.10.10.2
routeA ip route add 10.1.1.0/24 via 10.10.10.1

测试私网和隧道

PC2 ping 10.2.1.2 通路
PC1 ping 10.1.1.2 通路

1
2
3
4
5
6
7
# root @ routerA in / [11:37:33]
$ tcpdump -nvvvi gre1
tcpdump: listening on gre1, link-type LINUX_SLL (Linux cooked), capture size 262144 bytes
12:31:30.309857 IP (tos 0x0, ttl 63, id 59102, offset 0, flags [DF], proto ICMP (1), length 84)
    10.10.10.1 > 10.2.1.2: ICMP echo request, id 6, seq 1, length 64
12:31:30.310258 IP (tos 0x0, ttl 63, id 17143, offset 0, flags [none], proto ICMP (1), length 84)
    10.2.1.2 > 10.10.10.1: ICMP echo reply, id 6, seq 1, length 64

删除隧道

1
2
3
4
5
6
7
8
ip link set gre1 down
ip tunnel del gre1
``

## 卸载模块

```sh
rmmod ip_gre

TODO 创建多个隧道

两台设备之间建立多条gre通道

额外问题

  1. 查看内核是否支持gre

    1
    2
    grep GRE /boot/config-$(uname -r)
    sysctl net.netfilter.nf_conntrack_helper=1

  2. kernel: conntrack: generic helper won't handle protocol 47. Please consider loading the specific helper module.

    1
    modprobe nf_conntrack_proto_gre

  3. 若需要grep nf_nat_proto_gre || modprobe nf_nat_proto_gre

  4. GRE是将一个数据包封装到另一个数据包中，因此你可能会遇到GRE的数据报大于网络接口所设定的数据包最大尺寸的情况。解决这种问题的方法是在隧道接口上配置ip tcp adjust-mss 1436。另外，虽然GRE并不支持加密，但是你可以通过tunnel key命令在隧道的两头各设置一个密钥。这个密钥其实就是一个明文的密码。或者使用gre over ipsec，那样就比较复杂了再另说。

  5. GRE隧道没有状态控制，可能隧道的一端已经关闭，而另一端仍然开启。这一问题的解决方案就是在隧道两端开启keepalive数据包。它可以让隧道一端定时向另一端发送keepalive数据，确认端口保持开启状态。如果隧道的某一端没有按时收到keepalive数据，那么这一侧的隧道端口 也会关闭。

调试

代码打印

kernel/net/ipv4/ip_gre.c

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
static int ipgre_rcv(struct sk_buff *skb, const struct tnl_ptk_info *tpi)
{
    struct net *net = dev_net(skb->dev);
    struct metadata_dst *tun_dst = NULL;
    struct ip_tunnel_net *itn;
    const struct iphdr *iph;
    struct ip_tunnel *tunnel;
    char *p = NULL;
    char sip[18]={0};
    char dip[18]={0};

    pr_info("ipgre_rcv\n");
    if (tpi->proto == htons(ETH_P_TEB)){
        pr_info("tpi->proto == htons(ETH_P_TEB)\n");
        itn = net_generic(net, gre_tap_net_id);
    }
    else{
        pr_info("tpi->proto != htons(ETH_P_TEB)\n");
        itn = net_generic(net, ipgre_net_id);
    }

    iph = ip_hdr(skb);
    if(iph != NULL){
        p = (char *) &iph->saddr;
        sprintf(sip, "%d.%d.%d.%d", (p[0] & 255), (p[1] & 255), (p[2] & 255), (p[3] & 255));
        p = (char *) &iph->daddr;
        sprintf(dip, "%d.%d.%d.%d", (p[0] & 255), (p[1] & 255), (p[2] & 255), (p[3] & 255));
        pr_info("sip:%s -> dip:%s\n", sip, dip);
    }
    pr_info("skb->dev->ifindex:%d tpi->flags%d tpi->key%d\n", skb->dev->ifindex, tpi->flags, tpi->key);
    tunnel = ip_tunnel_lookup(itn, skb->dev->ifindex, tpi->flags,
            iph->saddr, iph->daddr, tpi->key);

    if (tunnel) {
        pr_info("ip_tunnel_lookup return not NULL\n");
        skb_pop_mac_header(skb);
        if (tunnel->collect_md) {
            __be16 flags;
            __be64 tun_id;

            pr_info("tunnel->collect_md != 0\n");
            flags = tpi->flags & (TUNNEL_CSUM | TUNNEL_KEY);
            tun_id = key_to_tunnel_id(tpi->key);
            tun_dst = ip_tun_rx_dst(skb, flags, tun_id, 0);
            if (!tun_dst){
                pr_info("ip_tun_rx_dst return NULL\n");
                return PACKET_REJECT;
            }
        }

        pr_info("ip_tunnel_rcv\n");
        ip_tunnel_rcv(tunnel, skb, tpi, tun_dst, log_ecn_error);
        return PACKET_RCVD;
    }
    pr_info("ip_tunnel_lookup return NULL\n");
    return PACKET_REJECT;
}

static int gre_rcv(struct sk_buff *skb)
{
    struct tnl_ptk_info tpi;
    bool csum_err = false;
    int hdr_len;

    pr_info("gre_rcv\n");
#ifdef CONFIG_NET_IPGRE_BROADCAST
    pr_info("gre_rcv is multicast?\n");
    if (ipv4_is_multicast(ip_hdr(skb)->daddr)) {
        pr_info("gre_rcv is multicast yes\n");
        /* Looped back packet, drop it! */
        if (rt_is_output_route(skb_rtable(skb)))
            goto drop;
    }
#endif

    hdr_len = parse_gre_header(skb, &tpi, &csum_err);
    pr_info("parse_gre_header\n");
    if (hdr_len < 0) {
        pr_info("hdr_len < 0\n");
        goto drop;
    }
    if (iptunnel_pull_header(skb, hdr_len, tpi.proto) < 0) {
        pr_info("iptunnel_pull_header < 0\n");
        goto drop;
    }

    if (ipgre_rcv(skb, &tpi) == PACKET_RCVD) {
        pr_info("ipgre_rcv == %d\n", PACKET_RCVD);
        return 0;
    }

    icmp_send(skb, ICMP_DEST_UNREACH, ICMP_PORT_UNREACH, 0);
    pr_info("icmp_send\n");
drop:
    pr_info("drop\n");
    kfree_skb(skb);
    return 0;
}

kernel/net/ipv4/ip_tunnel.c

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
/* Fallback tunnel: no source, no destination, no key, no options

   Tunnel hash table:
   We require exact key match i.e. if a key is present in packet
   it will match only tunnel with the same key; if it is not present,
   it will match only keyless tunnel.

   All keysless packets, if not matched configured keyless tunnels
   will match fallback tunnel.
   Given src, dst and key, find appropriate for input tunnel.
*/
struct ip_tunnel *ip_tunnel_lookup(struct ip_tunnel_net *itn,
        int link, __be16 flags,
        __be32 remote, __be32 local,
        __be32 key)
{
    unsigned int hash;
    struct ip_tunnel *t, *cand = NULL;
    struct hlist_head *head;

    pr_info("ip_tunnel_lookup\n");

    hash = ip_tunnel_hash(key, remote);
    pr_info("ip_tunnel_hash(%d, %d)-1>hash:%d\n", key, remote, hash);

    head = &itn->tunnels[hash];
    pr_info("1head:%lu\n", (long unsigned int)head);

    hlist_for_each_entry_rcu(t, head, hash_node) {
        pr_info("1local%d, saddr%d, remote%d, daddr%d, flags%x, IFF_UP%x\n",
                local, t->parms.iph.saddr, remote, t->parms.iph.daddr, t->dev->flags, IFF_UP);
        if (local != t->parms.iph.saddr ||
                remote != t->parms.iph.daddr ||
                !(t->dev->flags & IFF_UP)){
            pr_info("local%d != t->parms.iph.saddr%d || remote%d != t->parms.iph.daddr%d || !(t->dev->flags%x & IFF_UP%x) continue\n",
                    local, t->parms.iph.saddr, remote, t->parms.iph.daddr, t->dev->flags, IFF_UP);
            continue;
        }

        if (!ip_tunnel_key_match(&t->parms, flags, key)){
            pr_info("!ip_tunnel_key_match(&t->parms, %x, %d) continue\n", flags, key);
            continue;
        }

        if (t->parms.link == link){
            pr_info("t->parms.link%d == link%d return t\n", t->parms.link, link);
            return t;
        }
        else{
            pr_info("t->parms.link%d != link%d cand=t\n", t->parms.link, link);
            cand = t;
        }
    }

    hlist_for_each_entry_rcu(t, head, hash_node) {
        pr_info("remote:%d, daddr:%d, saddr:%d, flags%x&%x:%d\n", remote, t->parms.iph.daddr, t->parms.iph.saddr, t->dev->flags, IFF_UP, (t->dev->flags & IFF_UP));
        if (remote != t->parms.iph.daddr ||
                t->parms.iph.saddr != 0 ||
                !(t->dev->flags & IFF_UP)){
            pr_info("remote%d != t->parms.iph.daddr%d || t->parms.iph.saddr%d != 0 || !(t->dev->flags%x & IFF_UP%x) continue\n",
                    remote, t->parms.iph.daddr, t->parms.iph.saddr, t->dev->flags, IFF_UP);
            continue;
        }

        if (!ip_tunnel_key_match(&t->parms, flags, key)){
            pr_info("!ip_tunnel_key_match(&t->parms, %x, %d) continue\n", flags, key);
            continue;
        }

        if (t->parms.link == link){
            pr_info("t->parms.link%d == link%d return t\n", t->parms.link, link);
            return t;
        }
        else if (!cand){
            pr_info("t->parms.link%d != link%d cand=t\n", t->parms.link, link);
            cand = t;
        }
    }

    hash = ip_tunnel_hash(key, 0);
    pr_info("ip_tunnel_hash(%d, 0)-2>hash:%d\n", remote, hash);
    head = &itn->tunnels[hash];
    pr_info("2head:%lu\n", (long unsigned int)head);

    hlist_for_each_entry_rcu(t, head, hash_node) {
        pr_info("local:%d, saddr:%d, daddr:%d\n", local, t->parms.iph.saddr, t->parms.iph.daddr);
        if ((local != t->parms.iph.saddr || t->parms.iph.daddr != 0) &&
                (local != t->parms.iph.daddr || !ipv4_is_multicast(local))){
            pr_info("(local%d != t->parms.iph.saddr%d || t->parms.iph.daddr%d != 0) && (local%d != t->parms.iph.daddr%d || !ipv4_is_multicast(local)) continue\n",
                    local, t->parms.iph.saddr, t->parms.iph.daddr, local, t->parms.iph.daddr);
            continue;
        }

        if (!(t->dev->flags & IFF_UP)){
            pr_info("!(t->dev->flags%x & IFF_UP%x) continue\n", t->dev->flags, IFF_UP);
            continue;
        }

        if (!ip_tunnel_key_match(&t->parms, flags, key)){
            pr_info("!ip_tunnel_key_match(&t->parms, %x, %d) continue\n", flags, key);
            continue;
        }

        if (t->parms.link == link){
            pr_info("t->parms.link%d == link%d return t\n", t->parms.link, link);
            return t;
        }
        else if (!cand){
            pr_info("t->parms.link%d != link%d cand=t\n", t->parms.link, link);
            cand = t;
        }
    }

    if (flags & TUNNEL_NO_KEY){
        pr_info("flags%x & TUNNEL_NO_KEY%x goto skip_key_lookup\n", flags, TUNNEL_NO_KEY);
        goto skip_key_lookup;
    }

    hlist_for_each_entry_rcu(t, head, hash_node) {
        if (t->parms.i_key != key ||
                t->parms.iph.saddr != 0 ||
                t->parms.iph.daddr != 0 ||
                !(t->dev->flags & IFF_UP)){
            pr_info("t->parms.i_key%d != key%d || t->parms.iph.saddr%d != 0 || t->parms.iph.daddr%d != 0 || !(t->dev->flags%x & IFF_UP%x) continue\n",
                    t->parms.i_key, key, t->parms.iph.saddr, t->parms.iph.daddr, t->dev->flags, IFF_UP);
            continue;
        }

        if (t->parms.link == link){
            pr_info("t->parms.link%d == link%d return t\n", t->parms.link, link);
            return t;
        }
        else if (!cand){
            pr_info("t->parms.link%d != link%d cand=t\n", t->parms.link, link);
            cand = t;
        }
    }

skip_key_lookup:
    if (cand){
        pr_info("cand not NULL return cand\n");
        return cand;
    }

    t = rcu_dereference(itn->collect_md_tun);
    pr_info("t = rcu_dereference(itn->collect_md_tun)\n");
    if (t){
        pr_info("t not NULL return t\n");
        return t;
    }

    if (itn->fb_tunnel_dev && itn->fb_tunnel_dev->flags & IFF_UP){
        pr_info("itn->fb_tunnel_dev && itn->fb_tunnel_dev->flags & IFF_UP return netdev_priv(itn->fb_tunnel_dev);\n");
        return netdev_priv(itn->fb_tunnel_dev);
    }

    return NULL;
}
EXPORT_SYMBOL_GPL(ip_tunnel_lookup);

编译加载模块

1
2
3
4
modinfo ip_gre
modprobe gre
#make M=net/ipv4
make modules SUBDIRS=net/ipv4;rmmod ip_gre;rmmod ip_tunnel;insmod net/ipv4/ip_tunnel.ko;insmod net/ipv4/ip_gre.ko

设备直连时

隧道是正常通信的

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
[ 2990.916244] output info dev:eth1 protocol:0x0800, 20.1.1.1->20.1.1.2 protocol:47 ip csum:578(1400) len:92
[ 3056.971477] input info dev:eth1 protocol:0x0800, 20.1.1.2->20.1.1.1 protocol:47 ip csum:bacb(47819) len:92
[ 3056.971494] ip_gre: gre_rcv
[ 3056.971496] ip_gre: gre_rcv is multicast?
[ 3056.971499] ip_gre: parse_gre_header
[ 3056.971502] ip_gre: ipgre_rcv
[ 3056.971504] ip_gre: tpi->proto != htons(ETH_P_TEB)
[ 3056.971508] ip_gre: sip:20.1.1.2 -> dip:20.1.1.1
[ 3056.971511] ip_gre: skb->dev->ifindex:3 tpi->flags1024 tpi->key33554432
[ 3056.971513] ip_tunnel: ip_tunnel_lookup
[ 3056.971516] ip_tunnel: ip_tunnel_hash(33554432, 33620244)-1>hash:73
[ 3056.971519] ip_tunnel: 1head:3957364008
[ 3056.971523] ip_tunnel: 1local16843028, saddr16843028, remote33620244, daddr33620244, flags91, IFF_UP1
[ 3056.971525] ip_tunnel: t->parms.link3 == link3 return t
[ 3056.971527] ip_gre: ip_tunnel_lookup return not NULL
[ 3056.971530] ip_gre: ip_tunnel_rcv
[ 3056.971533] ip_gre: ipgre_rcv == 0
[ 3056.971561] input info dev:gre2 protocol:0x0800, 10.10.10.4->10.10.10.3 protocol:1 ip csum:8ca5(36005) len:64, type8
[ 3056.971612] output info dev:gre2 protocol:0x0800, 10.10.10.3->10.10.10.4 protocol:1 ip csum:7f7a(32634) len:64, type0
[ 3056.971632] output info dev:eth1 protocol:0x0800, 20.1.1.1->20.1.1.2 protocol:47 ip csum:f158(61784) len:92

测试跨网段时

隧道是不通的

TOPO

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
[ 3498.496802] input info dev:eth1 protocol:0x0800, 20.1.1.2->20.1.1.1 protocol:47 ip csum:709a(28826) len:92
[ 3498.496810] ip_gre: gre_rcv
[ 3498.496847] ip_gre: gre_rcv is multicast?
[ 3498.497130] ip_gre: parse_gre_header
[ 3498.497136] ip_gre: ipgre_rcv
[ 3498.497141] ip_gre: tpi->proto != htons(ETH_P_TEB)
[ 3498.497149] ip_gre: sip:20.1.1.2 -> dip:20.1.1.1
[ 3498.497155] ip_gre: skb->dev->ifindex:3 tpi->flags1024 tpi->key16777216
[ 3498.497160] ip_tunnel: ip_tunnel_lookup
[ 3498.497166] ip_tunnel: ip_tunnel_hash(16777216, 33620244)-1>hash:75
[ 3498.497171] ip_tunnel: 1head:3957811504
[ 3498.497177] ip_tunnel: ip_tunnel_hash(33620244, 0)-2>hash:0
[ 3498.497183] ip_tunnel: 2head:3957811204
[ 3498.497189] ip_tunnel: local:16843028, saddr:0, daddr:0
[ 3498.497196] ip_tunnel: (local16843028 != t->parms.iph.saddr0 || t->parms.iph.daddr0 != 0) && (local16843028 != t->parms.iph.daddr0 || !ipv4_is_multicast(local)) continue
[ 3498.497204] ip_tunnel: t->parms.i_key0 != key16777216 || t->parms.iph.saddr0 != 0 || t->parms.iph.daddr0 != 0 || !(t->dev->flags80 & IFF_UP1) continue
[ 3498.497209] ip_tunnel: t = rcu_dereference(itn->collect_md_tun)
[ 3498.497214] ip_gre: ip_tunnel_lookup return NULL
[ 3498.497219] ip_gre: icmp_send
[ 3498.497223] ip_gre: drop

结论

gre用于同网段的A类地址（即公网地址）的两个设备之间，跨网段是无法建立隧道的（除非关闭RouterB的NAT）。