revised bootstrap (#599)

include/n2n_define.h

@@ -38,6 +38,7 @@
 /* Space needed to store socket and MAC address of a supernode */
 #define REG_SUPER_ACK_PAYLOAD_ENTRY_SIZE (sizeof(n2n_REGISTER_SUPER_ACK_payload_t))
 
+#define BOOTSTRAP_TIMEOUT                 3
 #define PURGE_REGISTRATION_FREQUENCY     30
 #define RE_REG_AND_PURGE_FREQUENCY       10
 #define REGISTRATION_TIMEOUT             60

include/n2n_typedefs.h

@@ -617,6 +617,7 @@ struct n2n_edge {
     /* Status */
     struct peer_info                 *curr_sn;                           /**< Currently active supernode. */
     uint8_t                          sn_wait;                            /**< Whether we are waiting for a supernode response. */
+    uint8_t                          sn_pong;                            /**< Whether we have seen a PONG since last time reset. */
     size_t                           sup_attempts;                       /**< Number of remaining attempts to this supernode. */
     tuntap_dev                       device;                             /**< All about the TUNTAP device */
     n2n_trans_op_t                   transop;                            /**< The transop to use when encoding */

src/edge.c

@@ -42,7 +42,10 @@ static cap_value_t cap_values[] = {
 int num_cap = sizeof(cap_values)/sizeof(cap_value_t);
 #endif
 
+// forward declaration for use in main()
 void send_register_super (n2n_edge_t *eee);
+void send_query_peer (n2n_edge_t * eee, const n2n_mac_t dst_mac);
+
 
 /* ***************************************************** */
 
@@ -704,10 +707,17 @@ BOOL WINAPI term_handler(DWORD sig)
 int main (int argc, char* argv[]) {
 
     int rc;
-    tuntap_dev tuntap;        /* a tuntap device */
+    tuntap_dev tuntap;            /* a tuntap device */
-    n2n_edge_t *eee;            /* single instance for this program */
+    n2n_edge_t *eee;              /* single instance for this program */
-    n2n_edge_conf_t conf; /* generic N2N edge config */
+    n2n_edge_conf_t conf;         /* generic N2N edge config */
-    n2n_tuntap_priv_config_t ec; /* config used for standalone program execution */
+    n2n_tuntap_priv_config_t ec;  /* config used for standalone program execution */
+    uint8_t runlevel = 0;         /* bootstrap: runlevel */
+    uint8_t seek_answer = 1;      /*            expecting answer from supernode */
+    time_t now_time, last_action; /*            timeout */
+    macstr_t mac_buf;             /*            output mac address */
+    fd_set socket_mask;           /*            for supernode answer */
+    struct timeval wait_time;     /*            timeout for sn answer */
+
 #ifndef WIN32
     struct passwd *pw = NULL;
 #endif
@@ -806,24 +816,120 @@ int main (int argc, char* argv[]) {
     } else {
         traceEvent(TRACE_NORMAL, "Automatically assign IP address by supernode.");
         eee->conf.tuntap_ip_mode = TUNTAP_IP_MODE_SN_ASSIGN;
+    }
 
-        // REVISIT: integrate into the (to be created) bootstrap, maybe even as part of a more stateful main loop
+    // mini main loop for bootstrap, not using main loop code because some of its mechanisms do not fit in here
-        eee->sn_wait = 1;
+    // for the sake of quickly establishing connection. REVISIT when a more elegant way to re-use main loop code
-        do {
+    // is found
-            fd_set socket_mask;
-            struct timeval wait_time;
 
-            // next supernode
+    // if more than one supernode given, find at least one who is alive to faster establish connection
-            if (eee->curr_sn->hh.next)
+    if(HASH_COUNT(eee->conf.supernodes) <= 1) {
-                eee->curr_sn = eee->curr_sn->hh.next;
+        // skip the initial supernode ping
-            else
+        traceEvent(TRACE_DEBUG, "Skip PING to supernode.");
+        runlevel = 2;
+    }
+
+    eee->last_sup = 1; /* to prevent gratuitous arp packet */
+    eee->curr_sn = eee->conf.supernodes;
+
+    while(runlevel < 5) {
+
+        now_time = time(NULL);
+
+        // we do not use switch-case because we also check for 'greater than'
+
+        if(runlevel == 0) { /* PING to all known supernodes */
+            last_action = now_time;
+            eee->sn_pong = 0;
+            send_query_peer(eee, null_mac);
+            traceEvent(TRACE_NORMAL, "Send PING to supernodes.");
+            runlevel++;
+        }
+
+        if(runlevel == 1) { /* PING has been sent to all known supernodes */
+            if(eee->sn_pong) {
+                // first answer
+                eee->sn_pong = 0;
+                sn_selection_sort(&(eee->conf.supernodes));
                 eee->curr_sn = eee->conf.supernodes;
+                traceEvent(TRACE_NORMAL, "Received first PONG from supernode [%s].", eee->curr_sn->ip_addr);
+                runlevel++;
+            }
+            if(last_action <= (now_time - BOOTSTRAP_TIMEOUT)) {
+                // timeout
+                runlevel--;
+                // skip waiting for answer to direcly go to send PING again
+                seek_answer = 0;
+                traceEvent(TRACE_DEBUG, "PONG timeout.");
+            }
+        }
 
-            send_register_super(eee);
+        // by the way, have every later PONG cause the remaining (!) list to be sorted because the entries
+        // before have already been tried; as opposed to initial PONG, do not change curr_sn
+        if(runlevel > 1) {
+            if(eee->sn_pong) {
+                eee->sn_pong = 0;
+                if(eee->curr_sn->hh.next) {
+                    sn_selection_sort((peer_info_t**)&(eee->curr_sn->hh.next));
+                    traceEvent(TRACE_DEBUG, "Received additional PONG from supernode.");
+                    // here, it is hard to detemine from which one, so no details to output
+                }
+            }
+        }
+
+        if(runlevel == 2) { /* send REGISTER_SUPER to get auto ip address from a supernode */
+            if(eee->conf.tuntap_ip_mode == TUNTAP_IP_MODE_SN_ASSIGN) {
+                last_action = now_time;
+                eee->sn_wait = 1;
+                send_register_super(eee);
+                runlevel++;
+                traceEvent(TRACE_NORMAL, "Send REGISTER_SUPER to supernode [%s] asking for IP address.",
+                                         eee->curr_sn->ip_addr);
+            } else {
+                runlevel += 2; /* skip waiting for TUNTAP IP address */
+                traceEvent(TRACE_DEBUG, "Skip auto IP address asignment.");
+            }
+        }
 
+        if(runlevel == 3) { /* REGISTER_SUPER to get auto ip address from a sn has been sent */
+            if(!eee->sn_wait) { /* TUNTAP IP address received */
+                runlevel++;
+                traceEvent(TRACE_NORMAL, "Received REGISTER_SUPER_ACK from supernode for IP address asignment.");
+                // it should be from curr_sn, but we can't determine definitely here, so no details to output
+            }
+            if(last_action <= (now_time - BOOTSTRAP_TIMEOUT)) {
+                // timeout, so try next supernode
+                if(eee->curr_sn->hh.next)
+                    eee->curr_sn = eee->curr_sn->hh.next;
+                else
+                    eee->curr_sn = eee->conf.supernodes;
+                runlevel--;
+                // skip waiting for answer to direcly go to send REGISTER_SUPER again
+                seek_answer = 0;
+                traceEvent(TRACE_DEBUG, "REGISTER_SUPER_ACK timeout.");
+            }
+        }
+
+        if(runlevel == 4) { /* configure the TUNTAP device */
+            if(tuntap_open(&tuntap, eee->tuntap_priv_conf.tuntap_dev_name, eee->tuntap_priv_conf.ip_mode,
+                           eee->tuntap_priv_conf.ip_addr, eee->tuntap_priv_conf.netmask,
+                           eee->tuntap_priv_conf.device_mac, eee->tuntap_priv_conf.mtu) < 0)
+                exit(1);
+            memcpy(&eee->device, &tuntap, sizeof(tuntap));
+            traceEvent(TRACE_NORMAL, "Created local tap device IP: %s, Mask: %s, MAC: %s",
+                                     eee->tuntap_priv_conf.ip_addr,
+                                     eee->tuntap_priv_conf.netmask,
+                                     macaddr_str(mac_buf, eee->device.mac_addr));
+            runlevel = 5;
+            // no more answers required
+            seek_answer = 0;
+        }
+
+        // we usually wait for some answer, there however are exceptions when going back to a previous runlevel
+        if(seek_answer) {
             FD_ZERO(&socket_mask);
             FD_SET(eee->udp_sock, &socket_mask);
-            wait_time.tv_sec = (SOCKET_TIMEOUT_INTERVAL_SECS / 10) + 1;
+            wait_time.tv_sec = BOOTSTRAP_TIMEOUT;
             wait_time.tv_usec = 0;
 
             if(select(eee->udp_sock + 1, &socket_mask, NULL, NULL, &wait_time) > 0) {
@@ -831,17 +937,12 @@ int main (int argc, char* argv[]) {
                     readFromIPSocket(eee, eee->udp_sock);
                 }
             }
-        } while(eee->sn_wait);
+        }
-        eee->last_register_req = 0;
+        seek_answer = 1;
     }
-
+    eee->sn_wait = 1;
-    if(tuntap_open(&tuntap, eee->tuntap_priv_conf.tuntap_dev_name, eee->tuntap_priv_conf.ip_mode,
+    eee->last_register_req = 0;
-                   eee->tuntap_priv_conf.ip_addr, eee->tuntap_priv_conf.netmask,
+    eee->last_sup = 0; /* to allow gratuitous arp packet after regular REGISTER_SUPER_ACK */
-                   eee->tuntap_priv_conf.device_mac, eee->tuntap_priv_conf.mtu) < 0) exit(1);
-    traceEvent(TRACE_NORMAL, "Local tap IP: %s, Mask: %s",
-               eee->tuntap_priv_conf.ip_addr, eee->tuntap_priv_conf.netmask);
-    memcpy(&eee->device, &tuntap, sizeof(tuntap));
-    //hexdump((unsigned char*)&tuntap,sizeof(tuntap_dev));
 
 #ifndef WIN32
     if(eee->tuntap_priv_conf.daemon) {

src/edge_utils.c

@@ -761,7 +761,7 @@ static void check_join_multicast_group (n2n_edge_t *eee) {
 /* ************************************** */
 
 /** Send a QUERY_PEER packet to the current supernode. */
-static void send_query_peer (n2n_edge_t * eee,
+void send_query_peer (n2n_edge_t * eee,
                              const n2n_mac_t dst_mac) {
 
     uint8_t pktbuf[N2N_PKT_BUF_SIZE];
@@ -903,6 +903,7 @@ static int sort_supernodes (n2n_edge_t *eee, time_t now) {
     struct peer_info *scan, *tmp;
 
     if(eee->curr_sn != eee->conf.supernodes) {
+        // have not been connected to the best/top one
         send_unregister_super(eee);
 
         eee->curr_sn = eee->conf.supernodes;
@@ -920,7 +921,6 @@ static int sort_supernodes (n2n_edge_t *eee, time_t now) {
             // this routine gets periodically called
             // it sorts supernodes in ascending order of their selection_criterion fields
             sn_selection_sort(&(eee->conf.supernodes));
-
         }
 
         HASH_ITER(hh, eee->conf.supernodes, scan, tmp) {
@@ -928,8 +928,12 @@ static int sort_supernodes (n2n_edge_t *eee, time_t now) {
         }
         sn_selection_criterion_common_data_default(eee);
 
+        // send PING to all the supernodes
         send_query_peer(eee, null_mac);
         eee->last_sweep = now;
+
+        // no answer yet (so far, unused in regular edge code; mainly used during bootstrap loading)
+        eee->sn_pong = 0;
     }
 
     return 0; /* OK */
@@ -2104,13 +2108,6 @@ void readFromIPSocket (n2n_edge_t * eee, int in_sock) {
 
                 memset(&ra, 0, sizeof(n2n_REGISTER_SUPER_ACK_t));
 
-                // Indicates successful connection between the edge and SN nodes
-                static int bTrace = 1;
-                if(bTrace) {
-                    traceEvent(TRACE_NORMAL, "[OK] Edge Peer <<< ================ >>> Super Node");
-                    bTrace = 0;
-                }
-
                 if(eee->sn_wait) {
                     decode_REGISTER_SUPER_ACK(&ra, &cmn, udp_buf, &rem, &idx, tmpbuf);
 
@@ -2122,7 +2119,7 @@ void readFromIPSocket (n2n_edge_t * eee, int in_sock) {
                     }
 
                     if(is_valid_peer_sock(&ra.sock))
-                            orig_sender = &(ra.sock);
+                        orig_sender = &(ra.sock);
 
                     traceEvent(TRACE_INFO, "Rx REGISTER_SUPER_ACK myMAC=%s [%s] (external %s). Attempts %u",
                                macaddr_str(mac_buf1, ra.edgeMac),
@@ -2130,6 +2127,7 @@ void readFromIPSocket (n2n_edge_t * eee, int in_sock) {
                                sock_to_cstr(sockbuf2, orig_sender),
                                (unsigned int)eee->sup_attempts);
 
+                    // this even holds true for auto ip assignment as own mac is null_mac
                     if(memcmp(ra.edgeMac, eee->device.mac_addr, N2N_MAC_SIZE)) {
                         traceEvent(TRACE_INFO, "readFromIPSocket dropped REGISTER_SUPER_ACK due to wrong addressing.");
                         return;
@@ -2173,20 +2171,24 @@ void readFromIPSocket (n2n_edge_t * eee, int in_sock) {
                             }
                         }
 
-                        if(!eee->last_sup) // send gratuitous ARP only upon first registration with supernode
+                        if(!eee->last_sup) {
+                            // indicates successful connection between the edge and a supernode
+                            traceEvent(TRACE_NORMAL, "[OK] Edge Peer <<< ================ >>> Super Node");
+                            // send gratuitous ARP only upon first registration with supernode
                             send_grat_arps(eee);
+                        }
 
                         eee->last_sup = now;
                         eee->sn_wait = 0;
                         eee->sup_attempts = N2N_EDGE_SUP_ATTEMPTS; /* refresh because we got a response */
 
-                        if(eee->cb.sn_registration_updated)
-                            eee->cb.sn_registration_updated(eee, now, &sender);
-
                         /* NOTE: the register_interval should be chosen by the edge node
                          * based on its NAT configuration. */
                         //eee->conf.register_interval = ra.lifetime;
 
+                        if(eee->cb.sn_registration_updated && !is_null_mac(ra.edgeMac))
+                            eee->cb.sn_registration_updated(eee, now, &sender);
+
                     } else {
                         traceEvent(TRACE_INFO, "Rx REGISTER_SUPER_ACK with wrong or old cookie.");
                     }
@@ -2247,15 +2249,18 @@ void readFromIPSocket (n2n_edge_t * eee, int in_sock) {
                 }
 
                 if(is_null_mac(pi.mac)) {
+                    // PONG - answer to PING (QUERY_PEER_INFO with null mac)
                     skip_add = SN_ADD_SKIP;
                     scan = add_sn_to_list_by_mac_or_sock(&(eee->conf.supernodes), &sender, pi.srcMac, &skip_add);
                     if(scan != NULL) {
+                        eee->sn_pong = 1;
                         scan->last_seen = now;
                         /* The data type depends on the actual selection strategy that has been chosen. */
                         sn_selection_criterion_calculate(eee, scan, &pi.data);
                         break;
                     }
                 } else {
+                    // regular PEER_INFO
                     HASH_FIND_PEER(eee->pending_peers, pi.mac, scan);
 
                     if(scan) {
@@ -2423,11 +2428,11 @@ int run_edge_loop (n2n_edge_t * eee, int *keep_running) {
                 eee->cb.ip_address_changed(eee, old_ip, eee->device.ip_addr);
         }
 
+        sort_supernodes(eee, nowTime);
+
         if(eee->cb.main_loop_period)
             eee->cb.main_loop_period(eee, nowTime);
 
-        sort_supernodes(eee, nowTime);
-
     } /* while */
 
 #ifdef WIN32

src/sn_utils.c

@@ -1365,22 +1365,30 @@ static int process_udp (n2n_sn_t * sss,
                     traceEvent(TRACE_DEBUG, "Tx REGISTER_SUPER_NAK for %s",
                                macaddr_str(mac_buf, reg.edgeMac));
                 } else {
+                    // if this is not already forwarded from a supernode, ...
                     if(!(cmn.flags & N2N_FLAGS_SOCKET)) {
-                        reg.sock.family = AF_INET;
+                        // ... forward to all other supernodes (note try_broadcast()'s behavior with
-                        reg.sock.port = ntohs(sender_sock->sin_port);
+                        //     NULL comm and from_supernode parameter)
-                        memcpy(reg.sock.addr.v4, &(sender_sock->sin_addr.s_addr), IPV4_SIZE);
 
-                        cmn2.pc = n2n_register_super;
+                        // exception: do not forward auto ip draw
-                        encode_REGISTER_SUPER(ackbuf, &encx, &cmn2, &reg);
+                        if(!is_null_mac(reg.edgeMac)) {
+                            reg.sock.family = AF_INET;
+                            reg.sock.port = ntohs(sender_sock->sin_port);
+                            memcpy(reg.sock.addr.v4, &(sender_sock->sin_addr.s_addr), IPV4_SIZE);
 
-                        if(comm->header_encryption == HEADER_ENCRYPTION_ENABLED) {
+                            cmn2.pc = n2n_register_super;
-                            packet_header_encrypt(ackbuf, encx, encx,
+                            encode_REGISTER_SUPER(ackbuf, &encx, &cmn2, &reg);
-                                                  comm->header_encryption_ctx, comm->header_iv_ctx,
+
-                                                  time_stamp());
+                            if(comm->header_encryption == HEADER_ENCRYPTION_ENABLED) {
-                        }
+                                packet_header_encrypt(ackbuf, encx, encx,
+                                                      comm->header_encryption_ctx, comm->header_iv_ctx,
+                                                      time_stamp());
+                            }
 
-                        try_broadcast(sss, NULL, &cmn, reg.edgeMac, from_supernode, ackbuf, encx);
+                            try_broadcast(sss, NULL, &cmn, reg.edgeMac, from_supernode, ackbuf, encx);
+                        }
 
+                        // send REGISTER_SUPER_ACK
                         encx = 0;
                         cmn2.pc = n2n_register_super_ack;