onnode all ctdb setdebug INFO ; sleep 2 ; date ; ifconfig eth0 down ; for i in {1..10} ; do date && ctdb status && ctdb ip&&sleep 1 ; done  ; onnode all ctdb setdebug NOTICE 

monitor_interfaces()
{
    get_all_interfaces                                                 
    delete_unexpected_ips

    fail=false
    up_interfaces_found=false

    for iface in $all_interfaces ;do
        ip addr show $iface 2>/dev/null >/dev/null || {
        echo "WARNING: Interface $iface does not exist but it is used by public addresses."
        continue
        }

        # These interfaces are sometimes bond devices
        # When we use VLANs for bond interfaces, there will only
        # be an entry in /proc for the underlying real interface
        /*注意下面这段逻辑是处理bond相关的配置的*/
        realiface=`echo $iface |sed -e 's/\..*$//'`
        bi=$(get_proc "net/bonding/$realiface" 2>/dev/null) && {
        echo "$bi" | grep -q 'Currently Active Slave: None' && {
            echo "ERROR: No active slaves for bond device $realiface"
            mark_down $iface
            continue
        }
        echo "$bi" | grep -q '^MII Status: up' || {
            echo "ERROR: public network interface $realiface is down"
            mark_down $iface
            continue
        }                                                                                                                                              
        echo "$bi" | grep -q '^Bonding Mode: IEEE 802.3ad Dynamic link aggregation' && {
            # This works around a bug in the driver where the
            # overall bond status can be up but none of the actual
            # physical interfaces have a link.
            echo "$bi" | grep 'MII Status:' | tail -n +2 | grep -q '^MII Status: up' || {
                echo "ERROR: No active slaves for 802.ad bond device $realiface"
                mark_down $iface
                continue
            }
        }
        mark_up $iface
        continue
        }
        
        /*如果不是bond，则走下面的逻辑*/
        case $iface in
        lo*)
        # loopback is always working
        mark_up $iface
        ;;
        ib*)
        # we dont know how to test ib links
        mark_up $iface
        ;;
        *)
        /*通过ethtool检测Link detected来判断网卡是否up*/
        [ -z "$iface" ] || {
            [ "$(basename $(readlink /sys/class/net/$iface/device/driver) 2>/dev/null)" = virtio_net ] ||
            ethtool $iface | grep -q 'Link detected: yes' || {
            # On some systems, this is not successful when a
            # cable is plugged but the interface has not been
            # brought up previously. Bring the interface up and                                                                                        
            # try again...
            ip link set $iface up
            ethtool $iface | grep -q 'Link detected: yes' || {
                echo "ERROR: No link on the public network interface $iface"
                mark_down $iface
                continue
            }
            }
            mark_up $iface
        }
        ;;
        esac

    done

    /*考虑多张网卡绑定的bond模式，分一下场景考虑：
     * 所有网卡都OK，那么自然$fail = False,直接返回0 
     * 如果存在某张网卡down，那么$fail =True，那么不返回，而是执行下面的一句
     * 只要up_interface_found True,表示bond中至少有1个可用，如果CTDB_PARTIALLY_ONLINE_INTERFACES为yes，
     * 仍然返回0，否则返回1,表示该node UNHEALTHY/
    $fail || return 0

    $up_interfaces_found && \
        [ "$CTDB_PARTIALLY_ONLINE_INTERFACES" = "yes" ] && \
        return 0

    return 1
}

+ ctdb_check_args monitor
+ case "$1" in
+ case "$1" in
+ monitor_interfaces
+ get_all_interfaces
++ sed -e 's/^[^\t ]*[\t ]*//' -e 's/,/ /g' -e 's/[\t ]*$//' /etc/ctdb/public_addresses
+ all_interfaces=
+ '[' bond0 ']'
+ all_interfaces='bond0 '
+ '[' '' ']'
++ ctdb -Y ip -v
++ sed -e 1d -e 's/:[^:]*:$//' -e 's/^.*://' -e 's/,/ /g'
+ ctdb_ifaces='bond0
bond0
bond0'
++ echo bond0 bond0 bond0 bond0
++ sort -u
++ tr ' ' '\n'
+ all_interfaces=bond0
+ delete_unexpected_ips
+ '[' '' = yes ']'
+ return
+ fail=false
+ up_interfaces_found=false
+ for iface in '$all_interfaces'
+ ip addr show bond0
++ echo bond0
++ sed -e 's/\..*$//'
+ realiface=bond0
++ get_proc net/bonding/bond0
+ bi='Ethernet Channel Bonding Driver: v3.7.1 (April 27, 2011)

Bonding Mode: fault-tolerance (active-backup)
Primary Slave: None
Currently Active Slave: eth0
MII Status: up
MII Polling Interval (ms): 100
Up Delay (ms): 0
Down Delay (ms): 0

Slave Interface: eth0
MII Status: up
Speed: 1000 Mbps
Duplex: full
Link Failure Count: 0
Permanent HW addr: 00:0c:29:77:bd:53
Slave queue ID: 0'
+ echo 'Ethernet Channel Bonding Driver: v3.7.1 (April 27, 2011)

Bonding Mode: fault-tolerance (active-backup)
Primary Slave: None
Currently Active Slave: eth0
MII Status: up
MII Polling Interval (ms): 100
Up Delay (ms): 0
Down Delay (ms): 0

Slave Interface: eth0
MII Status: up
Speed: 1000 Mbps
Duplex: full
Link Failure Count: 0
Permanent HW addr: 00:0c:29:77:bd:53
Slave queue ID: 0'
+ grep -q 'Currently Active Slave: None'
+ echo 'Ethernet Channel Bonding Driver: v3.7.1 (April 27, 2011)

Bonding Mode: fault-tolerance (active-backup)
Primary Slave: None
Currently Active Slave: eth0
MII Status: up
MII Polling Interval (ms): 100
Up Delay (ms): 0
Down Delay (ms): 0

Slave Interface: eth0
MII Status: up
Speed: 1000 Mbps
Duplex: full
Link Failure Count: 0
Permanent HW addr: 00:0c:29:77:bd:53
Slave queue ID: 0'
+ grep -q '^MII Status: up'
+ echo 'Ethernet Channel Bonding Driver: v3.7.1 (April 27, 2011)

Bonding Mode: fault-tolerance (active-backup)
Primary Slave: None
Currently Active Slave: eth0
MII Status: up
MII Polling Interval (ms): 100
Up Delay (ms): 0
Down Delay (ms): 0

Slave Interface: eth0
MII Status: up
Speed: 1000 Mbps
Duplex: full
Link Failure Count: 0
Permanent HW addr: 00:0c:29:77:bd:53
Slave queue ID: 0'
+ grep -q '^Bonding Mode: IEEE 802.3ad Dynamic link aggregation'
+ mark_up bond0
+ up_interfaces_found=true
+ ctdb setifacelink bond0 up
+ continue
+ false
+ return 0
+ exit 0