Problems with initial install and setup of 20.08

Derelict · Oct 28, 2020, 8:20 PM

Looks at /var/log/messages to see what is happening with the IPsec connection.

KenRunner · Oct 28, 2020, 8:24 PM

Here is a copy of the startup_db file from node 1:

<config>
   <dataplane-config xmlns="urn:netgate:xml:yang:netgate-dataplane">
      <dpdk>
         <dev>
            <id>0000:02:00.0</id>
            <name>LAN</name>
         </dev>
         <dev>
            <id>0000:07:00.0</id>
            <name>WAN</name>
         </dev>
         <uio-driver>igb_uio</uio-driver>
      </dpdk>
   </dataplane-config>
   <interfaces-config xmlns="urn:netgate:xml:yang:netgate-interface">
      <interface>
         <name>LAN</name>
         <description><![CDATA[LAN]]></description>
         <enabled>true</enabled>
         <ipv4>
            <address>
               <ip>10.5.5.1/24</ip>
            </address>
         </ipv4>
      </interface>
      <interface>
         <name>WAN</name>
         <description><![CDATA[WAN]]></description>
         <enabled>true</enabled>
         <ipv4>
            <address>
               <ip>10.0.0.1/30</ip>
            </address>
         </ipv4>
      </interface>
      <interface>
         <name>ipip0</name>
         <enabled>true</enabled>
         <ipv4>
            <address>
               <ip>10.30.0.1/30</ip>
            </address>
         </ipv4>
      </interface>
   </interfaces-config>
   <ipsec-config xmlns="urn:netgate:xml:yang:netgate-ipsec">
      <tunnel>
         <instance>0</instance>
         <local-addr>10.0.0.1</local-addr>
         <remote-addr>10.0.0.2</remote-addr>
         <crypto>
            <config-type>ike</config-type>
            <ike>
               <version>2</version>
               <lifetime>28800</lifetime>
               <proposals>
                  <name>1</name>
                  <encryption-algorithm>aes256</encryption-algorithm>
                  <integrity-algorithm>sha256</integrity-algorithm>
                  <dh-group>modp2048</dh-group>
               </proposals>
               <identity>
                  <peer>local</peer>
                  <type>address</type>
                  <value>10.0.0.1</value>
               </identity>
               <identity>
                  <peer>remote</peer>
                  <type>address</type>
                  <value>10.0.0.2</value>
               </identity>
               <authentication>
                  <peer>local</peer>
                  <round>
                     <number>1</number>
                     <type>psk</type>
                     <psk>1234567890</psk>
                  </round>
               </authentication>
               <authentication>
                  <peer>remote</peer>
                  <round>
                     <number>1</number>
                     <type>psk</type>
                     <psk>1234567890</psk>
                  </round>
               </authentication>
               <child-sa>
                  <name>1</name>
                  <lifetime>3600</lifetime>
                  <proposal>
                     <name>1</name>
                     <encryption-algorithm>aes256</encryption-algorithm>
                     <integrity-algorithm>sha256</integrity-algorithm>
                     <dh-group>modp2048</dh-group>
                  </proposal>
               </child-sa>
            </ike>
         </crypto>
      </tunnel>
   </ipsec-config>
   <route-table-config xmlns="urn:netgate:xml:yang:netgate-route-table">
      <static-routes>
         <route-table>
            <name>ipv4-VRF:0</name>
            <address-family>ipv4</address-family>
            <ipv4-routes>
               <route>
                  <destination-prefix>10.10.10.0/24</destination-prefix>
                  <next-hop>
                     <hop>
                        <hop-id>0</hop-id>
                        <ipv4-address>10.30.0.2</ipv4-address>
                     </hop>
                  </next-hop>
               </route>
            </ipv4-routes>
         </route-table>
      </static-routes>
   </route-table-config>
   <nacm xmlns="urn:ietf:params:xml:ns:yang:ietf-netconf-acm">
      <enable-nacm>true</enable-nacm>
      <read-default>deny</read-default>
      <write-default>deny</write-default>
      <exec-default>deny</exec-default>
      <groups>
         <group>
            <name>admin</name>
            <user-name>root</user-name>
            <user-name>tnsr</user-name>
         </group>
      </groups>
      <rule-list>
         <name>admin-rules</name>
         <group>admin</group>
         <rule>
            <name>permit-all</name>
            <module-name>*</module-name>
            <access-operations>*</access-operations>
            <action>permit</action>
         </rule>
      </rule-list>
   </nacm>
   <modules-state xmlns="urn:ietf:params:xml:ns:yang:ietf-yang-library">
      <module-set-id>20.08</module-set-id>
      <module>
         <name>clixon-lib</name>
         <revision>2020-04-23</revision>
         <namespace>http://clicon.org/lib</namespace>
      </module>
      <module>
         <name>clixon-rfc5277</name>
         <revision>2008-07-01</revision>
         <namespace>urn:ietf:params:xml:ns:netmod:notification</namespace>
      </module>
      <module>
         <name>ietf-inet-types</name>
         <revision>2013-07-15</revision>
         <namespace>urn:ietf:params:xml:ns:yang:ietf-inet-types</namespace>
      </module>
      <module>
         <name>ietf-netconf</name>
         <revision>2011-06-01</revision>
         <namespace>urn:ietf:params:xml:ns:netconf:base:1.0</namespace>
      </module>
      <module>
         <name>ietf-netconf-acm</name>
         <revision>2018-02-14</revision>
         <namespace>urn:ietf:params:xml:ns:yang:ietf-netconf-acm</namespace>
      </module>
      <module>
         <name>ietf-restconf</name>
         <revision>2017-01-26</revision>
         <namespace>urn:ietf:params:xml:ns:yang:ietf-restconf</namespace>
      </module>
      <module>
         <name>ietf-yang-library</name>
         <revision>2016-06-21</revision>
         <namespace>urn:ietf:params:xml:ns:yang:ietf-yang-library</namespace>
      </module>
      <module>
         <name>ietf-yang-types</name>
         <revision>2013-07-15</revision>
         <namespace>urn:ietf:params:xml:ns:yang:ietf-yang-types</namespace>
      </module>
      <module>
         <name>netgate-acl</name>
         <revision>2020-06-15</revision>
         <namespace>urn:netgate:xml:yang:netgate-acl</namespace>
      </module>
      <module>
         <name>netgate-bfd</name>
         <revision>2020-06-15</revision>
         <namespace>urn:netgate:xml:yang:netgate-bfd</namespace>
      </module>
      <module>
         <name>netgate-bgp</name>
         <revision>2020-06-15</revision>
         <namespace>urn:netgate:xml:yang:netgate-bgp</namespace>
      </module>
      <module>
         <name>netgate-common</name>
         <revision>2020-06-15</revision>
         <namespace>urn:netgate:xml:yang:netgate-common</namespace>
      </module>
      <module>
         <name>netgate-dataplane</name>
         <revision>2020-06-15</revision>
         <namespace>urn:netgate:xml:yang:netgate-dataplane</namespace>
      </module>
      <module>
         <name>netgate-frr</name>
         <revision>2020-06-15</revision>
         <namespace>urn:netgate:xml:yang:netgate-frr</namespace>
      </module>
      <module>
         <name>netgate-frr-types</name>
         <revision>2020-06-15</revision>
         <namespace>urn:netgate:xml:yang:netgate-frr-types</namespace>
      </module>
      <module>
         <name>netgate-gre</name>
         <revision>2020-06-15</revision>
         <namespace>urn:netgate:xml:yang:netgate-gre</namespace>
      </module>
      <module>
         <name>netgate-host</name>
         <revision>2020-06-15</revision>
         <namespace>urn:netgate:xml:yang:netgate-host</namespace>
      </module>
      <module>
         <name>netgate-host-interface</name>
         <revision>2020-06-15</revision>
         <namespace>urn:netgate:xml:yang:netgate-hostif</namespace>
      </module>
      <module>
         <name>netgate-http</name>
         <revision>2020-06-15</revision>
         <namespace>urn:ietf:params:xml:ns:yang:netgate-http</namespace>
      </module>
      <module>
         <name>netgate-interface</name>
         <revision>2020-07-15</revision>
         <namespace>urn:netgate:xml:yang:netgate-interface</namespace>
      </module>
      <module>
         <name>netgate-interface-extensions</name>
         <revision>2020-06-15</revision>
         <namespace>urn:netgate:xml:yang:netgate-ifext</namespace>
      </module>
      <module>
         <name>netgate-ip</name>
         <revision>2020-06-15</revision>
         <namespace>urn:netgate:xml:yang:netgate-ip</namespace>
      </module>
      <module>
         <name>netgate-ipsec</name>
         <revision>2020-06-15</revision>
         <namespace>urn:netgate:xml:yang:netgate-ipsec</namespace>
      </module>
      <module>
         <name>netgate-kea</name>
         <revision>2020-06-15</revision>
         <namespace>urn:netgate:xml:yang:netgate-kea</namespace>
      </module>
      <module>
         <name>netgate-lldp</name>
         <revision>2020-06-15</revision>
         <namespace>urn:netgate:xml:yang:netgate-lldp</namespace>
      </module>
      <module>
         <name>netgate-macip</name>
         <revision>2020-06-15</revision>
         <namespace>urn:netgate:xml:yang:netgate-macip</namespace>
      </module>
      <module>
         <name>netgate-map</name>
         <revision>2020-06-15</revision>
         <namespace>urn:netgate:xml:yang:netgate-map</namespace>
      </module>
      <module>
         <name>netgate-master</name>
         <revision>2020-06-15</revision>
         <namespace>urn:netgate:xml:yang:netgate-master</namespace>
      </module>
      <module>
         <name>netgate-nat</name>
         <revision>2020-06-15</revision>
         <namespace>urn:netgate:xml:yang:netgate-nat</namespace>
      </module>
      <module>
         <name>netgate-neighbor</name>
         <revision>2020-06-15</revision>
         <namespace>urn:netgate:xml:yang:netgate-neighbor</namespace>
      </module>
      <module>
         <name>netgate-ntp</name>
         <revision>2020-06-15</revision>
         <namespace>urn:netgate:xml:yang:netgate-ntp</namespace>
      </module>
      <module>
         <name>netgate-ospf</name>
         <revision>2020-06-15</revision>
         <namespace>urn:netgate:xml:yang:netgate-ospf</namespace>
      </module>
      <module>
         <name>netgate-ospf6</name>
         <revision>2020-06-15</revision>
         <namespace>urn:netgate:xml:yang:netgate-ospf6</namespace>
      </module>
      <module>
         <name>netgate-package</name>
         <revision>2020-06-15</revision>
         <namespace>urn:netgate:xml:yang:netgate-package</namespace>
      </module>
      <module>
         <name>netgate-pki</name>
         <revision>2020-06-15</revision>
         <namespace>urn:netgate:xml:yang:netgate-pki</namespace>
      </module>
      <module>
         <name>netgate-rip</name>
         <revision>2020-06-15</revision>
         <namespace>urn:netgate:xml:yang:netgate-rip</namespace>
      </module>
      <module>
         <name>netgate-route</name>
         <revision>2020-06-15</revision>
         <namespace>urn:netgate:xml:yang:netgate-route</namespace>
      </module>
      <module>
         <name>netgate-route-table</name>
         <revision>2020-07-15</revision>
         <namespace>urn:netgate:xml:yang:netgate-route-table</namespace>
      </module>
      <module>
         <name>netgate-snmp</name>
         <revision>2020-06-15</revision>
         <namespace>https://netgate.com/ns/netgate-snmp</namespace>
      </module>
      <module>
         <name>netgate-span</name>
         <revision>2020-06-15</revision>
         <namespace>urn:netgate:xml:yang:netgate-span</namespace>
      </module>
      <module>
         <name>netgate-ssh-server</name>
         <revision>2020-06-15</revision>
         <namespace>urn:netgate:xml:yang:netgate-ssh-server</namespace>
      </module>
      <module>
         <name>netgate-sysctl</name>
         <revision>2020-06-15</revision>
         <namespace>urn:netgate:xml:yang:netgate-sysctl</namespace>
      </module>
      <module>
         <name>netgate-system</name>
         <revision>2020-06-15</revision>
         <namespace>urn:netgate:xml:yang:netgate-system</namespace>
      </module>
      <module>
         <name>netgate-unbound</name>
         <revision>2020-06-15</revision>
         <namespace>urn:netgate:xml:yang:netgate-unbound</namespace>
      </module>
      <module>
         <name>netgate-vpp-prometheus</name>
         <revision>2020-07-30</revision>
         <namespace>urn:netgate:xml:yang:netgate-vpp-prometheus</namespace>
      </module>
      <module>
         <name>netgate-vrrp</name>
         <revision>2020-06-15</revision>
         <namespace>urn:netgate:xml:yang:netgate-vrrp</namespace>
      </module>
      <module>
         <name>netgate-vxlan</name>
         <revision>2020-06-15</revision>
         <namespace>urn:netgate:xml:yang:netgate-vxlan</namespace>
      </module>
   </modules-state>
</config>

KenRunner · Oct 28, 2020, 8:29 PM

/var/log/messages from node 1:

Oct 28 15:23:56 tnsr-test1 clixon_backend[2043]: ipsec_job_child_bringup_tunnel: Initiating tunnel 0
Oct 28 15:23:56 tnsr-test1 charon-systemd[2006]: vici initiate CHILD_SA 'child0'
Oct 28 15:24:08 tnsr-test1 systemd[2219]: Starting Mark boot as successful...
Oct 28 15:24:08 tnsr-test1 systemd[2219]: Started Mark boot as successful.
Oct 28 15:24:26 tnsr-test1 charon-systemd[2006]: retransmit 5 of request with message ID 0
Oct 28 15:24:26 tnsr-test1 charon-systemd[2006]: sending packet: from 10.0.0.1[500] to 10.0.0.2[500] (464 bytes)
Oct 28 15:24:26 tnsr-test1 clixon_backend[2043]: ipsec_job_child_bringup_tunnel: Initiating tunnel 0
Oct 28 15:24:26 tnsr-test1 charon-systemd[2006]: vici initiate CHILD_SA 'child0'
Oct 28 15:24:29 tnsr-test1 vnet[1534]: linux-cp/router: Failed to delete neighbor: 10.0.0.2 WAN
Oct 28 15:24:56 tnsr-test1 clixon_backend[2043]: ipsec_job_child_bringup_tunnel: Initiating tunnel 0
Oct 28 15:24:56 tnsr-test1 charon-systemd[2006]: vici initiate CHILD_SA 'child0'
Oct 28 15:25:26 tnsr-test1 clixon_backend[2043]: ipsec_job_child_bringup_tunnel: Initiating tunnel 0
Oct 28 15:25:26 tnsr-test1 charon-systemd[2006]: vici initiate CHILD_SA 'child0'
Oct 28 15:25:41 tnsr-test1 charon-systemd[2006]: giving up after 5 retransmits
Oct 28 15:25:41 tnsr-test1 charon-systemd[2006]: establishing IKE_SA failed, peer not responding
Oct 28 15:25:56 tnsr-test1 clixon_backend[2043]: ipsec_job_child_bringup_tunnel: Initiating tunnel 0
Oct 28 15:25:56 tnsr-test1 charon-systemd[2006]: vici initiate CHILD_SA 'child0'
Oct 28 15:25:56 tnsr-test1 charon-systemd[2006]: initiating IKE_SA ipip0[28] to 10.0.0.2
Oct 28 15:25:56 tnsr-test1 charon-systemd[2006]: generating IKE_SA_INIT request 0 [ SA KE No N(NATD_S_IP) N(NATD_D_IP) N(FRAG_SUP) N(HASH_ALG) N(REDIR_SUP) ]
Oct 28 15:25:56 tnsr-test1 charon-systemd[2006]: sending packet: from 10.0.0.1[500] to 10.0.0.2[500] (464 bytes)
Oct 28 15:25:59 tnsr-test1 vnet[1534]: linux-cp/router: Failed to delete neighbor: 10.0.0.2 WAN
Oct 28 15:26:00 tnsr-test1 charon-systemd[2006]: retransmit 1 of request with message ID 0
Oct 28 15:26:00 tnsr-test1 charon-systemd[2006]: sending packet: from 10.0.0.1[500] to 10.0.0.2[500] (464 bytes)
Oct 28 15:26:03 tnsr-test1 vnet[1534]: linux-cp/router: Failed to delete neighbor: 10.0.0.2 WAN

Derelict · Oct 29, 2020, 2:52 AM

@KenRunner said in Problems with initial install and setup of 20.08:

Oct 28 15:25:41 tnsr-test1 charon-systemd[2006]: giving up after 5 retransmits
Oct 28 15:25:41 tnsr-test1 charon-systemd[2006]: establishing IKE_SA failed, peer not responding

Looks like the peer at 10.0.0.2 is not responding to the ISAKMP packets being sent. What is being logged on the other side?

KenRunner · Oct 30, 2020, 3:52 PM

node2 10.0.0.2 /var/log/messages:

Oct 30 10:47:38 tnsr-test2 charon-systemd[1992]: initiating IKE_SA ipip0[898] to 10.0.0.1
Oct 30 10:47:38 tnsr-test2 charon-systemd[1992]: generating IKE_SA_INIT request 0 [ SA KE No N(NATD_S_IP) N(NATD_D_IP) N(FRAG_SUP) N(HASH_ALG) N(REDIR_SUP) ]
Oct 30 10:47:38 tnsr-test2 charon-systemd[1992]: sending packet: from 10.0.0.2[500] to 10.0.0.1[500] (464 bytes)
Oct 30 10:47:42 tnsr-test2 vnet[1550]: linux-cp/router: Failed to delete neighbor: 10.0.0.1 WAN
Oct 30 10:47:42 tnsr-test2 charon-systemd[1992]: retransmit 1 of request with message ID 0
Oct 30 10:47:42 tnsr-test2 charon-systemd[1992]: sending packet: from 10.0.0.2[500] to 10.0.0.1[500] (464 bytes)
Oct 30 10:47:46 tnsr-test2 vnet[1550]: linux-cp/router: Failed to delete neighbor: 10.0.0.1 WAN
Oct 30 10:47:50 tnsr-test2 charon-systemd[1992]: retransmit 2 of request with message ID 0
Oct 30 10:47:50 tnsr-test2 charon-systemd[1992]: sending packet: from 10.0.0.2[500] to 10.0.0.1[500] (464 bytes)
Oct 30 10:47:53 tnsr-test2 vnet[1550]: linux-cp/router: Failed to delete neighbor: 10.0.0.1 WAN
Oct 30 10:48:03 tnsr-test2 charon-systemd[1992]: retransmit 3 of request with message ID 0
Oct 30 10:48:03 tnsr-test2 charon-systemd[1992]: sending packet: from 10.0.0.2[500] to 10.0.0.1[500] (464 bytes)
Oct 30 10:48:06 tnsr-test2 vnet[1550]: linux-cp/router: Failed to delete neighbor: 10.0.0.1 WAN
Oct 30 10:48:08 tnsr-test2 clixon_backend[2029]: ipsec_job_child_bringup_tunnel: Initiating tunnel 0
Oct 30 10:48:08 tnsr-test2 charon-systemd[1992]: vici initiate CHILD_SA 'child0'
Oct 30 10:48:26 tnsr-test2 charon-systemd[1992]: retransmit 4 of request with message ID 0
Oct 30 10:48:26 tnsr-test2 charon-systemd[1992]: sending packet: from 10.0.0.2[500] to 10.0.0.1[500] (464 bytes)
Oct 30 10:48:29 tnsr-test2 vnet[1550]: linux-cp/router: Failed to delete neighbor: 10.0.0.1 WAN
Oct 30 10:48:38 tnsr-test2 clixon_backend[2029]: ipsec_job_child_bringup_tunnel: Initiating tunnel 0
Oct 30 10:48:38 tnsr-test2 charon-systemd[1992]: vici initiate CHILD_SA 'child0'
Oct 30 10:49:08 tnsr-test2 charon-systemd[1992]: retransmit 5 of request with message ID 0
Oct 30 10:49:08 tnsr-test2 charon-systemd[1992]: sending packet: from 10.0.0.2[500] to 10.0.0.1[500] (464 bytes)
Oct 30 10:49:08 tnsr-test2 clixon_backend[2029]: ipsec_job_child_bringup_tunnel: Initiating tunnel 0
Oct 30 10:49:08 tnsr-test2 charon-systemd[1992]: vici initiate CHILD_SA 'child0'
Oct 30 10:49:11 tnsr-test2 vnet[1550]: linux-cp/router: Failed to delete neighbor: 10.0.0.1 WAN

Derelict · Oct 30, 2020, 3:56 PM

On both nodes can you:

tnsr# show interface
tnsr# ping 10.0.0.1 source 10.0.0.2 and the reciprocal on the other node
tnsr# show neighbor
tnsr# show ipsec tunnel X where X is the ipsec instance

?

KenRunner · Oct 30, 2020, 5:41 PM

Results from the ping and status check:

tnsr-test2 tnsr# ping 10.0.0.1 source 10.0.0.2
PING 10.0.0.1 (10.0.0.1) from 10.0.0.2 : 56(84) bytes of data.
From 10.0.0.2 icmp_seq=1 Destination Host Unreachable
From 10.0.0.2 icmp_seq=2 Destination Host Unreachable
From 10.0.0.2 icmp_seq=3 Destination Host Unreachable

--- 10.0.0.1 ping statistics ---
3 packets transmitted, 0 received, +3 errors, 100% packet loss, time 27ms
pipe 3
tnsr-test2 tnsr# show neighbor
tnsr-test2 tnsr# show ipsec tunnel 0
IPsec Tunnel: 0
    IKE SA: ipip0    ID: 935    Version: IKEv2
        Local: 10.0.0.2[500]    Remote: 10.0.0.1[500]
        Status: CONNECTING
tnsr-test2 tnsr#

Both nodes gave the same responses

Derelict · Oct 30, 2020, 5:44 PM

It looks like there is no connectivity between those two hosts. They can't even ARP for each other.

What about show interface ??

KenRunner · Oct 30, 2020, 5:47 PM

The nodes are both connected to a switch and are on the same vlan. Here is the show interface from number 2:

tnsr-test2 tnsr# show interface
Interface: LAN
    Description: LAN
    Admin status: up
    Link up, link-speed 1000 Mbps, full duplex
    Link MTU: 9000 bytes
    MAC address: 0c:c4:7a:4c:8a:cc
    IPv4 MTU: 0 bytes
    IPv4 Route Table: ipv4-VRF:0
    IPv4 addresses:
        10.10.10.1/24
    IPv6 MTU: 0 bytes
    IPv6 Route Table: ipv6-VRF:0
    IPv6 addresses:
        fe80::ec4:7aff:fe4c:8acc/64
    VLAN tag rewrite: disable
    Rx-queues
        queue-id 0 : cpu-id 1
    counters:
      received: 6398619 bytes, 87388 packets, 0 errors
      transmitted: 86184 bytes, 1012 packets, 8 errors
      protocols: 0 IPv4, 0 IPv6
      87388 drops, 0 punts, 0 rx miss, 0 rx no buffer

Interface: WAN
    Description: WAN
    Admin status: up
    Link down, unknown duplex
    Link MTU: 9000 bytes
    MAC address: 0c:c4:7a:4c:86:e4
    IPv4 MTU: 0 bytes
    IPv4 Route Table: ipv4-VRF:0
    IPv4 addresses:
        10.0.0.2/30
    IPv6 MTU: 0 bytes
    IPv6 Route Table: ipv6-VRF:0
    IPv6 addresses:
        fe80::ec4:7aff:fe4c:86e4/64
    VLAN tag rewrite: disable
    Rx-queues
        queue-id 0 : cpu-id 1
    counters:
      received: 0 bytes, 0 packets, 0 errors
      transmitted: 0 bytes, 0 packets, 17909 errors
      protocols: 0 IPv4, 0 IPv6
      0 drops, 0 punts, 0 rx miss, 0 rx no buffer

Interface: ipip0
    Admin status: up
    Link up, unknown duplex
    Link MTU: 9000 bytes
    IPv4 MTU: 0 bytes
    IPv4 Route Table: ipv4-VRF:0
    IPv4 addresses:
        10.30.0.2/30
    IPv6 MTU: 0 bytes
    IPv6 Route Table: ipv6-VRF:0
    IPv6 addresses:
        fe80::d167:2cf6:12d4:497b/64
    VLAN tag rewrite: disable
    counters:
      received: 0 bytes, 0 packets, 0 errors
      transmitted: 68 bytes, 1 packets, 0 errors
      protocols: 0 IPv4, 0 IPv6
      0 drops, 0 punts, 0 rx miss, 0 rx no buffer

Derelict · Oct 30, 2020, 5:52 PM

How are they connected? They don't appear to be able to exchange traffic between each other. Nothing but transmit errors on WAN there.