Monday, July 14, 2014

RDO Setup Two Real Node (Controller+Compute) IceHouse Neutron ML2&OVS&VLAN Cluster on CentOS 7

As of 07/14/2014 Bug https://ask.openstack.org/en/question/35705/attempt-of-rdo-aio-install-icehouse-on-centos-7/ is still pending and workaround suggested above should be applied during two node RDO packstack installation.
  Successful implementation of Neutron ML2&&OVS&&VLAN multi node setup requires correct version of plugin.ini -> /etc/neutron/plugins/ml2/ml2_conf.ini which appears to be generated with errors by packstack.

Two boxes  have been setup , each one having 2  NICs (enp2s0,enp5s1) for
Controller && Compute Nodes setup. Before running
`packstack --answer-file=TwoRealNode Neutron ML2&OVS&VLAN.txt` SELINUX set to permissive on both nodes.Both enp5s1's assigned IPs and set to promiscuous mode (192.168.0.127, 192.168.0.137 ). Services firewalld and NetworkManager disabled, IPv4 firewall with iptables and service network are enabled and running. Packstack is bind to public IP of interface enp2s0 192.169.1.127, Compute Node is 192.169.1.137 ( view answer-file ).

Setup configuration

- Controller node: Nova, Keystone, Cinder, Glance, Neutron (using Open vSwitch plugin && VLAN )
- Compute node: Nova (nova-compute), Neutron (openvswitch-agent)


icehouse1.localdomain   -  Controller (192.168.1.127)
icehouse2.localdomain   -  Compute   (192.168.1.137)

Status after packstack install and updating /etc/neutron/plugin.ini as shown bellow

[root@icehouse1 neutron]# cat plugin.ini
[ml2]
type_drivers = vlan
tenant_network_types = vlan
mechanism_drivers = openvswitch
[ml2_type_vlan]
network_vlan_ranges = physnet1:100:200
[ovs]
network_vlan_ranges = physnet1:100:200
tenant_network_type = vlan
enable_tunneling = False
integration_bridge = br-int
bridge_mappings = physnet1:br-enp5s1
local_ip = 192.168.1.127
[AGENT]
polling_interval = 2
[SECURITYGROUP]
firewall_driver = neutron.agent.linux.iptables_firewall.OVSHybridIptablesFirewallDriver


[root@icehouse1 ~(keystone_admin)]# ls -l /etc/neutron
total 60
-rw-r--r--. 1 root root      193 Jul 13 09:06 api-paste.ini
-rw-r-----. 1 root neutron  3853 Jul 13 09:05 dhcp_agent.ini
-rw-r-----. 1 root neutron   208 Jul 13 09:06 fwaas_driver.ini
-rw-r-----. 1 root neutron  3431 Jul 13 09:06 l3_agent.ini
-rw-r-----. 1 root neutron  1400 Jun  8 01:38 lbaas_agent.ini
-rw-r-----. 1 root neutron  1481 Jul 13 09:06 metadata_agent.ini
-rw-r-----. 1 root neutron 19148 Jul 13 09:06 neutron.conf
lrwxrwxrwx. 1 root root       37 Jul 13 09:06 plugin.ini -> /etc/neutron/plugins/ml2/ml2_conf.ini
drwxr-xr-x. 4 root root       34 Jul 13 09:06 plugins
-rw-r-----. 1 root neutron  6148 Jun  8 01:38 policy.json
-rw-r--r--. 1 root root       78 Jun 24 15:14 release
-rw-r--r--. 1 root root     1216 Jun  8 01:38 rootwrap.conf

# systemctl disable NetworkManager
# systemctl stop  NetworkManager
# chkconfig network on 

Assigning IP to OVS bridge br-ex and making enp2s0 OVS port of br-ex :-

[root@icehouse1 network-scripts(keystone_admin)]# cat ifcfg-br-ex
DEVICE="br-ex"
BOOTPROTO="static"
IPADDR="192.168.1.127"
NETMASK="255.255.255.0"
DNS1="83.221.202.254"
BROADCAST="192.168.1.255"
GATEWAY="192.168.1.1"
NM_CONTROLLED="no"
DEFROUTE="yes"
IPV4_FAILURE_FATAL="yes"
IPV6INIT=no
ONBOOT="yes"
TYPE="OVSBridge"
DEVICETYPE="ovs"

[root@icehouse1 network-scripts(keystone_admin)]# cat ifcfg-enp2s0
DEVICE=enp2s0
ONBOOT="yes"
TYPE="OVSPort"
DEVICETYPE="ovs"
OVS_BRIDGE=br-ex
NM_CONTROLLED=no
IPV6INIT=no

# service network restart

[root@icehouse1 ~(keystone_admin)]# openstack-status
== Nova services ==
openstack-nova-api:                     active
openstack-nova-cert:                    active
openstack-nova-compute:                 inactive  (disabled on boot)
openstack-nova-network:                 inactive  (disabled on boot)
openstack-nova-scheduler:               active
openstack-nova-volume:                  inactive  (disabled on boot)
openstack-nova-conductor:               active
== Glance services ==
openstack-glance-api:                   active
openstack-glance-registry:              active
== Keystone service ==
openstack-keystone:                     active
== Horizon service ==
openstack-dashboard:                    active
== neutron services ==
neutron-server:                         active
neutron-dhcp-agent:                     active
neutron-l3-agent:                       active
neutron-metadata-agent:                 active
neutron-lbaas-agent:                    inactive  (disabled on boot)
neutron-openvswitch-agent:              active
neutron-linuxbridge-agent:              inactive  (disabled on boot)
neutron-ryu-agent:                      inactive  (disabled on boot)
neutron-nec-agent:                      inactive  (disabled on boot)
neutron-mlnx-agent:                     inactive  (disabled on boot)
== Cinder services ==
openstack-cinder-api:                   active
openstack-cinder-scheduler:             active
openstack-cinder-volume:                active
openstack-cinder-backup:                inactive  (disabled on boot)
== Support services ==
libvirtd:                               active
openvswitch:                            active
dbus:                                   active
tgtd:                                   inactive  (disabled on boot)
rabbitmq-server:                        active
memcached:                              active
== Keystone users ==
+----------------------------------+------------+---------+----------------------+
|                id                |    name    | enabled |        email         |
+----------------------------------+------------+---------+----------------------+
| d8a581fed7004a73b38ccdccfc3bc7ab |   admin    |   True  |    test@test.com     |
| d4c7dd1c19a54a89a084680b0a301cba |   boris    |   True  |                      |
| 1cf40879e9e74e8b893fb617ecd8a6fc | ceilometer |   True  | ceilometer@localhost |
| 84d41c99fa944eaea81ab5107d360d12 |   cinder   |   True  |   cinder@localhost   |
| 928a6cceed90448fbb93e78ddad4c3f8 |    demo    |   True  |                      |
| b9ff3b2044d14d70944d0af717aff468 |   glance   |   True  |   glance@localhost   |
| fd12b31ef7794fa2a57498e5195747eb |  neutron   |   True  |  neutron@localhost   |
| 4b3859e66b2f4fc9b924bc8f718b9b2f |    nova    |   True  |    nova@localhost    |
| c9921012c6d543c8ba109a2726599826 |   undef    |   True  |                      |
+----------------------------------+------------+---------+----------------------+
== Glance images ==
+--------------------------------------+-----------------+-------------+------------------+-----------+--------+
| ID                                   | Name            | Disk Format | Container Format | Size      | Status |
+--------------------------------------+-----------------+-------------+------------------+-----------+--------+
| 72dbae96-62b5-4011-ab3a-bb45872107c4 | cirros          | qcow2       | bare             | 13147648  | active |
| 94a45bd6-1872-4f1a-8276-6679329b063e | Ubuntu 07/12/14 | qcow2       | bare             | 254738944 | active |
| 70fc1c75-c8f9-4a3b-834c-302f8430aaa0 | VF20image       | qcow2       | bare             | 210829312 | active |
+--------------------------------------+-----------------+-------------+------------------+-----------+--------+
== Nova managed services ==
+------------------+-----------------------+----------+---------+-------+----------------------------+-----------------+
| Binary           | Host                  | Zone     | Status  | State | Updated_at                 | Disabled Reason |
+------------------+-----------------------+----------+---------+-------+----------------------------+-----------------+
| nova-consoleauth | icehouse1.localdomain | internal | enabled | up    | 2014-07-14T08:29:55.000000 | -               |
| nova-scheduler   | icehouse1.localdomain | internal | enabled | up    | 2014-07-14T08:29:55.000000 | -               |
| nova-conductor   | icehouse1.localdomain | internal | enabled | up    | 2014-07-14T08:29:52.000000 | -               |
| nova-cert        | icehouse1.localdomain | internal | enabled | up    | 2014-07-14T08:29:55.000000 | -               |
| nova-compute     | icehouse2.localdomain | nova     | enabled | up    | 2014-07-14T08:29:51.000000 | -               |
+------------------+-----------------------+----------+---------+-------+----------------------------+-----------------+
== Nova networks ==
+--------------------------------------+----------+------+
| ID                                   | Label    | Cidr |
+--------------------------------------+----------+------+
| 702e1fe7-62b5-47c8-95fc-a169231a5dde | public   | -    |
| 430a0ef1-7fc8-40d4-9d25-d69e068b1c98 | private  | -    |
| 1722093c-1852-4eab-a76f-1f9e135880a8 | internal | -    |
+--------------------------------------+----------+------+
== Nova instance flavors ==
+----+-----------+-----------+------+-----------+------+-------+-------------+-----------+
| ID | Name      | Memory_MB | Disk | Ephemeral | Swap | VCPUs | RXTX_Factor | Is_Public |
+----+-----------+-----------+------+-----------+------+-------+-------------+-----------+
| 1  | m1.tiny   | 512       | 1    | 0         |      | 1     | 1.0         | True      |
| 2  | m1.small  | 2048      | 20   | 0         |      | 1     | 1.0         | True      |
| 3  | m1.medium | 4096      | 40   | 0         |      | 2     | 1.0         | True      |
| 4  | m1.large  | 8192      | 80   | 0         |      | 4     | 1.0         | True      |
| 5  | m1.xlarge | 16384     | 160  | 0         |      | 8     | 1.0         | True      |
+----+-----------+-----------+------+-----------+------+-------+-------------+-----------+
== Nova instances ==
+----+------+--------+------------+-------------+----------+
| ID | Name | Status | Task State | Power State | Networks |
+----+------+--------+------------+-------------+----------+
+----+------+--------+------------+-------------+----------+

[root@icehouse1 ~(keystone_admin)]# ovs-vsctl show
bf332dbe-de2a-4485-8d76-21e15d44fbcb
    Bridge "br-enp5s1"
        Port "phy-br-enp5s1"
            Interface "phy-br-enp5s1"
        Port "enp5s1"
            Interface "enp5s1"
        Port "br-enp5s1"
            Interface "br-enp5s1"
                type: internal
    Bridge br-int
        Port br-int
            Interface br-int
                type: internal
        Port "qr-61cb5e96-a8"
            tag: 1
            Interface "qr-61cb5e96-a8"
                type: internal
        Port "tap50c5ff29-c0"
            tag: 3
            Interface "tap50c5ff29-c0"
                type: internal
        Port "qr-b4379334-1b"
            tag: 2
            Interface "qr-b4379334-1b"
                type: internal
        Port "qr-6cc14b98-7b"
            tag: 3
            Interface "qr-6cc14b98-7b"
                type: internal
        Port "tapc5a05b89-ae"
            tag: 2
            Interface "tapc5a05b89-ae"
                type: internal
        Port "tapb1808221-e9"
            tag: 1
            Interface "tapb1808221-e9"
                type: internal
        Port "int-br-enp5s1"
            Interface "int-br-enp5s1"
    Bridge br-ex
        Port "qg-877c3024-c7"
            Interface "qg-877c3024-c7"
                type: internal
        Port "qg-c5751b1f-b9"
            Interface "qg-c5751b1f-b9"
                type: internal
        Port br-ex
            Interface br-ex
                type: internal
        Port "enp2s0"
            Interface "enp2s0"
        Port "qg-1ede8ba6-a0"
            Interface "qg-1ede8ba6-a0"
                type: internal
    ovs_version: "2.0.0"

************************
Metadata Trobleshooting
************************

[root@icehouse1 ~(keystone_admin)]# netstat -antp| grep 8775
tcp        0      0 0.0.0.0:8775            0.0.0.0:*               LISTEN      1580/python      
[root@icehouse1 ~(keystone_admin)]# ps -ef | grep 1580
nova      1580     1  0 07:50 ?        00:01:28 /usr/bin/python /usr/bin/nova-api
nova      5192  1580  0 07:51 ?        00:00:00 /usr/bin/python /usr/bin/nova-api
nova      5193  1580  0 07:51 ?        00:00:00 /usr/bin/python /usr/bin/nova-api
nova      5218  1580  0 07:51 ?        00:00:04 /usr/bin/python /usr/bin/nova-api
nova      5219  1580  0 07:51 ?        00:00:05 /usr/bin/python /usr/bin/nova-api
nova      5226  1580  0 07:51 ?        00:00:00 /usr/bin/python /usr/bin/nova-api
nova      5228  1580  0 07:51 ?        00:00:00 /usr/bin/python /usr/bin/nova-api
root     19054 15837  0 12:16 pts/0    00:00:00 grep --color=auto 1580

[root@icehouse1 ~(keystone_admin)]# ip netns
qrouter-9bfc940b-ca64-4e36-9597-188f67ba44a4
qrouter-a3fad1f2-7c02-4952-ba35-fc1e836a1805
qrouter-9d4283e9-8d53-415e-aa5f-427e3ca69da1
qdhcp-12e1088e-1edf-488a-b8c9-0a4085ea2803
qdhcp-1722093c-1852-4eab-a76f-1f9e135880a8
qdhcp-430a0ef1-7fc8-40d4-9d25-d69e068b1c98

[root@icehouse1 ~(keystone_admin)]# ip netns exec qrouter-9bfc940b-ca64-4e36-9597-188f67ba44a4 iptables -S -t nat | grep 169.254
-A neutron-l3-agent-PREROUTING -d 169.254.169.254/32 -p tcp -m tcp --dport 80 -j REDIRECT --to-ports 9697

[root@icehouse1 ~(keystone_admin)]# ip netns exec qrouter-9bfc940b-ca64-4e36-9597-188f67ba44a4 netstat -antp
Active Internet connections (servers and established)
Proto Recv-Q Send-Q Local Address           Foreign Address         State       PID/Program name  
tcp        0      0 0.0.0.0:9697            0.0.0.0:*               LISTEN      6954/python      

[root@icehouse1 ~(keystone_admin)]# ps -ef | grep 6954
root      6954     1  0 07:52 ?        00:00:00 /usr/bin/python /bin/neutron-ns-metadata-proxy --pid_file=/var/lib/neutron/external/pids/9bfc940b-ca64-4e36-9597-188f67ba44a4.pid --metadata_proxy_socket=/var/lib/neutron/metadata_proxy --router_id=9bfc940b-ca64-4e36-9597-188f67ba44a4 --state_path=/var/lib/neutron --metadata_port=9697 --verbose --log-file=neutron-ns-metadata-proxy-9bfc940b-ca64-4e36-9597-188f67ba44a4.log --log-dir=/var/log/neutron
root     19599 15837  0 12:20 pts/0    00:00:00 grep --color=auto 6954