From 7843a67be68bf72d91bf304afb6867f4353af334 Mon Sep 17 00:00:00 2001
From: liuweicai <liuuweicai@gmail.com>
Date: Wed, 28 May 2014 10:07:43 +0800
Subject: [PATCH] Fix listing security group rules
It was failing when there are too many security groups, since we're
hitting a RequestURITooLong exception.
Co-Authored-By: Vincent Untz <vuntz@suse.com>
Co-Authored-By: Ilya Shakhat <ishakhat@mirantis.com>
Change-Id: If94b6a2ed6878efad1224e056ecf43e65e85e821
Closes-Bug: 1271462
---
neutronclient/neutron/v2_0/securitygroup.py | 29 ++++++-
.../tests/unit/test_cli20_securitygroup.py | 97 ++++++++++++++++++++++
2 files changed, 123 insertions(+), 3 deletions(-)
diff --git a/neutronclient/neutron/v2_0/securitygroup.py b/neutronclient/neutron/v2_0/securitygroup.py
index 7e66e8a..af8e8f0 100644
--- a/neutronclient/neutron/v2_0/securitygroup.py
+++ b/neutronclient/neutron/v2_0/securitygroup.py
@@ -17,6 +17,7 @@
import argparse
import logging
+from neutronclient.common import exceptions
from neutronclient.neutron import v2_0 as neutronV20
from neutronclient.openstack.common.gettextutils import _
@@ -141,9 +142,31 @@ class ListSecurityGroupRule(neutronV20.ListCommand):
for rule in data:
for key in self.replace_rules:
sec_group_ids.add(rule[key])
- search_opts.update({"id": sec_group_ids})
- secgroups = neutron_client.list_security_groups(**search_opts)
- secgroups = secgroups.get('security_groups', [])
+ sec_group_ids = list(sec_group_ids)
+
+ def _get_sec_group_list(sec_group_ids):
+ search_opts['id'] = sec_group_ids
+ return neutron_client.list_security_groups(
+ **search_opts).get('security_groups', [])
+
+ try:
+ secgroups = _get_sec_group_list(sec_group_ids)
+ except exceptions.RequestURITooLong as uri_len_exc:
+ # Length of a query filter on security group rule id
+ # id=<uuid>& (with len(uuid)=36)
+ sec_group_id_filter_len = 40
+ # The URI is too long because of too many sec_group_id filters
+ # Use the excess attribute of the exception to know how many
+ # sec_group_id filters can be inserted into a single request
+ sec_group_count = len(sec_group_ids)
+ max_size = ((sec_group_id_filter_len * sec_group_count) -
+ uri_len_exc.excess)
+ chunk_size = max_size / sec_group_id_filter_len
+ secgroups = []
+ for i in range(0, sec_group_count, chunk_size):
+ secgroups.extend(
+ _get_sec_group_list(sec_group_ids[i: i + chunk_size]))
+
sg_dict = dict([(sg['id'], sg['name'])
for sg in secgroups if sg['name']])
for rule in data:
diff --git a/neutronclient/tests/unit/test_cli20_securitygroup.py b/neutronclient/tests/unit/test_cli20_securitygroup.py
index e6127ff..554471d 100644
--- a/neutronclient/tests/unit/test_cli20_securitygroup.py
+++ b/neutronclient/tests/unit/test_cli20_securitygroup.py
@@ -17,7 +17,10 @@
import sys
from mox3 import mox
+import six
+from neutronclient.common import exceptions
+from neutronclient.common import utils
from neutronclient.neutron.v2_0 import securitygroup
from neutronclient.tests.unit import test_cli20
@@ -186,6 +189,100 @@ class CLITestV20SecurityGroupsJSON(test_cli20.CLITestV20Base):
mox.IgnoreArg())
self._test_list_resources(resources, cmd, True)
+ def _test_extend_list(self, mox_calls):
+ resources = "security_groups"
+
+ data = [{'name': 'default',
+ 'security_group_id': 'secgroupid%02d' % i,
+ 'remote_group_id': 'remgroupid%02d' % i}
+ for i in range(10)]
+
+ cmd = securitygroup.ListSecurityGroupRule(
+ test_cli20.MyApp(sys.stdout), None)
+ self.mox.StubOutWithMock(cmd, "get_client")
+ self.mox.StubOutWithMock(self.client.httpclient, "request")
+
+ cmd.get_client().MultipleTimes().AndReturn(self.client)
+ path = getattr(self.client, resources + '_path')
+ mox_calls(path, data)
+ self.mox.ReplayAll()
+ known_args, _vs = cmd.get_parser(
+ 'list' + resources).parse_known_args()
+
+ cmd.extend_list(data, known_args)
+ self.mox.VerifyAll()
+ self.mox.UnsetStubs()
+
+ def _build_test_data(self, data, excess=0):
+ # Length of a query filter on security group rule id
+ # in these testcases, id='secgroupid%02d' (with len(id)=12)
+ sec_group_id_filter_len = 12
+
+ response = []
+ replace_rules = {'security_group_id': 'security_group',
+ 'remote_group_id': 'remote_group'}
+
+ search_opts = {'fields': ['id', 'name']}
+ sec_group_ids = set()
+ for rule in data:
+ for key in replace_rules:
+ sec_group_ids.add(rule[key])
+ response.append({'id': rule[key], 'name': 'default'})
+ sec_group_ids = list(sec_group_ids)
+
+ result = []
+
+ sec_group_count = len(sec_group_ids)
+ max_size = ((sec_group_id_filter_len * sec_group_count) - excess)
+ chunk_size = max_size / sec_group_id_filter_len
+
+ for i in range(0, sec_group_count, chunk_size):
+ search_opts['id'] = sec_group_ids[i: i + chunk_size]
+ params = utils.safe_encode_dict(search_opts)
+ resp_str = self.client.serialize({'security_groups': response})
+
+ result.append({
+ 'filter': six.moves.urllib.parse.urlencode(params, doseq=1),
+ 'response': (test_cli20.MyResp(200), resp_str),
+ })
+
+ return result
+
+ def test_extend_list(self):
+ def mox_calls(path, data):
+ responses = self._build_test_data(data)
+ self.client.httpclient.request(
+ test_cli20.MyUrlComparator(test_cli20.end_url(
+ path, responses[0]['filter']), self.client),
+ 'GET',
+ body=None,
+ headers=mox.ContainsKeyValue(
+ 'X-Auth-Token', test_cli20.TOKEN)).AndReturn(
+ responses[0]['response'])
+
+ self._test_extend_list(mox_calls)
+
+ def test_extend_list_exceed_max_uri_len(self):
+ def mox_calls(path, data):
+ # 1 char of extra URI len will cause a split in 2 requests
+ self.mox.StubOutWithMock(self.client, '_check_uri_length')
+ self.client._check_uri_length(mox.IgnoreArg()).AndRaise(
+ exceptions.RequestURITooLong(excess=1))
+ responses = self._build_test_data(data, excess=1)
+
+ for item in responses:
+ self.client._check_uri_length(
+ mox.IgnoreArg()).AndReturn(None)
+ self.client.httpclient.request(
+ test_cli20.end_url(path, item['filter']),
+ 'GET',
+ body=None,
+ headers=mox.ContainsKeyValue(
+ 'X-Auth-Token', test_cli20.TOKEN)).AndReturn(
+ item['response'])
+
+ self._test_extend_list(mox_calls)
+
def test_list_security_group_rules_pagination(self):
resources = "security_group_rules"
cmd = securitygroup.ListSecurityGroupRule(