diff -ru a/ufo/bin/gluster-swift-gen-builders b/ufo/bin/gluster-swift-gen-builders
--- a/ufo/bin/gluster-swift-gen-builders 2012-12-07 12:24:00.000000000 -0500
+++ b/ufo/bin/gluster-swift-gen-builders 2013-04-29 15:16:22.748000000 -0400
@@ -1,9 +1,25 @@
#!/bin/bash
+# Note that these port numbers must match the configured values for the
+# various servers in their configuration files.
+declare -A port=(["account.builder"]=6012 ["container.builder"]=6011 \
+ ["object.builder"]=6010)
+
+builder_files="account.builder container.builder object.builder"
+
function create {
- swift-ring-builder $1 create 0 1 1
- swift-ring-builder $1 add z1-127.0.0.1:$2/$3_ 100.0
+ swift-ring-builder $1 create 1 1 1 >> /tmp/out
+}
+
+function add {
+ swift-ring-builder $1 add z$2-127.0.0.1:$3/$4_ 100.0
+}
+
+function rebalance {
swift-ring-builder $1 rebalance
+}
+
+function build {
swift-ring-builder $1
}
@@ -12,8 +28,17 @@
exit 1
fi
-# Note that these port numbers must match the configured values for the
-# various servers in their configuration files.
-create account.builder 6012 $1
-create container.builder 6011 $1
-create object.builder 6010 $1
+for builder_file in $builder_files
+do
+ create $builder_file
+
+ zone=1
+ for volname in $@
+ do
+ add $builder_file $zone ${port[$builder_file]} $volname
+ zone=$(expr $zone + 1)
+ done
+
+ rebalance $builder_file
+ build $builder_file
+done
diff -ru a/ufo/etc/fs.conf-gluster b/ufo/etc/fs.conf-gluster
--- a/ufo/etc/fs.conf-gluster 2012-12-07 12:24:00.000000000 -0500
+++ b/ufo/etc/fs.conf-gluster 2013-04-29 15:16:22.752000000 -0400
@@ -3,10 +3,6 @@
# local host.
mount_ip = localhost
-# The GlusterFS server need not be local, a remote server can also be used
-# by setting "remote_cluster = yes".
-remote_cluster = no
-
# By default it is assumed the Gluster volumes can be accessed using other
# methods besides UFO (not object only), which disables a caching
# optimizations in order to keep in sync with file system changes.
diff -ru a/ufo/gluster/swift/common/constraints.py b/ufo/gluster/swift/common/constraints.py
--- a/ufo/gluster/swift/common/constraints.py 2012-12-07 12:24:00.000000000 -0500
+++ b/ufo/gluster/swift/common/constraints.py 2013-04-29 15:16:22.749000000 -0400
@@ -16,7 +16,8 @@
from webob.exc import HTTPBadRequest
import swift.common.constraints
-from gluster.swift.common import Glusterfs
+import swift.common.ring as _ring
+from gluster.swift.common import Glusterfs, ring
MAX_OBJECT_NAME_COMPONENT_LENGTH = swift.common.constraints.constraints_conf_int(
@@ -80,3 +81,9 @@
# Replace the original check mount with ours
swift.common.constraints.check_mount = gluster_check_mount
+
+# Save the original Ring class
+__Ring = _ring.Ring
+
+# Replace the original Ring class
+_ring.Ring = ring.Ring
diff -ru a/ufo/gluster/swift/common/Glusterfs.py b/ufo/gluster/swift/common/Glusterfs.py
--- a/ufo/gluster/swift/common/Glusterfs.py 2012-12-07 12:24:00.000000000 -0500
+++ b/ufo/gluster/swift/common/Glusterfs.py 2013-04-29 15:16:22.753000000 -0400
@@ -12,33 +12,35 @@
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
+
import logging
import os, fcntl, time
-from ConfigParser import ConfigParser
-from swift.common.utils import TRUE_VALUES
+from ConfigParser import ConfigParser, NoSectionError, NoOptionError
+from swift.common.utils import TRUE_VALUES, search_tree
from gluster.swift.common.fs_utils import mkdirs
-
#
# Read the fs.conf file once at startup (module load)
#
_fs_conf = ConfigParser()
MOUNT_IP = 'localhost'
-REMOTE_CLUSTER = False
OBJECT_ONLY = False
+RUN_DIR='/var/run/swift'
+SWIFT_DIR = '/etc/swift'
if _fs_conf.read(os.path.join('/etc/swift', 'fs.conf')):
try:
MOUNT_IP = _fs_conf.get('DEFAULT', 'mount_ip', 'localhost')
except (NoSectionError, NoOptionError):
pass
try:
- REMOTE_CLUSTER = _fs_conf.get('DEFAULT', 'remote_cluster', False) in TRUE_VALUES
+ OBJECT_ONLY = _fs_conf.get('DEFAULT', 'object_only', "no") in TRUE_VALUES
except (NoSectionError, NoOptionError):
pass
try:
- OBJECT_ONLY = _fs_conf.get('DEFAULT', 'object_only', "no") in TRUE_VALUES
+ RUN_DIR = _fs_conf.get('DEFAULT', 'run_dir', '/var/run/swift')
except (NoSectionError, NoOptionError):
pass
+
NAME = 'glusterfs'
@@ -60,7 +62,7 @@
if drive == export:
break
else:
- logging.error('No export found in %r matching drive %s', el, drive)
+ logging.error('No export found in %r matching drive, %s', el, drive)
return False
# NOTE: root is typically the default value of /mnt/gluster-object
@@ -68,13 +70,12 @@
if not os.path.isdir(full_mount_path):
mkdirs(full_mount_path)
- pid_dir = "/var/lib/glusterd/vols/%s/run/" % drive
- pid_file = os.path.join(pid_dir, 'swift.pid');
+ lck_file = os.path.join(RUN_DIR, '%s.lock' %drive);
- if not os.path.exists(pid_dir):
- mkdirs(pid_dir)
+ if not os.path.exists(RUN_DIR):
+ mkdirs(RUN_DIR)
- fd = os.open(pid_file, os.O_CREAT|os.O_RDWR)
+ fd = os.open(lck_file, os.O_CREAT|os.O_RDWR)
with os.fdopen(fd, 'r+b') as f:
try:
fcntl.lockf(f, fcntl.LOCK_EX|fcntl.LOCK_NB)
@@ -100,19 +101,12 @@
logging.error('Unable to unmount %s %s' % (full_mount_path, NAME))
def _get_export_list():
- if REMOTE_CLUSTER:
- cmnd = 'gluster --remote-host=%s volume info' % MOUNT_IP
- else:
- cmnd = 'gluster volume info'
+ cmnd = 'gluster --remote-host=%s volume info' % MOUNT_IP
export_list = []
if os.system(cmnd + ' >> /dev/null'):
- if REMOTE_CLUSTER:
- logging.error('Getting volume info failed for %s, make sure '\
- 'gluster --remote-host=%s works', NAME, MOUNT_IP)
- else:
- logging.error('Getting volume info failed for %s', NAME)
+ logging.error('Getting volume info failed for %s', NAME)
else:
fp = os.popen(cmnd)
while True:
@@ -124,3 +118,20 @@
export_list.append(item.split(':')[1].strip(' '))
return export_list
+
+def get_mnt_point(vol_name, conf_dir=SWIFT_DIR, conf_file="object-server*"):
+ """Read the object-server's configuration file and return
+ the device value"""
+
+ mnt_dir = ''
+ conf_files = search_tree(conf_dir, conf_file, '.conf')
+ if not conf_files:
+ raise Exception("Config file not found")
+
+ _conf = ConfigParser()
+ if _conf.read(conf_files[0]):
+ try:
+ mnt_dir = _conf.get('DEFAULT', 'devices', '')
+ except (NoSectionError, NoOptionError):
+ raise
+ return os.path.join(mnt_dir, vol_name)
diff -ru a/ufo/gluster/swift/common/ring.py b/ufo/gluster/swift/common/ring.py
--- a/ufo/gluster/swift/common/ring.py 2013-04-30 08:21:55.948000000 -0400
+++ b/ufo/gluster/swift/common/ring.py 2013-04-29 15:16:22.755000000 -0400
@@ -0,0 +1,111 @@
+# Copyright (c) 2013 Red Hat, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from ConfigParser import ConfigParser
+from swift.common.ring import ring
+from swift.common.utils import search_tree
+from gluster.swift.common.Glusterfs import SWIFT_DIR
+
+reseller_prefix = "AUTH_"
+conf_files = search_tree(SWIFT_DIR, "proxy-server*", 'conf')
+if conf_files:
+ conf_file = conf_files[0]
+
+_conf = ConfigParser()
+if conf_files and _conf.read(conf_file):
+ if _conf.defaults().get("reseller_prefix", None):
+ reseller_prefix = _conf.defaults().get("reseller_prefix")
+ else:
+ for key, value in _conf._sections.items():
+ if value.get("reseller_prefix", None):
+ reseller_prefix = value["reseller_prefix"]
+ break
+
+if not reseller_prefix.endswith('_'):
+ reseller_prefix = reseller_prefix + '_'
+
+class Ring(ring.Ring):
+ def _get_part_nodes(self, part):
+ seen_ids = set()
+ nodes = [dev for dev in self._devs \
+ if dev['device'] == self.acc_name \
+ and not (dev['id'] in seen_ids \
+ or seen_ids.add(dev['id']))]
+ if not nodes:
+ nodes = [self.false_node]
+ return nodes
+
+ def get_part_nodes(self, part):
+ """
+ Get the nodes that are responsible for the partition. If one
+ node is responsible for more than one replica of the same
+ partition, it will only appear in the output once.
+
+ :param part: partition to get nodes for
+ :returns: list of node dicts
+
+ See :func:`get_nodes` for a description of the node dicts.
+ """
+ return self._get_part_nodes(part)
+
+ def get_nodes(self, account, container=None, obj=None):
+ """
+ Get the partition and nodes for an account/container/object.
+ If a node is responsible for more than one replica, it will
+ only appear in the output once.
+ :param account: account name
+ :param container: container name
+ :param obj: object name
+ :returns: a tuple of (partition, list of node dicts)
+
+ Each node dict will have at least the following keys:
+ ====== ===============================================================
+ id unique integer identifier amongst devices
+ weight a float of the relative weight of this device as compared to
+ others; this indicates how many partitions the builder will try
+ to assign to this device
+ zone integer indicating which zone the device is in; a given
+ partition will not be assigned to multiple devices within the
+ same zone
+ ip the ip address of the device
+ port the tcp port of the device
+ device the device's name on disk (sdb1, for example)
+ meta general use 'extra' field; for example: the online date, the
+ hardware description
+ ====== ===============================================================
+ """
+ self.false_node = {'zone': 1, 'weight': 100.0, 'ip': '127.0.0.1', 'id': 0, \
+ 'meta': '', 'device': 'volume_not_in_ring', \
+ 'port': 6012}
+ if account.startswith(reseller_prefix):
+ self.acc_name = account.replace(reseller_prefix, '', 1)
+ else:
+ self.acc_name = account
+
+ part = 0
+ return part, self._get_part_nodes(part)
+
+
+ def get_more_nodes(self, part):
+ """
+ Generator to get extra nodes for a partition for hinted handoff.
+
+ :param part: partition to get handoff nodes for
+ :returns: generator of node dicts
+
+ See :func:`get_nodes` for a description of the node dicts.
+ Should never be called in the swift UFO environment, so yield nothing
+ """
+ yield self.false_node
diff -ru a/ufo/test/unit/common/test_ring.py b/ufo/test/unit/common/test_ring.py
--- a/ufo/test/unit/common/test_ring.py 2013-04-30 08:22:08.975000000 -0400
+++ b/ufo/test/unit/common/test_ring.py 2013-04-29 15:16:22.756000000 -0400
@@ -0,0 +1,81 @@
+# Copyright (c) 2013 Red Hat, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import unittest
+import gluster.swift.common.constraints
+from gluster.swift.common.ring import *
+from gluster.swift.common.Glusterfs import SWIFT_DIR
+
+def _mock_ring_data():
+ return [{'zone': 1, 'weight': 100.0, 'ip': '127.0.0.1', 'port': 6012, \
+ 'meta': '', 'device': 'test', 'id': 0},
+ {'zone': 2, 'weight': 100.0, 'ip': '127.0.0.1', 'id': 1, \
+ 'meta': '', 'device': 'iops', 'port': 6012}]
+
+class TestRing(unittest.TestCase):
+ """ Tests for common.utils """
+
+ def setUp(self):
+ self.ring = Ring(SWIFT_DIR, ring_name='object')
+
+ def test_first_device(self):
+ try:
+ __devs = self.ring._devs
+ self.ring._devs = _mock_ring_data()
+
+ part, node = self.ring.get_nodes('test')
+ assert node[0]['device'] == 'test'
+ node = self.ring.get_part_nodes(0)
+ assert node[0]['device'] == 'test'
+ for node in self.ring.get_more_nodes(0):
+ assert node['device'] == 'volume_not_in_ring'
+ finally:
+ self.ring._devs = __devs
+
+ def test_invalid_device(self):
+ try:
+ __devs = self.ring._devs
+ self.ring._devs = _mock_ring_data()
+
+ part, node = self.ring.get_nodes('test2')
+ assert node[0]['device'] == 'volume_not_in_ring'
+ node = self.ring.get_part_nodes(0)
+ assert node[0]['device'] == 'volume_not_in_ring'
+ finally:
+ self.ring._devs = __devs
+
+ def test_second_device(self):
+ try:
+ __devs = self.ring._devs
+ self.ring._devs = _mock_ring_data()
+
+ part, node = self.ring.get_nodes('iops')
+ assert node[0]['device'] == 'iops'
+ node = self.ring.get_part_nodes(0)
+ assert node[0]['device'] == 'iops'
+ for node in self.ring.get_more_nodes(0):
+ assert node['device'] == 'volume_not_in_ring'
+ finally:
+ self.ring._devs = __devs
+
+ def test_second_device_with_reseller_prefix(self):
+ try:
+ __devs = self.ring._devs
+ self.ring._devs = _mock_ring_data()
+
+ part, node = self.ring.get_nodes('AUTH_iops')
+ assert node[0]['device'] == 'iops'
+ finally:
+ self.ring._devs = __devs