Blob Blame History Raw
diff --git a/shell/src/lost.py b/shell/src/lost.py
index 221c5b32..ed66c74b 100644
--- a/shell/src/lost.py
+++ b/shell/src/lost.py
@@ -513,7 +513,7 @@ def _run_ssh_command(client, cmd, timeout=0):
             if c.recv_ready():
                 stdout_bytes.write(channel.recv(len(c.in_buffer)))
             if c.recv_stderr_ready():
-                stderr_bytes.write(channel.recv(len(c.in_buffer)))
+                stderr_bytes.write(channel.recv_stderr(len(c.in_buffer)))
 
     exit_status = channel.recv_exit_status()
     channel.shutdown_read()
diff --git a/shell/src/migrate.py b/shell/src/migrate.py
index f889c34c..9ba1c2db 100644
--- a/shell/src/migrate.py
+++ b/shell/src/migrate.py
@@ -313,7 +313,7 @@ class dpm(object):
                     if c.recv_ready():
                         stdout_bytes.write(channel.recv(len(c.in_buffer)))
                     if c.recv_stderr_ready():
-                        stderr_bytes.write(channel.recv(len(c.in_buffer)))
+                        stderr_bytes.write(channel.recv_stderr(len(c.in_buffer)))
 
             exit_status = channel.recv_exit_status()
             channel.shutdown_read()
@@ -589,6 +589,11 @@ class dpm(object):
         record_time_mono = start_time_mono
         record_time_cpu = start_time_cpu
 
+        # get all data from dpm_db database here, because this connection is idle
+        # for a long time and it seems some MySQL configuration can drop it
+        # (we currently don't implement automatic reconnection)
+        space_config = [x for x in self._get_space_configuration()]
+
         #with io.open(nsfile, 'w', encoding='utf-8', newline='') as f: # FIXME: csv writing with python 2.7
         with io.open(nsfile, 'w', newline='') as csvfile:
             writer = csv.writer(csvfile, quotechar="'", quoting=csv.QUOTE_MINIMAL)
@@ -670,7 +675,7 @@ class dpm(object):
             fslist = set()
             projectcfg = {}
             spacetokencfg = {}
-            for project, spacetoken, path, groups, space, hostfss in self._get_space_configuration():
+            for project, spacetoken, path, groups, space, hostfss in space_config:
                 spacetokencfg[spacetoken] = (project, path, groups, space, hostfss)
                 pspacetokens, pspace, phostfss = projectcfg.get(project, (set(), 0, set()))
                 pspacetokens.add(spacetoken)
@@ -1154,6 +1159,9 @@ class dcache(object):
 
 
     def _get_spacetoken_id(self, spacetoken):
+        if spacetoken == None:
+            return None
+
         if self._spacetoken2id == None:
             self._spacetoken2id = {}
             #cursor = self._conn.cursor()
@@ -1428,6 +1436,9 @@ class dcache(object):
     def import_csv(self, nsfile, skip_acl, skip_writetoken):
         _log.info("import data from %s to dCache database (skip_acl=%s, skip_writetoken=%s)", nsfile, skip_acl, skip_writetoken)
 
+        if len(self._config['spacetoken']):
+            raise Exception("Importing dCache namespace without any spacetoken is not currently supported")
+
         # spacetoken to project mapping from real info
         # about used space regardless of mismatched filesystem
         spacetoken2project = {}
@@ -1492,14 +1503,34 @@ class dcache(object):
 
                 elif tobj == 'FILE':
                     size, csums, pool, host, fs, sfn = row[10:]
+                    project = spacetoken2project.get(spacetoken)
+
                     if spacetoken == '':
-                        _log.error("ignoring file with empty spacetoken: %s", path)
-                        cnt['UNKNOWN'] = cnt.get('UNKNOWN', 0) + 1
+                        _log.warn("file with empty spacetoken: %s", path)
                     elif spacetoken not in spacetoken2project:
-                        _log.error("ignoring file with spacetoken %s not mapped to any project: %s", spacetoken, path)
+                        _log.warn("file with spacetoken %s not mapped to any project: %s", spacetoken, path)
+
+                    if project == None and (host, fs) in self._config['filesystem']:
+                        # try to find right project for host filesystem
+                        hostfs_projects, hostfs_spacetokens, hostfs_space, hostfs_used = self._config['filesystem'][(host, fs)]
+                        if len(hostfs_projects) == 1:
+                            project = hostfs_projects[0]
+                        else:
+                            # find project with biggest space and use it for data
+                            # that were not associated with any spacetoken
+                            max_project_space = 0
+                            for project_project in self._config['project'].keys():
+                                if project_project not in hostfs_projects: continue
+                                project_spacetokens, project_space, project_used = self._config['project']
+                                if project_space > max_project_space:
+                                    project = project_project
+                                    max_project_space = project_space
+
+                    if project == None:
+                        _log.error("ignoring file that can't be associated with any project: %s (spacetoken %s)", path, spacetoken)
                         cnt['UNKNOWN'] = cnt.get('UNKNOWN', 0) + 1
+
                     else:
-                        project = spacetoken2project[spacetoken]
                         dpool = self._get_dpool(host, fs, project)
 
                         if path != lastpath:
@@ -1753,7 +1784,7 @@ class dcache(object):
 #
 # Tape write pools are usually not big and you can use 1024m. A large file system
 # cache is more useful on such pools.
-dcache.java.memory.heap = 2048m
+dcache.java.memory.heap = 4096m
 
 # The default is 512m, but in particular with xrootd this isn't quite enough. 
 #
@@ -1787,7 +1818,9 @@ dcache.net.wan.port.max = 25000
 # LAN port range for internal pool to pool communication
 dcache.net.lan.port.min = 33115
 dcache.net.lan.port.max = 33145
-
+# use same ports for all protocols (GridFTP, WebDAV, xroot)
+pool.mover.xrootd.port.min = ${dcache.net.wan.port.min}
+pool.mover.xrootd.port.max = ${dcache.net.wan.port.max}
 
 # Allow Let's encrypt certificates that doesn't provide CRLs
 #dcache.authn.hostcert.verify=true
@@ -2130,7 +2163,6 @@ group:atlasde_vokac_vorole     gid:2110
 # ====================================
 # location: /etc/dcache/layouts/layout-{0}.conf
 
-# TODO: cleanup headnode configuration
 [centralDomain]
 dcache.broker.scheme = core
 [centralDomain/zookeeper]
@@ -2222,7 +2254,10 @@ srm.net.port = 8446
                     fs2info[fs_fs] = (spacetokens, space)
                 for fs in sorted(fs2info.keys()):
                     spacetokens, space = fs2info[fs]
-                    f.write("# filesystem {0} size {1} (spacetokens: {2})\n".format(fs, prettySize(space), ', '.join(spacetokens)))
+                    if space != 0:
+                        f.write("# filesystem {0} size {1} (spacetokens: {2})\n".format(fs, prettySize(space), ', '.join(spacetokens)))
+                    else:
+                        f.write("# filesystem {0} (spacetokens: {1})\n".format(fs, ', '.join(spacetokens)))
                 f.write("\n")
                 f.write("""
 [doorsDomain-${host.name}]
@@ -2249,7 +2284,8 @@ ftp.authn.protocol=gsi
                     f.write("pool.name={0}\n".format(dpool))
                     f.write("pool.tags=hostname=${{host.name}} poolgroup={0}\n".format(project))
                     f.write("pool.path={0}/dcache/{1}\n".format(fs, dpool))
-                    f.write("pool.size={0}G\n".format(expected / 1024**3))
+                    # pool.size is zero when DPM DB dump was not done on headnode
+                    #f.write("pool.size={0}G\n".format(expected / 1024**3))
                     #f.write("pool.wait-for-files=${pool.path}/data\n")
                     f.write("\n")