diff --git a/shell/src/lost.py b/shell/src/lost.py index 221c5b32..ed66c74b 100644 --- a/shell/src/lost.py +++ b/shell/src/lost.py @@ -513,7 +513,7 @@ def _run_ssh_command(client, cmd, timeout=0): if c.recv_ready(): stdout_bytes.write(channel.recv(len(c.in_buffer))) if c.recv_stderr_ready(): - stderr_bytes.write(channel.recv(len(c.in_buffer))) + stderr_bytes.write(channel.recv_stderr(len(c.in_buffer))) exit_status = channel.recv_exit_status() channel.shutdown_read() diff --git a/shell/src/migrate.py b/shell/src/migrate.py index f889c34c..9ba1c2db 100644 --- a/shell/src/migrate.py +++ b/shell/src/migrate.py @@ -313,7 +313,7 @@ class dpm(object): if c.recv_ready(): stdout_bytes.write(channel.recv(len(c.in_buffer))) if c.recv_stderr_ready(): - stderr_bytes.write(channel.recv(len(c.in_buffer))) + stderr_bytes.write(channel.recv_stderr(len(c.in_buffer))) exit_status = channel.recv_exit_status() channel.shutdown_read() @@ -589,6 +589,11 @@ class dpm(object): record_time_mono = start_time_mono record_time_cpu = start_time_cpu + # get all data from dpm_db database here, because this connection is idle + # for a long time and it seems some MySQL configuration can drop it + # (we currently don't implement automatic reconnection) + space_config = [x for x in self._get_space_configuration()] + #with io.open(nsfile, 'w', encoding='utf-8', newline='') as f: # FIXME: csv writing with python 2.7 with io.open(nsfile, 'w', newline='') as csvfile: writer = csv.writer(csvfile, quotechar="'", quoting=csv.QUOTE_MINIMAL) @@ -670,7 +675,7 @@ class dpm(object): fslist = set() projectcfg = {} spacetokencfg = {} - for project, spacetoken, path, groups, space, hostfss in self._get_space_configuration(): + for project, spacetoken, path, groups, space, hostfss in space_config: spacetokencfg[spacetoken] = (project, path, groups, space, hostfss) pspacetokens, pspace, phostfss = projectcfg.get(project, (set(), 0, set())) pspacetokens.add(spacetoken) @@ -1154,6 +1159,9 @@ class dcache(object): def _get_spacetoken_id(self, spacetoken): + if spacetoken == None: + return None + if self._spacetoken2id == None: self._spacetoken2id = {} #cursor = self._conn.cursor() @@ -1428,6 +1436,9 @@ class dcache(object): def import_csv(self, nsfile, skip_acl, skip_writetoken): _log.info("import data from %s to dCache database (skip_acl=%s, skip_writetoken=%s)", nsfile, skip_acl, skip_writetoken) + if len(self._config['spacetoken']): + raise Exception("Importing dCache namespace without any spacetoken is not currently supported") + # spacetoken to project mapping from real info # about used space regardless of mismatched filesystem spacetoken2project = {} @@ -1492,14 +1503,34 @@ class dcache(object): elif tobj == 'FILE': size, csums, pool, host, fs, sfn = row[10:] + project = spacetoken2project.get(spacetoken) + if spacetoken == '': - _log.error("ignoring file with empty spacetoken: %s", path) - cnt['UNKNOWN'] = cnt.get('UNKNOWN', 0) + 1 + _log.warn("file with empty spacetoken: %s", path) elif spacetoken not in spacetoken2project: - _log.error("ignoring file with spacetoken %s not mapped to any project: %s", spacetoken, path) + _log.warn("file with spacetoken %s not mapped to any project: %s", spacetoken, path) + + if project == None and (host, fs) in self._config['filesystem']: + # try to find right project for host filesystem + hostfs_projects, hostfs_spacetokens, hostfs_space, hostfs_used = self._config['filesystem'][(host, fs)] + if len(hostfs_projects) == 1: + project = hostfs_projects[0] + else: + # find project with biggest space and use it for data + # that were not associated with any spacetoken + max_project_space = 0 + for project_project in self._config['project'].keys(): + if project_project not in hostfs_projects: continue + project_spacetokens, project_space, project_used = self._config['project'] + if project_space > max_project_space: + project = project_project + max_project_space = project_space + + if project == None: + _log.error("ignoring file that can't be associated with any project: %s (spacetoken %s)", path, spacetoken) cnt['UNKNOWN'] = cnt.get('UNKNOWN', 0) + 1 + else: - project = spacetoken2project[spacetoken] dpool = self._get_dpool(host, fs, project) if path != lastpath: @@ -1753,7 +1784,7 @@ class dcache(object): # # Tape write pools are usually not big and you can use 1024m. A large file system # cache is more useful on such pools. -dcache.java.memory.heap = 2048m +dcache.java.memory.heap = 4096m # The default is 512m, but in particular with xrootd this isn't quite enough. # @@ -1787,7 +1818,9 @@ dcache.net.wan.port.max = 25000 # LAN port range for internal pool to pool communication dcache.net.lan.port.min = 33115 dcache.net.lan.port.max = 33145 - +# use same ports for all protocols (GridFTP, WebDAV, xroot) +pool.mover.xrootd.port.min = ${dcache.net.wan.port.min} +pool.mover.xrootd.port.max = ${dcache.net.wan.port.max} # Allow Let's encrypt certificates that doesn't provide CRLs #dcache.authn.hostcert.verify=true @@ -2130,7 +2163,6 @@ group:atlasde_vokac_vorole gid:2110 # ==================================== # location: /etc/dcache/layouts/layout-{0}.conf -# TODO: cleanup headnode configuration [centralDomain] dcache.broker.scheme = core [centralDomain/zookeeper] @@ -2222,7 +2254,10 @@ srm.net.port = 8446 fs2info[fs_fs] = (spacetokens, space) for fs in sorted(fs2info.keys()): spacetokens, space = fs2info[fs] - f.write("# filesystem {0} size {1} (spacetokens: {2})\n".format(fs, prettySize(space), ', '.join(spacetokens))) + if space != 0: + f.write("# filesystem {0} size {1} (spacetokens: {2})\n".format(fs, prettySize(space), ', '.join(spacetokens))) + else: + f.write("# filesystem {0} (spacetokens: {1})\n".format(fs, ', '.join(spacetokens))) f.write("\n") f.write(""" [doorsDomain-${host.name}] @@ -2249,7 +2284,8 @@ ftp.authn.protocol=gsi f.write("pool.name={0}\n".format(dpool)) f.write("pool.tags=hostname=${{host.name}} poolgroup={0}\n".format(project)) f.write("pool.path={0}/dcache/{1}\n".format(fs, dpool)) - f.write("pool.size={0}G\n".format(expected / 1024**3)) + # pool.size is zero when DPM DB dump was not done on headnode + #f.write("pool.size={0}G\n".format(expected / 1024**3)) #f.write("pool.wait-for-files=${pool.path}/data\n") f.write("\n")