#!/bin/env python #copytest.py #Marco Mambelli """Test for copy tools """ import commands from timed_command import timed_command from futil import * timeout = 60*5 # 5 min # class for options class MyOpt: pass opt = MyOpt() del MyOpt opt.verbose = 3 # verbosity level: 0 ... opt.addtoname = '' # string added to output file name to help with name uniqueness (to run multiple tests opt.runtestsetup = True opt.dryrun = False #True #opt.addtoname = 't2.' # string added to output file name to help with name uniqueness (to run multiple tests if opt.addtoname: opt.runtestsetup = False # General setup (prepare directories, ...) test_setup_str = "rm -rf /tmp/marco/cptest; mkdir -p /tmp/marco/cptest; \ echo 'Remember to delete on uct3-edge3 /pnfs/uct3/test/cptestmarco-* uct2-grid6 /pnfs/uchicago.edu/test/cptestmarco-*'" default_setup = 'source /local/inst2/wlcg-client080725/setup.sh' # /share/wlcg-client/setup.sh' """ srmcp: -streams_num= -srm_protocol_version= -pushmode= -space_token= -server_mode= -cksm_type= calculate and verify server and client checksum values using this type (adler32|MD4|MD5| .... ). If checksum value has been omitted, missing value will be computed over the local file. If negotiate is set - client will attempt to negotiate cksm type for the file checksum value avilable at the server. For gridftp transfers to/from servers that support checksumming features -cksm_value= override dynamic calucation of the local checksum with this value. If cksm_type option has not been set, adler32 will be assumed. For gridftp transfers to/from servers that support checksumming features srm-copy: -spacetoken (valid space token returned by srm-space request, used to put a file in the reserved space, or get a file from the reserved space.) ngcp: -p, -passive -r, -recursive level operate recursively (if possible) up to specified level (0 - no recursion) -R, -retries number how many times to retry transfer of every file before failing -d -3 ... 3 lcg-cp: export LCG_GFAL_INFOSYS=lcg-bdii.cern.ch:2170 lcg-cp [-h,--help] [-i,--insecure] [-c,--config config_file] [-n nbstreams] [-s,--sst src_spacetokendesc] [-S,--dst dest_spacetokendesc] [-D,--defaultsetype se|srmv1|srmv2] [-T,--srcsetype se|srmv1|srmv2] [-U,--dstsetype se|srmv1|srmv2] [-b,--nobdii] [-t timeout] [-v,--verbose] [-V,--vo vo] [--version] src_file dest_file """ lcg_cp_setup = '%s; export LCG_GFAL_INFOSYS="lcg-bdii.cern.ch:2170"' % (default_setup) copy_commands = { 'globus': ['', 'globus-url-copy' ], 'lcg': [lcg_cp_setup, 'lcg-cp', ], 'lcg2': ['', 'lcg-cp -b', ], 'lcg3': ['', 'lcg-cp -b -T srmv1', ], 'lcg4': ['', 'lcg-cp -b -T srmv2', ], 'lcg5': ['', 'lcg-cp -b --vo atlas', ], 'nordugrid': ['source /local/inst2/ng080721/nordugrid-arc-standalone-0.6.3/setup.sh /local/inst2/ng080721/nordugrid-arc-standalone-0.6.3/', 'ngcp -d 2', ], 'srmf': ['', 'srmcp', ], 'srmf2': ['', 'srmcp -2', ], 'srmf3': ['', 'srmcp -1', ], 'srmf4': ['', 'srmcp -streams_num=1', ], 'srml': ['', 'srm-copy', ], 'srmg': ['source /share/glite/glite/etc/profile.d/grid-env.sh', 'glite-srm-get', ], } """ lcg-cp -v --vo atlas srm://dcsrm.usatlas.bnl.gov:8443/pnfs/usatlas.bnl.gov/ESD01/valid1/ESD/valid1.005200.T1_McAtNlo_Jimmy.recon.ESD.e322_s454_r489_tid023919_sub02072646/ESD.023919._00001.pool.root.1 file:///tmp/test080721.root srmcp -retry_num=1 -streams_num=1 -x509_user_trusted_certificates=/share/osg-client/globus/TRUSTED_CA -srm_protocol_version=2 srm://uct2-dc1.uchicago.edu:8443/srm/managerv2?SFN=/pnfs/uchicago.edu/atlasdatadisk/fdr08_run2/AOD/fdr08_run2.0052293.physics_Jet.merge.AOD.o3_f8_m10/fdr08_run2.0052293.physics_Jet.merge.AOD.o3_f8_m10._0001.1 file:////ecache/efeng/dummy/./fdr08_run2.0052293.physics_Jet.merge.AOD.o3_f8_m10._0001.1 lcg-cp srm://uct2-dc1.uchicago.edu/pnfs/uchicago.edu/atlasdatadisk/fdr08_run2/AOD/fdr08_run2.0052280.physics_Jet.merge.AOD.o3_f8_m10/fdr08_run2.0052280.physics_Jet.merge.AOD.o3_f8_m10._0012.1 gsiftp://uct3-edge1.uchicago.edu/pnfs/uct3/atlas/fdr08_run2/AOD/fdr08_run2.0052280.physics_Jet.merge.AOD.o3_f8_m10/fdr08_run2.0052280.physics_Jet.merge.AOD.o3_f8_m10._0012.1__DQ2-1216649656 lcg-cp [-h,--help] [-i,--insecure] [-c,--config config_file] [-n nbstreams] [-s,--sst src_spacetokendesc] [-S,--dst dest_spacetokendesc] [-D,--defaultsetype se|srmv1|srmv2] [-T,--srcsetype se|srmv1|srmv2] [-U,--dstsetype se|srmv1|srmv2] [-b,--nobdii] [-t timeout] [-v,--verbose] [-V,--vo vo] [--version] src_file dest_file """ h_bnl_file1 = '/pnfs/usatlas.bnl.gov/ESD01/valid1/ESD/valid1.005200.T1_McAtNlo_Jimmy.recon.ESD.e322_s454_r489_tid023919_sub02072646/ESD.023919._00001.pool.root.1' h_uc_file1 = "/pnfs/uchicago.edu/atlasdatadisk/fdr08_run2/AOD/fdr08_run2.0052293.physics_Jet.merge.AOD.o3_f8_m10/fdr08_run2.0052293.physics_Jet.merge.AOD.o3_f8_m10._0001.1" # 2.3G h_uc_file2 = "/pnfs/uchicago.edu/data/ddm1/user.RikutaroYoshida.test.22July.fdr2.0052304.Egamma.mwt2_sub02082889/user.RikutaroYoshida.test.22July.fdr2.0052304.Egamma.mwt2.AANT._00001.root" # 258M copy_tests = { 'srmV1BNL': ['srm://dcsrm.usatlas.bnl.gov:8443%s' % (h_bnl_file1,), 'file:///tmp/marco/cptest/file1.pool.root', 'SRMv1 copy from BNL'], 'srmV1BNL2UC': ['srm://dcsrm.usatlas.bnl.gov:8443%s' % (h_bnl_file1,), 'srm://uct2-dc1.uchicago.edu:8443/pnfs/uchicago.edu/test/cptestmarco-filesrmV1BNL2UC.pool.root', 'SRMv1 copy from BNL to UC srm'], 'srmV2BNL2UC': ['srm://dcsrm.usatlas.bnl.gov:8443/srm/managerv2?SFN=%s' % (h_bnl_file1,), 'srm://uct2-dc1.uchicago.edu:8443/srm/managerv2?SFN=/pnfs/uchicago.edu/test/cptestmarco-filesrmV2BNL2UC.pool.root', 'SRMv2 copy from BNL to UC srm'], 'srmV2': ['srm://uct2-dc1.uchicago.edu:8443/srm/managerv2?SFN=%s' % (h_uc_file2,), 'file:///tmp/marco/cptest/file2.pool.root', 'SRMv2 from Chicago, URLSs as from LRC'], 'srmV2f4': ['srm://uct2-dc1.uchicago.edu:8443/srm/managerv2?SFN=%s' % (h_uc_file2,), 'file:////tmp/marco/cptest/file2.pool.root', 'SRMv2 from Chicago, URLSs with file:////... (4/)'], 'srmV1': ['srm://uct2-dc1.uchicago.edu:8443%s' % (h_uc_file2,), 'file:///tmp/marco/cptest/file2b.pool.root', 'SRMv1 from Chicago, modified URL'], 'srmV1np': ['srm://uct2-dc1.uchicago.edu%s' % (h_uc_file2,), 'file:///tmp/marco/cptest/file2c.pool.root', 'SRMv1 from Chicago, no port # in the SRM URL'], 'gsifp': ['gsiftp://uct2-dc1.uchicago.edu%s' % (h_uc_file2,), 'file:///tmp/marco/cptest/file2d.pool.root', 'gsiftp copy from Chicago - some URL in LRC still use it'], 'srm3pV1': ['srm://uct2-dc1.uchicago.edu:8443%s' % (h_uc_file2,), 'gsiftp://uct3-edge1.uchicago.edu/pnfs/uct3/test/cptestmarco-file3.pool.root', 'SRMv1 to GSIftp copy, 3 party transfer'], # ['', ''], } skip_cc = [] #'nordugrid', 'lcg4'] #'lcg', 'lcg2', 'srm', 'srm2'] skip_test = [] results = {} # print header if opt.verbose>1: print "***** TESTS PERFORMED:" for (i, j) in copy_tests.items(): if len(j)>2: print i, ": ", j[2] else: print i, ": - " if opt.verbose>2: print " from:", j[0], " to:", j[1] if opt.runtestsetup: if opt.dryrun: print "No setup - dryrun - would be: ", test_setup_str else: ec, out = commands.getstatusoutput(test_setup_str) if opt.verbose>0 or ec<>0: print "Test Setup (", ec, "): ", out def modifyURL(url, client=''): ret = url if client.startswith('srmf'): ret = ret.replace('////','///') ret = ret.replace('file:///','file:////') return ret if opt.verbose>1: print "***** EXECUTION:" for ii in copy_commands.items(): if ii[0] in skip_cc: continue i = ii[1] results[ii[0]] = [] first_test = True for jj in copy_tests.items(): if jj[0] in skip_test: continue j = jj[1] if i[0]: c_setup = i[0] else: c_setup = default_setup c_line = "%s; %s %s %s.%s%s" % (c_setup, i[1], modifyURL(j[0], ii[0]), modifyURL(j[1], ii[0]), opt.addtoname, ii[0]) if opt.verbose>0: if first_test or opt.verbose>1: print "Executing: ", c_line ec = -1 if not opt.dryrun: try: # ec, out = commands.getstatusoutput(c_line) ec, telapsed, cout, cerr = timed_command(c_line, timeout) out = cerr+cout if ec != 0: check_syserr(ec, out) if is_timeout(ec): if opt.verbose>0: print "Timed out!" # return ERR_TIMEOUT if opt.verbose>0: if first_test or opt.verbose>1: print "Exec %s - %s %s (%s): %s" % (ii[0], jj[0], telapsed, ec, out) except: print get_exc() first_test = False results[ii[0]].append(ec == 0) print "***** TEST RESULTS:" print "Tests: \t", [ i for i in copy_tests.keys() if i not in skip_test] for i in results.items(): print i[0], ": \t", i[1]