Functions
NovaGridUtils Namespace Reference

Functions

def fail (msg)
 
def warn (msg)
 
def info (msg)
 
def get_credentials (role)
 
def check_file (fname)
 
def check_dir (dname)
 
def check_make_dir (dname)
 
def check_tag (tag)
 
def is_group_writable (fname)
 
def check_is_group_writable (fname)
 
def check_is_production_writable (fname)
 
def get_file_owner (fname)
 
def get_prod_dropbox ()
 
def setup_production (args)
 
def setup_calibration (args)
 
def setup_analysis (args)
 
def add_node_features_arg (parser)
 
def make_jobsub_node_features_arg (features)
 

Function Documentation

def NovaGridUtils.add_node_features_arg (   parser)

Definition at line 143 of file NovaGridUtils.py.

144  parser.add_argument(
145  '--node-features',
146  help = (
147  'Comma separated list of grid node requirements.'
148  ' Example: ssse3,sse4_1'
149  ),
150  dest = 'node_features',
151  default = '',
152  type = lambda s : [ x for x in s.split(',') if x ],
153  )
154 
def add_node_features_arg(parser)
def NovaGridUtils.check_dir (   dname)

Definition at line 35 of file NovaGridUtils.py.

References fail().

35 def check_dir(dname):
36  dname = os.path.expandvars(dname)
37  if not os.path.isdir(dname):
38  fail("Invalid directory %s" %dname)
39 
def fail(msg)
Definition: NovaGridUtils.py:7
def check_dir(dname)
def NovaGridUtils.check_file (   fname)

Definition at line 30 of file NovaGridUtils.py.

References fail().

30 def check_file(fname):
31  fname = os.path.expandvars(fname)
32  if not os.path.isfile(fname):
33  fail("Invalid file %s" %fname)
34 
def fail(msg)
Definition: NovaGridUtils.py:7
def check_file(fname)
def NovaGridUtils.check_is_group_writable (   fname)

Definition at line 75 of file NovaGridUtils.py.

References fail(), and is_group_writable().

76  if not is_group_writable(fname):
77  fail("%s is not group writable, but should be" % fname)
78 
def fail(msg)
Definition: NovaGridUtils.py:7
def check_is_group_writable(fname)
def is_group_writable(fname)
def NovaGridUtils.check_is_production_writable (   fname)

Definition at line 79 of file NovaGridUtils.py.

References fail(), get_file_owner(), and is_group_writable().

Referenced by setup_calibration(), and setup_production().

80  # Production writable means either owned by "novapro" or group writable
81  file_owner = get_file_owner(fname)
82  if not (file_owner == "novapro" or is_group_writable(fname)):
83  fail("%s is neither group writable nor owned by novapro, but should be" % fname)
84 
def fail(msg)
Definition: NovaGridUtils.py:7
def get_file_owner(fname)
def check_is_production_writable(fname)
def is_group_writable(fname)
def NovaGridUtils.check_make_dir (   dname)

Definition at line 40 of file NovaGridUtils.py.

40 def check_make_dir(dname):
41  dname = os.path.expandvars(dname)
42  if not os.path.isdir(dname):
43  print "Making directory",dname
44  os.makedirs(dname)
45 
def check_make_dir(dname)
def NovaGridUtils.check_tag (   tag)

Definition at line 46 of file NovaGridUtils.py.

References fail(), and if().

46 def check_tag(tag):
47  novasoft_development = "/cvmfs/nova-development.opensciencegrid.org/novasoft/"
48  if tag == "development":
49  tag = os.path.split(os.readlink(novasoft_development + "releases/development"))[1]
50  print "Note: using current nightly release '%s' as development..." % tag
51  else:
52  srt_dist = os.getenv("SRT_DIST", "undefined")
53  if srt_dist != "undefined":
54  if tag[0] == 'N':
55  srt_dist = novasoft_development
56 
57  tag_dir = srt_dist + "/releases/" + tag
58  if not os.path.isdir(tag_dir):
59  tag_dir = "/cvmfs/nova.opensciencegrid.org/novasoft/slf?/novasoft/releases/" + tag
60 
61  if not glob.glob(tag_dir):
62  fail("Specified tag %s doesn't exist" % tag)
63  else:
64  novasoftloc = os.getenv("NOVASOFT_FQ_DIR", "undefined")
65  if(novasoftloc == "undefined"):
66  fail("novasoft location is unspecified, was the ups product setup?")
67  return tag
68 
def fail(msg)
Definition: NovaGridUtils.py:7
if(dump)
def check_tag(tag)
def NovaGridUtils.fail (   msg)

Definition at line 7 of file NovaGridUtils.py.

Referenced by check_dir(), check_file(), check_is_group_writable(), check_is_production_writable(), check_tag(), get_credentials(), and setup_production().

7 def fail(msg):
8  sys.stderr.write("\033[91mERROR in %s\033[0m " %prog)
9  sys.stderr.write(msg)
10  sys.stderr.write("\n")
11  sys.exit(-1)
12 
def fail(msg)
Definition: NovaGridUtils.py:7
def NovaGridUtils.get_credentials (   role)

Definition at line 24 of file NovaGridUtils.py.

References fail().

Referenced by setup_analysis(), setup_calibration(), and setup_production().

24 def get_credentials(role):
25  retval = os.system("cigetcert -ns fifebatch.fnal.gov")
26  if retval != 0:
27  fail("Failed to get a certificate. Check kerberos credentials or run cigetcert -s fifebatch.fnal.gov")
28  return
29 
def fail(msg)
Definition: NovaGridUtils.py:7
def get_credentials(role)
def NovaGridUtils.get_file_owner (   fname)

Definition at line 85 of file NovaGridUtils.py.

Referenced by check_is_production_writable().

85 def get_file_owner(fname):
86  stat_info = os.stat(fname)
87  file_stat_uid = stat_info.st_uid
88  return pwd.getpwuid(file_stat_uid)[0]
89 
def get_file_owner(fname)
def NovaGridUtils.get_prod_dropbox ( )

Definition at line 90 of file NovaGridUtils.py.

91  hostname=socket.getfqdn()
92  if hostname.endswith("smu.edu"):
93  return "/scratch/data/srm/nova/production/DropBox/"
94  else:
95  return "/pnfs/nova/scratch/fts/dropbox"
96 
97 
def get_prod_dropbox()
def NovaGridUtils.info (   msg)

Definition at line 18 of file NovaGridUtils.py.

18 def info(msg):
19  sys.stderr.write("\033[94mINFO in %s\033[0m " %prog)
20  sys.stderr.write(msg)
21  sys.stderr.write("\n")
22 
23 
def info(msg)
def NovaGridUtils.is_group_writable (   fname)

Definition at line 69 of file NovaGridUtils.py.

Referenced by check_is_group_writable(), and check_is_production_writable().

69 def is_group_writable(fname):
70  stat_info = os.stat(fname)
71  file_stat_mode = stat_info.st_mode
72  return file_stat_mode & stat.S_IWGRP
73 
74 
def is_group_writable(fname)
def NovaGridUtils.make_jobsub_node_features_arg (   features)

Definition at line 155 of file NovaGridUtils.py.

156 
157  if len(features) == 0:
158  return ''
159 
160  result = "&&".join([ "(TARGET.has_%s==true)" % (x) for x in features ])
161  result = "--append_condor_requirements='(%s)'" % (result)
162 
163  return result
164 
165 
def make_jobsub_node_features_arg(features)
def NovaGridUtils.setup_analysis (   args)

Definition at line 136 of file NovaGridUtils.py.

References get_credentials().

136 def setup_analysis(args):
137 
138  role="Analysis"
139  args.role=role
140 
141  get_credentials(role)
142 
def setup_analysis(args)
def get_credentials(role)
def NovaGridUtils.setup_calibration (   args)

Definition at line 125 of file NovaGridUtils.py.

References check_is_production_writable(), and get_credentials().

126 
127  role="Production"
128  args.role=role
129 
130  get_credentials(role)
131 
132  dest=args.dest
133  # Production writable means either owned by "novapro" or group writable
135 
def get_credentials(role)
def check_is_production_writable(fname)
def setup_calibration(args)
def NovaGridUtils.setup_production (   args)

Definition at line 98 of file NovaGridUtils.py.

References check_is_production_writable(), fail(), get_credentials(), and warn().

98 def setup_production(args):
99 
100  role="Production"
101  args.role=role
102 
103  get_credentials(role)
104 
105  dest=args.dest
106  # Production writable means either owned by "novapro" or group writable
107  if dest.startswith("s3://") and not args.amazon:
108  fail("Not using amazon, but output destination is on s3!")
109  elif not dest.startswith("s3://") and args.amazon:
110  fail("Using amazon, but output destination is not on s3!")
111  elif dest != "/pnfs/nova/scratch/fts/dropbox" and not args.test_submission:
113  warn("Destination directory is "+dest+", not \"/pnfs/nova/scratch/fts/dropbox\", but you are submitting production jobs. Is that what you meant? Pausing for 5s")
114  sleep(5)
115 
116  if not dest.startswith("s3://"):
117  args.hashDirs = True
118 
119  args.jsonMetadata = True
120  args.zipLogs = True
121  if (None == args.outTier) and (None == args.cafTier) and (None == args.h5Tier):
122  fail("Running in production mode, but no tiers (art or caf) specified")
123 
124 
def setup_production(args)
def fail(msg)
Definition: NovaGridUtils.py:7
def get_credentials(role)
def check_is_production_writable(fname)
def warn(msg)
def NovaGridUtils.warn (   msg)

Definition at line 13 of file NovaGridUtils.py.

Referenced by setup_production().

13 def warn(msg):
14  sys.stderr.write("\033[93mWARNING in %s\033[0m " %prog)
15  sys.stderr.write(msg)
16  sys.stderr.write("\n")
17 
def warn(msg)