makeTestPickles.py
Go to the documentation of this file.
1 # This program will make some fake pickle files for testing
2 print "run : --- Make test pickles"
3 ########################### Imports
4 import Tier, Chain
5 import ParserDBUsage as db_parser
6 import numpy as np
7 import cPickle
8 import time
9 import os
10 ########################### Clean start
11 #os.system("rm -f /Users/mt354/Analysis/Production/web/test_a/*.pkl")
12 ########################### Config
13 n_chains = np.random.random_integers(1,5)
14 #n_chains = 2
15 n_tiers = np.random.random_integers(2,5)
16 #n_tiers = 2
17 vv_metrics = ["open_files",
18  "connections"]
19 
20 v_metrics = ["memory",
21  "memory_percent",
22  "cpu_percent",
23  "u_cpu_times",
24  "s_cpu_times",
25  "read_bytes",
26  "write_bytes",
27  "rss_memory",
28  "vms_memory",
29  ]
30 
31 
32 metrics = ["input_events",
33  "used_events",
34  "user_cpu",
35  "system_cpu",
36  "peak_memory"]
37 
38 time_range = np.random.random()*100.
39 #time_range = 10.
40 time_steps = 1.
41 n_time_steps = int(time_range/time_steps)
42 events = {1.:"Begin processing the 1st event",
43  2.:" query took 5 ms",
44  3.:" parsing took 5 ms",
45  5.:"Begin processing the 2nd event",
46  7.:"Begin processing the 3rd event",
47  }
48 ########################### Make fake chains
49 for i_chain in range(n_chains):
50  chain = Chain.Chain("Chain_%i"%i_chain)
51  for i_tier in range(n_tiers):
52  tier = Tier.Tier(short_name ="tier_%i"%i_tier,
53  input_name ="input_name_%i"%i_tier,
54  fcl ="fcl_%i"%i_tier,
55  output_name="output_name_%i"%i_tier,
56  )
57  tier.metric.run = True
58  tier.metric.output_name = []
59  tier.metric.output_name.append(tier.output_name)
60  tier.metric.output_size = [np.random.random()*1000.]
61  tier.metric.output_events = [int(np.random.random()*1000.)]
62  tier.metric.return_code = 0
63  tier.metric.sample_times = []
64  tier.full_fcl = "# fake fhicl\n#import something.fcl\nservices = something\n"
65 
66  for metric in metrics:
67  setattr(tier.metric, metric, np.random.random()*1000.)
68 
69  for metric in v_metrics:
70  setattr(tier.metric, metric, np.random.random(n_time_steps))
71 
72  for metric in vv_metrics:
73  setattr(tier.metric, metric, [[]])
74 
75 
76  for i_time in range(n_time_steps):
77  tier.log += "TimeMarker: %f\n"%(i_time*time_steps)
78  tier.metric.sample_times.append(i_time*time_steps)
79  if i_time in events.keys():
80  tier.log += "%s\n"%events[i_time]
81 
82  f = open("tmp.txt","wb")
83  f.write(tier.log)
84  f.close()
85  tier.metric.db_usage = db_parser.parseSTDOUT(f.name)
86 
87  #print tier.metric
88  chain.AddTier(tier)
89  ########################### Pickle
90  output_name = "/Users/mt354/Analysis/Production/web/test_b/pkl_%i.pkl"%i_chain
91  chain.env = os.environ
92  chain.env["USER"] = "user"
93  chain.env["HOSTNAME"] = "host"
94  chain.env["SRT_BASE_RELEASE"] = "base_release"
95  chain.env["SRT_QUAL"] = "quals"
96  chain.env["SRT_PUBLIC_CONTEXT"] = "srt_public"
97  chain.env["SRT_PRIVATE_CONTEXT"] = "srt_private"
98  chain.time_str = time.strftime("%Y-%m-%d %H:%M:%S (GMT%z)", time.localtime())
99  chain.time_str = chain.time_str.replace(chain.time_str[-3:],":%s"%str(chain.time_str[-3:]))
100  cPickle.dump(chain,open(output_name,"wb"))
101 ########################### Done
102 print "run : --- Done"
Definition: Tier.py:1
procfile open("FD_BRL_v0.txt")