Author: tmckay
Date: 2011-01-26 19:35:08 +0000 (Wed, 26 Jan 2011)
New Revision: 4486
Modified:
branches/scale_testing/mech_scripts/test_driver.py
Log:
Record custom headers for offline processing
Modified: branches/scale_testing/mech_scripts/test_driver.py
===================================================================
--- branches/scale_testing/mech_scripts/test_driver.py 2011-01-24 19:06:57 UTC (rev 4485)
+++ branches/scale_testing/mech_scripts/test_driver.py 2011-01-26 19:35:08 UTC (rev 4486)
@@ -33,7 +33,7 @@
self.visited = {}
self.file_init = False
- self.file_number = -1
+ self.file_number = 0
self.file_iterations = 0
def next_file(self):
@@ -46,43 +46,67 @@
else:
self.file_list = [self.link_file]
- self.file_number += 1
- if self.file_number == len(self.file_list):
+ name = self.file_list[self.file_number]
+ if self.file_number == len(self.file_list) - 1:
self.file_iterations += 1
self.file_number = 0
- return self.file_list[self.file_number]
+ else:
+ self.file_number += 1
+ return name
+ def handle_custom_headers(self, curr_file, link, resp, insert_blank):
+ fname = self.group + self.thread + "_" + os.path.basename(curr_file)
+ if os.path.isfile(fname):
+ f = open(fname, "a")
+ if insert_blank:
+ f.write("\n")
+ else:
+ f = open(fname, "w")
+
+ f.write(link)
+ for i in ("page_before_accept", "page_after_accept",
"page_put_time",
+ "page_after_get", "page_begin_service",
"page_begin_proc",
+ "page_end_proc","page_send_headers"):
+ f.write(","+resp.getheader(i,"0"))
+ f.write("\n")
+ f.close()
+
def run(self):
try:
- # parse options on first invocation, get machine and links file
+ # use this to get an idea of tets overhead.
+ # Sum the times for individual gets to look just at page requests
+ start_timer = time_time()
+
+ # parse options on first invocation
if self.machine == None:
parser = optparse.OptionParser()
parser.add_option('--machine', dest='machine',
type='str', default="localhost:45672")
- parser.add_option('--file', dest='file',
type='str', default="")
- parser.add_option('--iter', dest='iter',
type='int', default=0)
+ parser.add_option('--file', dest='file',
type='str', default="./links")
+ parser.add_option('--iter', dest='iter',
type='int', default=1)
parser.add_option('--info', dest='info',
action="store_true", default=False)
+
+ # combine these to create an output file name for web server internal
stats
+ parser.add_option('--group', dest='group',
type='str', default="headers")
+ parser.add_option('--thread', dest='thread',
type='str', default="")
+
if type(self.script_options) == str:
self.script_options = self.script_options.split()
opts, args = parser.parse_args(self.script_options)
+
self.machine = opts.machine
self.link_file = os.path.abspath(opts.file)
self.iteration_limit = opts.iter
self.print_info = opts.info
-
- start_timer = time_time()
- f = open(self.next_file(), "r")
+ self.group = opts.group
+ self.thread = opts.thread
+
+ curr_file = self.next_file()
+ f = open(curr_file, "r")
if self.print_info:
- print f
+ print curr_file
- # If an iteration limit was set, bail after going through file list
- # that many times
- if self.iteration_limit and self.file_iterations == self.iteration_limit:
- return True
-
- total_gets = 0
- total_size = 0
- get_latency = 0
- proc_time = 0
+ total_gets = total_size = get_latency = 0
+ first_link = True
for link in f:
link_info = link.split(',')
@@ -104,6 +128,11 @@
conn.request('GET', link)
resp = conn.getresponse()
data = resp.read()
+ get_latency += time_time() - time_for_get
+ total_gets += 1
+ total_size += len(data)
+
+ # Handle response, update visited times if necessary
if resp.status == 200:
self.visited[link] = datetime.utcnow().strftime(self.http_date_gmt)
if self.print_info:
@@ -115,26 +144,22 @@
self.visited[link] = None
if self.print_info:
print ("other status", resp.status, link)
- get_latency += time_time() - time_for_get
- total_gets += 1
- total_size += len(data)
- # get processing time header
- proc_time_header = resp.getheader("page_request_time",None)
- if proc_time_header != None:
- tmp = dateparser.parse(proc_time_header)
- proc_time += tmp.second + tmp.microsecond / 1000000.0
- if self.print_info:
- print proc_time_header
+ # write custom headers to a file -- too much overhead to
+ # embed in the test. parse(datetime) is apparently expensive
+ self.handle_custom_headers(curr_file, link, resp, first_link)
+ first_link = False
- latency = time_time() - start_timer
- self.custom_timers['Page File'] = latency
+ # Record some custom stuff
+ self.custom_timers['Page File'] = get_latency
self.custom_timers['Avg Get'] = get_latency / total_gets
self.stand_alone_stats['Total Gets'] = total_gets
self.stand_alone_stats['Total Size kb'] = total_size / 1024.0
- self.stand_alone_stats['Page File Proc'] = proc_time
- self.stand_alone_stats['Get Proc'] = proc_time / total_gets
+ # Get run time for entire test
+ latency = time_time() - start_timer
+ self.custom_timers['Total Test'] = latency
+
except Exception, e:
traceback.print_exc()
@@ -142,26 +167,32 @@
trans = Transaction()
trans.script_options=sys.argv[1:]
avg_pg_file = 0
- avg_pg_proc = 0
- avg_get_proc = 0
avg_get = 0
avg_num_get = 0
avg_kb =0
+ avg_total_test = 0
samples = 0
- while not trans.run():
+ while True:
+ trans.run()
samples += 1
avg_pg_file += trans.custom_timers['Page File']
- avg_pg_proc += trans.stand_alone_stats['Page File Proc']
- avg_get_proc += trans.stand_alone_stats['Get Proc']
avg_get += trans.custom_timers['Avg Get']
+ avg_total_test += trans.custom_timers['Total Test']
avg_num_get += trans.stand_alone_stats['Total Gets']
avg_kb += trans.stand_alone_stats['Total Size kb']
+
+ # If an iteration limit was set, bail after going through file list
+ # that many times
+ if trans.iteration_limit and trans.file_iterations >= trans.iteration_limit:
+ break
- print "page: ", avg_pg_file / samples, \
- "page proc: ", avg_pg_proc / samples, \
- "get proc: ", avg_get_proc / samples, \
- "get:", avg_get / samples, "num_get:", avg_num_get /samples,
"kb:", avg_kb / samples
+ print "avg page: ", avg_pg_file / samples, \
+ "\navg get:", avg_get / samples, \
+ "\navg num_get:", avg_num_get /samples, \
+ "\navg page kb:", avg_kb / samples, \
+ "\navg test:", avg_total_test / samples
+
#print trans.custom_timers
#print trans.stand_alone_stats