Merge branch 'master' of https://github.com/Node-DC/Node-DC-EIS

Node-DC-EIS-client/README.md

@@ -26,10 +26,11 @@ This client directory contains,
         - You will need to change the IP address and port of your server in ‘runspec.py’ or in config.json. 
         - Takes additional command line parameters.
         - Default parameters in the script or can be read from a configuration file with -f/--config option (command line has the maximum priority).
-        -h gives the available options.
+        - h gives the available options.
+        - Configurable option -g or --showgraph for graph generation(1 to generate output graphs or 0 for no graph)
         - the server ip address and port can be changed in config.json or directly in runspec.py
-        -This script sends requests using python's requests module, generates log file and generates output graphs.
- 
+        - This script sends requests using python's requests module, generates log file and generates output graphs.
+
 
 ## Client result files
    - A temporary log file (request-based-run) which contains details like request-number,write-time, read-time, response-time for every request.

Node-DC-EIS-client/config.json

@@ -5,7 +5,7 @@
     "request" : "10000",
     "concurrency" : "200",
     "run_mode" : "1",
-    "interval" : "15",
+    "interval" : "10",
     "rampup_rampdown": "25",
     "tempfile":"RTdata",
     "total_urls":"100",
@@ -14,20 +14,20 @@
     "root_endpoint": "/",
     "nameurl_count":"25", 
     "zipurl_count":"25",
-    "idurl_ratio":"50",
+    "idurl_count":"50",
     "html": false
   },
 "db_params": 
   { 
     "dbrecord_count": "10000", 
-    "name_ratio":"25",
-    "zip_ratio":"25"
+    "name_ratio":"5",
+    "zip_ratio":"5"
   },
 "url_params": 
   { 
-    "get_ratio": "100", 
-    "post_ratio":"0",
-    "delete_ratio":"0"
+    "get_ratio": "90", 
+    "post_ratio":"5",
+    "delete_ratio":"5"
   },
 "memory_params": 
   { 

Node-DC-EIS-client/node_dc_eis_testurls.py

@@ -52,6 +52,8 @@
 conn_err = 0
 http_err = 0
 bad_url = 0
+static_post = 0
+
 s = requests.Session()
 a = requests.adapters.HTTPAdapter(max_retries=20)
 b = requests.adapters.HTTPAdapter(max_retries=20)
@@ -81,7 +83,7 @@ def get_ip(hostname):
   ip_cache[hostname] = ip
   return ip
 
-def get_url(url, request_num, log, phase, accept_header):
+def get_url(url, url_type, request_num, log, phase, accept_header):
   """
   # Desc  : Function to send get requests to the server. Type 1 is get requests
   #         handles 3 types of GET requests based on ID, last_name and zipcode.
@@ -94,6 +96,7 @@ def get_url(url, request_num, log, phase, accept_header):
   global http_err
   global bad_url 
   data = ""
+  headers = ""
 
   urlo = urlparse.urlparse(url)
   ip = get_ip(urlo.hostname)
@@ -117,10 +120,12 @@ def get_url(url, request_num, log, phase, accept_header):
     sock.settimeout(30)
     sock.connect((ip, urlo.port))
     sock.sendall(req)
-    while len(data) < 15:
-      data += sock.recv(15)
+    while "\r\n\r\n" not in data:
+      data = sock.recv(256)
+      headers += data
+
     http_status = data[9:12]
-    if http_status[0] != '3' or http_status[0] != '2':
+    if http_status[0] != '2':
       http_err += 1
 
   except socket.timeout:
@@ -131,7 +136,9 @@ def get_url(url, request_num, log, phase, accept_header):
     sock.close()
     end = time.time()
     response_time = end - start
-  util.printlog(log,phase,request_num,url,start,end,response_time)
+    total_length = calculate_len_get(headers)
+
+  util.printlog(log,phase,url_type,request_num,url,start,end,response_time,total_length)
   return 
 
 def post_function(url, post_data):
@@ -175,7 +182,7 @@ def post_function(url, post_data):
     print e
   return r
 
-def post_url(url,request_num,log,phase): 
+def post_url(url,url_type,request_num,log,phase): 
   """
   # Desc  : Function to send post requests to the server. Type 2 is post requests
   #         Retries if the post request fails
@@ -191,16 +198,21 @@ def post_url(url,request_num,log,phase):
   global http_err
   global bad_url
   global post_datalist
+  global static_post
   r = None
+
   post_data = static_postdata
   if post_datalist:
     post_data = post_datalist[0]
+  else:
+    static_post = static_post + 1
   start = time.time()
   end = start
   for i in range(100):
     r =  post_function(url, post_data)
     if r:
       end = time.time()
+      total_length = calculate_len_postdel(r)
       break
   if not r:
     print "Post request failed. Received null response.Exiting run"
@@ -221,10 +233,11 @@ def post_url(url,request_num,log,phase):
       print("Exception -- Post did not return a valid employee_id")
       print post_data
       exit(1)
-  util.printlog(log,phase,request_num,url,start,end,response_time)
+
+  util.printlog(log,phase,url_type,request_num,url,start,end,response_time,total_length)
   return
 
-def delete_url(url,request_num,log,phase): 
+def delete_url(url,url_type,request_num,log,phase): 
   """
   # Desc  : Function to send delete requests to the server. Type 3 is delete requests
   #         also captures the data record being deleted and saves it in a list(post/_datalist)
@@ -239,7 +252,7 @@ def delete_url(url,request_num,log,phase):
   global bad_url 
   global employee_idlist
   global post_datalist
-  start = time.time()
+
   #1. issue a get request to get the record that is being deleted
   try:
     try:
@@ -276,10 +289,51 @@ def delete_url(url,request_num,log,phase):
   finally:
     end = time.time()
     response_time = end-start
+    total_length = calculate_len_postdel(r)
 
-  util.printlog(log,phase,request_num,url,start,end,response_time)
+  util.printlog(log,phase,url_type,request_num,url,start,end,response_time,total_length)
   return
 
+def calculate_len_get(headers):
+  """
+  # Desc  : Function to calculate total length of data received for a get request.
+  #         Calculated as sum of header length and the content length
+  # Input : Header from a get request
+  # Output: Returns the total length of data received
+  """
+  sock_header_len = 0 
+  sock_content_len = 0
+  total_length = 0
+  sock_header_len = headers.find('\r\n\r\n') + 4
+  for line in headers.splitlines():
+    if "content-length" in line.lower():
+      sock_content_len = int(line.split(':')[1].strip())
+  total_length = sock_header_len + sock_content_len
+  return total_length
+
+def calculate_len_postdel(response):
+  """
+  # Desc  : Function to calculate total length of data received for post/delete request.
+  #         Calculated as sum of header length and the content length.
+  # Input : response from post/delete request
+  # Output: Returns the total length of data received
+  """
+  header_len = 0
+  content_len = 0
+  total_length = 0
+  content_len = len(response.content)
+  header = response.headers
+  header_len = (
+    17 + # size of 'HTTP/1.1 200 OK\r\n' at the top
+    # size of keys + size of values
+    sum(len(kk) + len(vv) for kk, vv in header.iteritems()) +
+    # size of the extra ': ' between key and value and '\r\n' per header
+    len(header) * 4 +
+    2 # size of the empty line at the end
+  )
+  total_length = header_len + content_len
+  return total_length
+
 def main_entry(url, request_num, url_type, log_dir, phase, interval,
                run_mode, temp_log, accept_header):
   """
@@ -320,8 +374,9 @@ def main_entry(url, request_num, url_type, log_dir, phase, interval,
       sys.exit(1)
 
   if url_type == 1:
-    get_url(url, request_num, log, phase, accept_header)
+    get_url(url,url_type,request_num, log, phase, accept_header)
   if url_type == 2:
-    post_url(url,request_num,log,phase)
+    post_url(url,url_type,request_num,log,phase)
   if url_type == 3:
-    delete_url(url,request_num,log,phase)
+    delete_url(url,url_type,request_num,log,phase)
+