My python script is taking 18sec to insert 5lacs records to couchbase database with 5 threads, However same logic implemented in C++ takes 9secs to insert the same . Can anyone please help me to improve the python code performance.
import random
import multiprocessing
import string
from couchbase import Couchbase
from couchbase.cluster import Cluster, ClassicAuthenticator,PasswordAuthenticator
from couchbase.exceptions import CouchbaseError
from couchbase.n1ql import N1QLQuery
import logging,time
import ConfigParser
import io
import json
from couchbase.bucket import Bucket
threads = []
procs = []
def input_check():
for x in range(0,ThreadCount):
p = multiprocessing.Process(
target=conf_thread,
args=(x,))
procs.append(p)
p.start()
# Wait for all worker processes to finish
for p in procs:
p.join()
def conf_thread(x):
bucket = Bucket(couchbaseDBIP,password=couchbasePassword)
bucket.n1ql_timeout=10000
bucket.timeout=10000
opsSequence = []
letters = string.digits
rand_str = ( ''.join(random.choice(letters) for i in range(1000)) )
if OP_TYPE == 1 :
updateproportion = 100
readproportion = 0
elif OP_TYPE == 2 :
updateproportion = 0
readproportion = 100
elif OP_TYPE == 3 :
updateproportion = 50
readproportion = 50
count=0
for b in range(updateproportion):
opsSequence.append(1)
count+=1
for b in range(readproportion):
opsSequence.append(2)
count+=1
start_time = time.time()
readCount = 0
insertCount = 0
for j in range(0,NumofDoc): // NumofDoc 100000
k = j%100
optype = opsSequence[k]
c = readCount % 5000
d = insertCount % 10000
if optype == 1 :
insert_doc = "Python_"+str(x)+"_"+str(insertCount)
#writehandler(bucket,insert_doc,x)
list1={"Thread_id":str(x), "KTAB":"INSERT","SyncBuffer":rand_str }
bucket.upsert(insert_doc,list1)
insertCount = insertCount+1
elif optype == 2:
get_doc = "Python_"+str(x)+"_"+str(c)
getDocument(bucket,get_doc)
readCount+=1
else :
print "Invalid Operation"
timediff = time.time() - start_time
avgLatency = (timediff)/(insertCount+readCount);
opsPerSec = 1/avgLatency;
print("THREAD_ID :"+str(x)+",TOTAL WRITE : "+str(insertCount)+", TOTAL READ : "+str(readCount)+", TOTAL OPERATION TIME : "+str(timediff)+", AVG_LATENCY = "+str(avgLatency)+", OPS_PER_SECOND ="+str(opsPerSec))
with open("config.ini") as f:
sample_config = f.read()
config = ConfigParser.RawConfigParser(allow_no_value=True)
config.readfp(io.BytesIO(sample_config))
NumofDoc=config.getint("couchbase", "NumofDoc") // 100000
couchbaseDBIP=config.get("couchbase", "ServerIp")
couchbaseUserName=config.get("couchbase", "Username")
couchbasePassword=config.get("couchbase", "Password")
couchbaseBucketName=config.get("couchbase", "BucketName")
ThreadCount=config.getint("couchbase", "ThreadCount") //5
Port=config.getint("couchbase", "Port")
OP_TYPE=config.getint("couchbase", "OP_TYPE") // 1 -insert , 2-get
print "Config File name Passed : Config.ini"
print "ThreadCount : "+str(ThreadCount)
print "Server IP : "+couchbaseDBIP
print "Number of Requests per thread : "+str(NumofDoc)
input_check()