der_columns="Fcheck_bank, Facc_day, Fbankaccno, Fbankusername, Famount, Fdc_type_id, Fori_accno, Fbank_status,Fid,Fbank_listid,Fbatchno"
#data
order_base="'4251','2018022721','6225425177777777800004','wltest','1','16','6225425177777777800004','00'"
tablename ="epcc_check_201810.t_dc_list_06"
def writeDownSqlData(fileName,content):
fo = open(fileName,"w")
fo.write(content)
fo.close()
def make_t_tcpay_list(deal_num,Flistid,Ftde_id,Fbank_list):
filename ="data_order_"+Fbank_list+".text"
Fbatchno = 'B201810070011'
orderDataList = []
for i in range(int(deal_num)):
orderData=""
orderData = "%s,'%s','%s','%s'" % (order_base, Ftde_id, Fbank_list, Fbatchno)
orderDataList.append(orderData)
Ftde_id = int(Ftde_id) + 1
Flistid = int(Flistid) + 1
Fbank_list = int(Fbank_list) + 1
writeDownSqlData(resultDir+filename, "\n".join(orderDataList) + "\n")
loadDataIntoDB(resultDir+filename,tablename,order_columns)
def loadDataIntoDB(filename,tableName,order_columns):
mysqlCmd = r"LOAD DATA LOCAL INFILE '%s' into table %s FIELDS TERMINATED BY ',' OPTIONALLY ENCLOSED BY '\'' LINES TERMINATED BY '\n' (%s"%(filename,tableName,order_columns)+")"
mysqlConCmd = r"mysql -u%s -p%s -h%s"%(dbUser,dbPasswd,dbHost)
runCmd = mysqlConCmd+' -e "'+mysqlCmd+'"'
result = call(runCmd,shell=True)
if result != 0:
print "load local data into database failed,exit "
sys.exit()
if __name__ == "__main__":
start = time.time()
begin_Flistid ='110180809100012153304210311120'
begin_Ftde_id ="1"
begin_Fbank_seq ="2018100800000110734321790770100"
total_num =20000000
threadpool_num = 20
func_var=[]
seperate =2000
pool = threadpool.ThreadPool(threadpool_num)
for i in range(seperate):
list_temp =[]
list_temp =[str(total_num/seperate),begin_Flistid,begin_Ftde_id,begin_Fbank_seq]
func_var.append((list_temp,None))
begin_Flistid = str(int(begin_Flistid)+ total_num/threadpool_num)
begin_Ftde_id = str(int(begin_Ftde_id)+ total_num/threadpool_num)
begin_Fbank_seq = str(int(begin_Fbank_seq)+ total_num/threadpool_num)
pool = threadpool.ThreadPool(threadpool_num)
requests = threadpool.makeRequests(make_t_tcpay_list, func_var)
for req in requests:
pool.putRequest(req)
pool.wait()
end = time.time()
print end - start