Php - 批量客戶創建
[英]Php - Bulk Customer Creation
問題描述
這是我的腳本，它從CSV中獲取50k記錄並使用它們進行CRUD操作。 不幸的是，性能存在很多問題。
CSV字段為： msisdn, resellerid, product
以下是執行的步驟：
- 閱讀CSV。
- 創建臨時表。
- 選擇前1000條記錄並使用隨機字符串標記狀態以將其鎖定。
- 執行這些記錄，同時另一個請求到達此文件，然后在鎖定狀態下執行下一個1000批次。
- 它讀取鎖定的狀態記錄並進行API調用。 如果呼叫是肯定的，那么它會創建客戶帳戶的記錄，否則繼續進行下一條記錄的API調用。
如果我們上傳記錄1k，2k，最多7k的CSV，代碼工作正常。
當我們像10k及更高版本一樣上傳時，處於鎖定狀態的相同記錄仍然被調用兩次並且多次，導致需要避免的不需要的API調用。 有時，當API調用成功時，即使沒有創建客戶帳戶。
所以我特此粘貼代碼，以便高技能的PHP和Mysql專家可以查看代碼並建議他們的改進視圖。
<?php
/*********************************************************************************
** The contents of this file are subject to the crm CRM Public License Version 1.0
* ("License"); You may not use this file except in compliance with the License
* The Original Code is: crm CRM Open Source
* The Initial Developer of the Original Code is crm.
* Portions created by crm are Copyright (C) crm.
* All Rights Reserved.
*
********************************************************************************/
ini_set('max_execution_time', 30000);
set_time_limit(0);
global $adb, $log;
require_once('include/utils/utils.php');
require_once('include/database/PearDatabase.php');
include_once 'includes/main/WebUI.php';
$result = $adb->pquery("SELECT bulkcustomerid,billingarea,apilogid FROM crm_bulkcustomer where `statusrecord` = 'inprogress'");
$noOfRows = $adb->num_rows($result);
if (!$noOfRows) {
$result = $adb->pquery("SELECT bulkcustomerid,billingarea,apilogid FROM crm_bulkcustomer where `statusrecord` = 'new'");
$noOfRows = $adb->num_rows($result);
}
if (!$noOfRows) {
// add inner join with crm_crmentity for created date interval day for one day.
//$result = $adb->pquery("SELECT bulkcustomerid,billingarea,apilogid FROM crm_bulkcustomer where `statusrecord` = 'closedwithfailure'");
$result = $adb->pquery("SELECT bulkcustomerid,billingarea,apilogid FROM crm_bulkcustomer inner join crm_crmentity on
crm_bulkcustomer.bulkcustomerid=crm_crmentity.crmid where `statusrecord` = 'closedwithfailure'");
$noOfRows = $adb->num_rows($result);
$isClosedWf = TRUE;
}
$process_data = 1000;
$log->debug('TOTAL NUMBEROFCSVS ' . $noOfRows);
$length = 9;
$randomString = substr(str_shuffle("0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ"), 0, $length);
if ($noOfRows > 0) { // for number of csv files .
for ($i = 0; $i < $noOfRows; $i++) {
$bulkcustomerid = $adb->query_result($result, $i, 'bulkcustomerid');
$apilogid = $adb->query_result($result, $i, 'apilogid'); // update the records for processing
$billingarea = $adb->query_result($result, $i, 'billingarea');
if (!$apilogid) {
$apilogid = 0;
}
$adb->query("UPDATE crm_bulkcustomer SET `statusrecord` = 'inprogress' where `bulkcustomerid` = '$bulkcustomerid'");
if ($isClosedWf) {
$log->debug('IN CLOSED WITH FAILURE PART');
//sleep(6800);
$adb->query("UPDATE temp_csv_$bulkcustomerid SET `status` = '$randomString' where STATUS = '0' OR STATUS REGEXP '[a-z]' ORDER BY id ASC LIMIT $process_data");
$servicetype = "Prepaid";
$log->debug('CRON TASK - After validating Call Create Customer API for Record ID ' . $bulkcustomerid);
$log->debug('select msisdn,package from temp_csv_$bulkcustomerid where status=' . $randomString . ' limit' . $process_data . ' FOR UPDATE');
//$transaction = "START TRANSACTION;";
//$adb->pquery($transaction, array());
$result2 = $adb->query("select msisdn,package from temp_csv_$bulkcustomerid where status='$randomString' limit $process_data");
$noOfRows2 = $adb->num_rows($result2);
$log->debug('Total row for this table is:- ' . $noOfRows2);
if ($noOfRows2 > 0) {
for ($j = 0; $j < $noOfRows2; $j++) {
$msisdnnum = $adb->query_result($result2, $j, 'msisdn');
$package = $adb->query_result($result2, $j, 'package');
/***** commented MSISDN
$log->debug('CRON TASK - Call Create Customer API for Table ID temp_csv_' . $bulkcustomerid . 'and mobile number is' . $msisdnnum);
$log->debug("for IF part BulkCustomer_SearchSim_Action::search_msisdn($msisdnnum,$apilogid)");
$result1 = BulkCustomer_SearchSim_Action::search_msisdn($msisdnnum, $apilogid, $bulkcustomerid);
$message = $result1['message'];
$code = $result1['code'];
if ($code == 0) {
$log->debug('MESSAGE IS--for code 0 ' . $message . 'code is--' . $code);
$log->debug('Sucess UPDATE crm_bulkcustomer SET statusrecord = closed where bulkcustomerid=' . $bulkcustomerid);
******/
$bulkresult = BulkCustomer_CreateCustomer_Action::Create_Customer($msisdnnum, $bulkcustomerid, $package, $servicetype,$billingarea,$apilogid);
$log->debug('CUSTOMER WITH ALL ACCOUNTS CREATED SUCCESSFULLY FOR MOBILENO=' . $msisdnnum . 'and current j value is=' . $j . 'and no of records to parse are' . $noOfRows2);
$bulkcode = $bulkresult['code'];
$bulkmessage = $bulkresult['message'];
if ($bulkcode == '0') {
$adb->query("UPDATE temp_csv_$bulkcustomerid SET `status` = '1',remarks='$bulkmessage' where `msisdn` = '$msisdnnum'");
} else {
$adb->query("UPDATE temp_csv_$bulkcustomerid SET `status` = '-6',remarks='$bulkmessage' where `msisdn` = '$msisdnnum'");
}
/****** search MSISDN } else {
$log->debug('MESSAGE IS-- for code -1 ' . $message . 'code is--' . $code);
$log->debug('Failure UPDATE temp_csv_' . $bulkcustomerid . 'SET status = Failed where msisdn =' . $msisdnnum);
$adb->query("UPDATE temp_csv_$bulkcustomerid SET `status` = '-9',remarks='$message' where `msisdn` = '$msisdnnum'");
}
search MSISDN *****/
$log->debug('j value is for loop' . $j);
}
}else{
$log->debug('No data to parse');
$adb->query("UPDATE `crm_bulkcustomer` SET `statusrecord` = 'closed' where `bulkcustomerid` = '$bulkcustomerid'");
}
} else { // for locking the columns with -7
$log->debug('IN PROGESS OR NEW PART');
$adb->query("UPDATE temp_csv_$bulkcustomerid SET `status` = '$randomString' where `status` = '0' ORDER BY id ASC LIMIT $process_data");
$servicetype = "prepaid";
$log->debug('CRON TASK - After validating Call Create Customer API for Record ID ' . $bulkcustomerid);
$log->debug('select msisdn,package from temp_csv_$bulkcustomerid where status=' . $randomString . ' limit' . $process_data . 'FOR UPDATE');
//$transaction = "START TRANSACTION;";
//$adb->pquery($transaction, array());
$result2 = $adb->query("select msisdn,package from temp_csv_$bulkcustomerid where status='$randomString' limit $process_data");
$noOfRows2 = $adb->num_rows($result2);
$log->debug('Total row for this table is:- ' . $noOfRows2);
if ($noOfRows2 > 0) {
for ($j = 0; $j < $noOfRows2; $j++) {
$msisdnnum = $adb->query_result($result2, $j, 'msisdn');
$package = $adb->query_result($result2, $j, 'package');
/***** commented MSISDN
$log->debug("ELSE BulkCustomer_SearchSim_Action::search_msisdn($msisdnnum,$apilogid)");
$result1 = BulkCustomer_SearchSim_Action::search_msisdn($msisdnnum, $apilogid, $bulkcustomerid);
$message = $result1['message'];
$code = $result1['code'];
if ($code == 0) {
$log->debug('MESSAGE IS--for code 0 ' . $message . 'code is--' . $code);
$log->debug('Sucess UPDATE crm_bulkcustomer SET statusrecord = closed where bulkcustomerid=' . $bulkcustomerid);
******/
$bulkresult = BulkCustomer_CreateCustomer_Action::Create_Customer($msisdnnum, $bulkcustomerid, $package, $servicetype,$billingarea,$apilogid);
$log->debug('CUSTOMER WITH ALL ACCOUNTS CREATED SUCCESSFULLY FOR MOBILENO=' . $msisdnnum . 'and current j value is=' . $j . 'and no of records to parse are' . $noOfRows2);
$bulkcode = $bulkresult['code'];
$bulkmessage = $bulkresult['message'];
if ($bulkcode == '0') {
$adb->query("UPDATE temp_csv_$bulkcustomerid SET `status` = '1',remarks='$bulkmessage' where `msisdn` = '$msisdnnum'");
} else {
$adb->query("UPDATE temp_csv_$bulkcustomerid SET `status` = '-6',remarks='$bulkmessage' where `msisdn` = '$msisdnnum'");
}
/****** } else {
$log->debug('Failure UPDATE temp_csv_' . $bulkcustomerid . 'SET status = Failed where msisdn =' . $msisdnnum);
$adb->query("UPDATE temp_csv_$bulkcustomerid SET `status` = '-9',remarks='$message' where `msisdn` = '$msisdnnum'");
}
comments ends for MSISDN ****/
$log->debug('j value is for loop' . $j);
}
}else{
$log->debug('No data to parse inprogress or new part');
$adb->query("UPDATE `crm_bulkcustomer` SET `statusrecord` = 'closedwithfailure' where `bulkcustomerid` = '$bulkcustomerid'");
}
//commit
} // for else part
// close the csv file
// get count of sucess and failure.
$sucesscount = $adb->pquery("SELECT * FROM temp_csv_$bulkcustomerid where `status` = '1'"); // fetch record for new
$noofsucess = $adb->num_rows($sucesscount);
$adb->query("UPDATE `crm_bulkcustomer` SET `successcount` = '$noofsucess' where `bulkcustomerid` = '$bulkcustomerid'");
// failure count
$failurecount = $adb->pquery("SELECT * FROM temp_csv_$bulkcustomerid where status!='1'"); // fetch record for fails
$nooffailure = $adb->num_rows($failurecount);
$reprocesscount = $adb->pquery("SELECT * FROM temp_csv_$bulkcustomerid where STATUS = '0' OR STATUS REGEXP '[a-z]'");
// fetch record for fails
//$noofreprocesscount = 0;
$noofreprocesscount = $adb->num_rows($reprocesscount);
$log->debug("UPDATE `crm_bulkcustomer` SET `failurecount` = '$nooffailure' where `bulkcustomerid` = '$bulkcustomerid'");
if ($nooffailure == '0') {
$adb->query("UPDATE `crm_bulkcustomer` SET `statusrecord` = 'closed' where `bulkcustomerid` = '$bulkcustomerid'");
$adb->query("UPDATE `crm_bulkcustomer` SET `failurecount` = '' where `bulkcustomerid` = '$bulkcustomerid'");
} elseif ($noofreprocesscount) {
// check if it is being closed before all records are executed
$adb->query("UPDATE `crm_bulkcustomer` SET `statusrecord` = 'inprogress' where `bulkcustomerid` = '$bulkcustomerid'");
} else {
$adb->query("UPDATE `crm_bulkcustomer` SET `failurecount` = '$nooffailure' where `bulkcustomerid` = '$bulkcustomerid'");
$adb->query("UPDATE `crm_bulkcustomer` SET `statusrecord` = 'closedwithfailure' where `bulkcustomerid` = '$bulkcustomerid'");
$filename = "$bulkcustomerid" . "-csv";
$file = fopen("tempcsv/$filename.csv", "w");
$val = array(
'MSISDN',
'ResellerID',
'Package',
'Remarks'
);
fputcsv($file, $val, ';', ' ');
$csvresult = $adb->query("select msisdn,package,resellerid,remarks from temp_csv_$bulkcustomerid where status!='1' ");
$csvrows = $adb->num_rows($csvresult);
$log->debug('Total row for this table in CSV PART is:- ' . $csvrows);
if ($csvrows > 0) {
for ($m = 0; $m < $csvrows; $m++) {
$msisdnnum = $adb->query_result($csvresult, $m, 'msisdn');
$package = $adb->query_result($csvresult, $m, 'package');
$resellerid = $adb->query_result($csvresult, $m, 'resellerid');
$remarks = $adb->query_result($csvresult, $m, 'remarks');
$remarks = preg_replace('/\s+/', '--', $remarks);
$arrayval = array(
"$msisdnnum",
"$resellerid",
"$package",
"$remarks"
);
fputcsv($file, $arrayval, ';', ' ');
}
}
}
}
} else {
$log->debug('No status with NEW or Pending of CSV RECORDS found');
}
?>
1 個解決方案
解決方案1
0 2016-04-19 07:14:14
這樣做的一個原因可能是，在你擁有的這些50K記錄中，可能有一些條目（7K之后）是模糊的，這就是問題所在。
所以也許嘗試創建大量的記錄，並試圖找出哪個塊正在給你一個問題，並試圖找出該條目。 這只是一個普遍的想法。
PHP - 條帶客戶端/客戶創建不起作用
聲明:本站的技術帖子網頁，遵循CC BY-SA 4.0協議，如果您需要轉載，請注明本站網址或者原文地址。任何問題請咨詢:yoyou2525@163.com.