From 16fb2cf0da906974a5c4d7872cf753a4550a086c Mon Sep 17 00:00:00 2001 From: Korina Cordero Date: Thu, 16 Jun 2022 08:24:05 +0000 Subject: [PATCH] Add bulk load serial script. Modify bulk script for loading serials. #684 --- .../bulk_load_serials.php | 187 ++++++++++++++++++ utils/warranty_motiv_local_bulk.sh | 2 +- utils/warranty_motiv_prod_bulk.sh | 2 +- 3 files changed, 189 insertions(+), 2 deletions(-) create mode 100644 utils/load_warranty_serial/bulk_load_serials.php diff --git a/utils/load_warranty_serial/bulk_load_serials.php b/utils/load_warranty_serial/bulk_load_serials.php new file mode 100644 index 00000000..bfdd547a --- /dev/null +++ b/utils/load_warranty_serial/bulk_load_serials.php @@ -0,0 +1,187 @@ +loadEnv(__DIR__.'/../../.env'); + +$db_info = $_ENV['DATABASE_URL']; + +// sample format of db_info: mysql://db_user:db_password@127.0.0.1:3306/resq?charset=utf8 +// dsn for PDO needs to be: mysql:host=127.0.0.1:3306;dbname=resq;charset=UTF8 + +preg_match('/^mysql:\/\/(.*):(.*)@(.*):(.*)\/(.*)\?(.*)/', $db_info, $result); + +$db_type = 'mysql:host='; +$user = $result[1]; +$pass = $result[2]; +$ip_port = $result[3] . ':' . $result[4] . ';'; +$db_name = 'dbname=' . $result[5] . ';'; +$charset = $result[6]; + +$dsn = $db_type . $ip_port . $db_name . $charset; + +//error_log($dsn); +//error_log($user); +//error_log($pass); + +// connect to db +$db = new PDO($dsn, $user, $pass, array(PDO::MYSQL_ATTR_LOCAL_INFILE => true)); + +// go through rows +$source = 'motiv'; +$data_array = []; +while (($row = fgetcsv($csv)) !== false) +{ + // TODO: verify if these are still the headers if there are headers + // possible lines in output file: + // (1) header in csv file + // SerialNumber,Sku,DispatchStatus,CreatedDate,InventoryStatus,CategoryID,CategoryName + // (2) No available data + // (3) CH2000012071,WCHD23BL-CPN00-LX,0,2020-08-11 04:05:27.090,0,4,CHAMPION MF + // (4) Empty line + // (5) empty sku + + // check if No available data + if ($row[0] == 'No available data') + { + // skip the line + error_log('No available data, skipping the line...'); + continue; + } + + // check if empty line + if ($row == array(null)) + { + // skip + error_log('Skipping empty line...'); + continue; + } + + // check if empty serial + if (empty($row[0])) + { + $err_message = "Empty serial. " . "\n"; + fwrite($output_fh, $err_message); + continue; + } + + // sample of line in output file: + // serial number, sku, dispatch status, created date, inventory status, category id, category name + // CH2000012071,WCHD23BL-CPN00-LX,0,2020-08-11 04:05:27.090,0,4,CHAMPION MF + // MG2000313690,N/A,1,2021-05-14T23:47:30.6430000+08:00,0,10,GOLD + $serial = trim(strtoupper($row[0])); + $sku = trim(strtoupper($row[1])); + $dispatch_status = trim($row[2]); + $str_date_create = trim($row[3]); + $inventory_status = trim($row[4]); + $cat_id = trim($row[5]); + $cat_name = trim(strtoupper($row[6])); + + error_log('Processing ' . $serial . ' and ' . $sku); + + // since some people cannot follow simple instructions... + // check the date format on the string + // try 2021-05-15T08:35:46+08:00 format on str_date_create + $date_create = DateTime::createFromFormat('Y-m-d\TH:i:sP', $str_date_create); + + if ($date_create == false) + { + // try this format: 2021-05-15T08:47:20.3330000+08:00 + // get the date, time and timezone from str_date_create + $str_date_time = substr($str_date_create, 0, 19); + $str_timezone = substr($str_date_create, 27); + $str_datetime_tz = $str_date_time . $str_timezone; + + // create DateTime object + // sample: 2021-05-15T12:16:06+08:00 + $date_create = DateTime::createFromFormat('Y-m-d\TH:i:sP', $str_datetime_tz); + + // check if datetime object was created + // if not, someone f*cked up and we have no date create + if ($date_create == false) + { + // log the serial + $message = "$serial - ERROR - " . "Invalid date format for create date." . "\n"; + fwrite($output_fh, $message); + continue; + } + } + + $created_date = $date_create->format('Y-m-d H:i:s'); + + //error_log($str_date_time); + //error_log($str_timezone); + //error_log($str_datetime_tz); + //error_log($date_create->format('Y-m-d H:i:s')); + + $meta_info = [ + 'dispatch_status' => $dispatch_status, + 'inventory_status' => $inventory_status, + 'category_id' => $cat_id, + 'category_name' => $cat_name, + ]; + + $info = json_encode($meta_info); + + if ($sku == 'N/A') + $sku = null; + + // create data for the infile + $data_array[] = [ + $serial, + $sku, + $created_date, + $source, + $info + ]; +} + +// cache directory +$cache_dir = __DIR__ . '/../../var/cache'; + +$file = $cache_dir . '/warranty_serial_data.tab'; +error_log('opening file for warranty serial - ' . $file); + +$fp = fopen($file, 'w'); +if ($fp === false) +{ + error_log('could not open file for load data infile - ' . $file); +} +else +{ + foreach ($data_array as $key => $data) + { + $line = implode('|', $data) . "\r\n"; + fwrite($fp, $line); + } +} + +fclose($fp); + +error_log('Loading warranty serial data...'); +$db->exec('LOAD DATA LOCAL INFILE \'' . $file . '\' INTO TABLE warranty_serial FIELDS TERMINATED BY \'|\' LINES TERMINATED BY \'\\r\\n\' (id, sku, date_create, source, meta_info)'); + +// close file +fclose($csv); +fclose($output_fh); + diff --git a/utils/warranty_motiv_local_bulk.sh b/utils/warranty_motiv_local_bulk.sh index 384afe81..cf0f11eb 100755 --- a/utils/warranty_motiv_local_bulk.sh +++ b/utils/warranty_motiv_local_bulk.sh @@ -78,4 +78,4 @@ touch /tmp/warranty_download_serial.csv /usr/bin/php /var/www/resq/utils/get_warranty_serial/new_get_serials.php 06-15-22 /tmp/warranty_download_serial.csv 0 /usr/bin/php /var/www/resq/utils/get_warranty_serial/new_get_serials.php 06-16-22 /tmp/warranty_download_serial.csv 0 touch /tmp/warranty_load_status.txt -/usr/bin/php /var/www/resq/utils/load_warranty_serial/load_serials.php /tmp/warranty_download_serial.csv /tmp/warranty_load_status.txt +/usr/bin/php /var/www/resq/utils/load_warranty_serial/bulk_load_serials.php /tmp/warranty_download_serial.csv /tmp/warranty_load_status.txt diff --git a/utils/warranty_motiv_prod_bulk.sh b/utils/warranty_motiv_prod_bulk.sh index 384afe81..cf0f11eb 100755 --- a/utils/warranty_motiv_prod_bulk.sh +++ b/utils/warranty_motiv_prod_bulk.sh @@ -78,4 +78,4 @@ touch /tmp/warranty_download_serial.csv /usr/bin/php /var/www/resq/utils/get_warranty_serial/new_get_serials.php 06-15-22 /tmp/warranty_download_serial.csv 0 /usr/bin/php /var/www/resq/utils/get_warranty_serial/new_get_serials.php 06-16-22 /tmp/warranty_download_serial.csv 0 touch /tmp/warranty_load_status.txt -/usr/bin/php /var/www/resq/utils/load_warranty_serial/load_serials.php /tmp/warranty_download_serial.csv /tmp/warranty_load_status.txt +/usr/bin/php /var/www/resq/utils/load_warranty_serial/bulk_load_serials.php /tmp/warranty_download_serial.csv /tmp/warranty_load_status.txt