From b300814b500523c6ed9f72e80eef1efbc5fd7de8 Mon Sep 17 00:00:00 2001 From: Korina Cordero Date: Fri, 4 Jun 2021 10:22:28 +0000 Subject: [PATCH] Add error checking to processing script. Create new script for downloading files from motiv. #576 --- utils/get_warranty_serial/get_serials.php | 17 ++++-- utils/get_warranty_serial/new_get_serials.php | 49 +++++++++++++++ utils/load_warranty_serial/load_serials.php | 59 +++++++++++++++---- utils/warranty_motiv_local_bulk.sh | 36 +++++++++++ 4 files changed, 143 insertions(+), 18 deletions(-) create mode 100644 utils/get_warranty_serial/new_get_serials.php create mode 100755 utils/warranty_motiv_local_bulk.sh diff --git a/utils/get_warranty_serial/get_serials.php b/utils/get_warranty_serial/get_serials.php index 3a0f3d24..e9538e3f 100644 --- a/utils/get_warranty_serial/get_serials.php +++ b/utils/get_warranty_serial/get_serials.php @@ -20,14 +20,20 @@ $conn_string = "BlobEndpoint=$blob_url;\nSharedAccessSignature=$sas_token"; $blob_client = BlobRestProxy::createBlobService($conn_string); -$current_date = new DateTime(); -$current_date->modify("-1 day"); +// get date argument +$proc_date = $argv[1]; -$date = $current_date->format('m-d-Y'); +error_log($proc_date); -$filename = 'warrantylogs' . $date . '.csv'; +//$current_date = new DateTime(); +//$current_date->modify("-1 day"); + +//$date = $current_date->format('m-d-Y'); + +$filename = 'warrantylogs' . $proc_date . '.csv'; error_log($filename); +/* try { // NOTE: via download blob $res = $blob_client->getBlob('warranty', $filename); @@ -36,8 +42,7 @@ try { file_put_contents("/tmp/warranty_download_serial.txt", $res->getContentStream()); } catch (Exception $e) { file_put_contents("/tmp/serial_download_error.txt", $filename . "\n" . $e->getMessage() . "\n" . "\n", FILE_APPEND); -} - +} */ /* // NOTE: getting via url diff --git a/utils/get_warranty_serial/new_get_serials.php b/utils/get_warranty_serial/new_get_serials.php new file mode 100644 index 00000000..45161f3a --- /dev/null +++ b/utils/get_warranty_serial/new_get_serials.php @@ -0,0 +1,49 @@ +getBlob('warranty', $filename); + // print_r($res); + + if ($flag_overwrite > 0) + { + file_put_contents($output_file, $res->getContentStream()); + } + else + { + file_put_contents($output_file, "\r\n", FILE_APPEND); + file_put_contents($output_file, $res->getContentStream(), FILE_APPEND); + } +} catch (Exception $e) { + file_put_contents("/tmp/serial_download_error.txt", $filename . "\n" . $e->getMessage() . "\n" . "\n", FILE_APPEND); +} + + + + + diff --git a/utils/load_warranty_serial/load_serials.php b/utils/load_warranty_serial/load_serials.php index cd1cce90..6eb1d918 100644 --- a/utils/load_warranty_serial/load_serials.php +++ b/utils/load_warranty_serial/load_serials.php @@ -5,6 +5,9 @@ $csv = fopen($argv[1], 'r'); $dsn = $argv[2]; $user = $argv[3]; $pass = $argv[4]; +$output_file = $argv[5]; + +$output_fh = fopen($output_file, "w"); // connect to db $db = new PDO($dsn, $user, $pass); @@ -13,33 +16,54 @@ $db = new PDO($dsn, $user, $pass); $sth = $db->prepare('insert into warranty_serial (id, sku, date_create, source, meta_info) values (:serial, :sku, :date_create, :source, :meta_info)'); // go through rows -$counter = 0; $source = 'motiv'; while (($row = fgetcsv($csv)) !== false) { - // skip first line - if ($counter == 0) + // TODO: verify if these are still the headers if there are headers + // possible lines in output file: + // (1) header in csv file + // SerialNumber,Sku,DispatchStatus,CreatedDate,InventoryStatus,CategoryID,CategoryName + // (2) No available data + // (3) CH2000012071,WCHD23BL-CPN00-LX,0,2020-08-11 04:05:27.090,0,4,CHAMPION MF + // (4) Empty line + + // check if csv file has a header by checking contents of line + if (($row[0] == 'SerialNumber') || ($row[1] == 'Sku') || + ($row[2] == 'DispatchStatus') || ($row[3] == 'CreatedDate') || + ($row[4] == 'InventoryStatus') || ($row[5] == 'CategoryID') || + ($row[6] == 'CategoryName')) { - $counter++; + // skip the header + error_log('Skipping the headers... '); continue; } - /* - $serial = trim(strtoupper($row[0])); - $sku = trim($row[1]); - $date_create = $row[2]; - $ref_id = $row[3]; */ + // check if No available data + if ($row[0] == 'No available data') + { + // skip the line + error_log('No available data, skipping the line...'); + continue; + } + // check if new line + if (empty($row[0])) + { + // skip + error_log('Skipping empty line...'); + continue; + } + // sample of line in output file: // serial number, sku, dispatch status, created date, inventory status, category id, category name // CH2000012071,WCHD23BL-CPN00-LX,0,2020-08-11 04:05:27.090,0,4,CHAMPION MF $serial = trim(strtoupper($row[0])); - $sku = trim($row[1]); + $sku = trim(strtoupper($row[1])); $dispatch_status = trim($row[2]); $date_create = $row[3]; $inventory_status = trim($row[4]); $cat_id = trim($row[5]); - $cat_name = trim($row[6]); + $cat_name = trim(strtoupper($row[6])); $meta_info = [ 'dispatch_status' => $dispatch_status, @@ -50,6 +74,8 @@ while (($row = fgetcsv($csv)) !== false) $info = json_encode($meta_info); + error_log('Processing ' . $serial . ' and ' . $sku); + if ($sku == 'N/A') $sku = null; @@ -63,12 +89,21 @@ while (($row = fgetcsv($csv)) !== false) if (!$res) { + // log the error $err = $sth->errorInfo(); - echo "Error ($serial) - " . $err[2] . "\n"; + $err_message = "$serial - ERROR - " . $err[2] . "\n"; + fwrite($output_fh, $err_message); + } + else + { + // log successful adding of serial + $message = "$serial - SUCCESS - " . "\n"; + fwrite($output_fh, $message); } } // close file fclose($csv); +fclose($output_fh); diff --git a/utils/warranty_motiv_local_bulk.sh b/utils/warranty_motiv_local_bulk.sh new file mode 100755 index 00000000..167e8566 --- /dev/null +++ b/utils/warranty_motiv_local_bulk.sh @@ -0,0 +1,36 @@ +#!/bin/bash +touch /tmp/warranty_download_serial.csv +/usr/bin/php /var/www/resq/utils/get_warranty_serial/new_get_serials.php 05-03-2021 /tmp/warranty_download_serial.csv 1 +/usr/bin/php /var/www/resq/utils/get_warranty_serial/new_get_serials.php 05-04-2021 /tmp/warranty_download_serial.csv 0 +/usr/bin/php /var/www/resq/utils/get_warranty_serial/new_get_serials.php 05-05-2021 /tmp/warranty_download_serial.csv 0 +/usr/bin/php /var/www/resq/utils/get_warranty_serial/new_get_serials.php 05-06-2021 /tmp/warranty_download_serial.csv 0 +/usr/bin/php /var/www/resq/utils/get_warranty_serial/new_get_serials.php 05-07-2021 /tmp/warranty_download_serial.csv 0 +/usr/bin/php /var/www/resq/utils/get_warranty_serial/new_get_serials.php 05-08-2021 /tmp/warranty_download_serial.csv 0 +/usr/bin/php /var/www/resq/utils/get_warranty_serial/new_get_serials.php 05-09-2021 /tmp/warranty_download_serial.csv 0 +/usr/bin/php /var/www/resq/utils/get_warranty_serial/new_get_serials.php 05-10-2021 /tmp/warranty_download_serial.csv 0 +/usr/bin/php /var/www/resq/utils/get_warranty_serial/new_get_serials.php 05-11-2021 /tmp/warranty_download_serial.csv 0 +/usr/bin/php /var/www/resq/utils/get_warranty_serial/new_get_serials.php 05-12-2021 /tmp/warranty_download_serial.csv 0 +/usr/bin/php /var/www/resq/utils/get_warranty_serial/new_get_serials.php 05-13-2021 /tmp/warranty_download_serial.csv 0 +/usr/bin/php /var/www/resq/utils/get_warranty_serial/new_get_serials.php 05-14-2021 /tmp/warranty_download_serial.csv 0 +/usr/bin/php /var/www/resq/utils/get_warranty_serial/new_get_serials.php 05-15-2021 /tmp/warranty_download_serial.csv 0 +/usr/bin/php /var/www/resq/utils/get_warranty_serial/new_get_serials.php 05-16-2021 /tmp/warranty_download_serial.csv 0 +/usr/bin/php /var/www/resq/utils/get_warranty_serial/new_get_serials.php 05-17-2021 /tmp/warranty_download_serial.csv 0 +/usr/bin/php /var/www/resq/utils/get_warranty_serial/new_get_serials.php 05-18-2021 /tmp/warranty_download_serial.csv 0 +/usr/bin/php /var/www/resq/utils/get_warranty_serial/new_get_serials.php 05-19-2021 /tmp/warranty_download_serial.csv 0 +/usr/bin/php /var/www/resq/utils/get_warranty_serial/new_get_serials.php 05-20-2021 /tmp/warranty_download_serial.csv 0 +/usr/bin/php /var/www/resq/utils/get_warranty_serial/new_get_serials.php 05-21-2021 /tmp/warranty_download_serial.csv 0 +/usr/bin/php /var/www/resq/utils/get_warranty_serial/new_get_serials.php 05-22-2021 /tmp/warranty_download_serial.csv 0 +/usr/bin/php /var/www/resq/utils/get_warranty_serial/new_get_serials.php 05-23-2021 /tmp/warranty_download_serial.csv 0 +/usr/bin/php /var/www/resq/utils/get_warranty_serial/new_get_serials.php 05-24-2021 /tmp/warranty_download_serial.csv 0 +/usr/bin/php /var/www/resq/utils/get_warranty_serial/new_get_serials.php 05-25-2021 /tmp/warranty_download_serial.csv 0 +/usr/bin/php /var/www/resq/utils/get_warranty_serial/new_get_serials.php 05-26-2021 /tmp/warranty_download_serial.csv 0 +/usr/bin/php /var/www/resq/utils/get_warranty_serial/new_get_serials.php 05-27-2021 /tmp/warranty_download_serial.csv 0 +/usr/bin/php /var/www/resq/utils/get_warranty_serial/new_get_serials.php 05-28-2021 /tmp/warranty_download_serial.csv 0 +/usr/bin/php /var/www/resq/utils/get_warranty_serial/new_get_serials.php 05-29-2021 /tmp/warranty_download_serial.csv 0 +/usr/bin/php /var/www/resq/utils/get_warranty_serial/new_get_serials.php 05-30-2021 /tmp/warranty_download_serial.csv 0 +/usr/bin/php /var/www/resq/utils/get_warranty_serial/new_get_serials.php 05-31-2021 /tmp/warranty_download_serial.csv 0 +/usr/bin/php /var/www/resq/utils/get_warranty_serial/new_get_serials.php 06-01-2021 /tmp/warranty_download_serial.csv 0 +/usr/bin/php /var/www/resq/utils/get_warranty_serial/new_get_serials.php 06-02-2021 /tmp/warranty_download_serial.csv 0 +/usr/bin/php /var/www/resq/utils/get_warranty_serial/new_get_serials.php 06-03-2021 /tmp/warranty_download_serial.csv 0 +touch /tmp/warranty_load_status.txt +/usr/bin/php /var/www/resq/utils/load_warranty_serial/load_serials.php /tmp/warranty_download_serial.csv "mysql:host=localhost;dbname=resq;charset=UTF8" root password /tmp/warranty_load_status.txt