From 7d65b1eb774ee8ac21a49da9cd22c5ae3997b363 Mon Sep 17 00:00:00 2001 From: Korina Cordero Date: Wed, 14 Sep 2022 07:33:51 +0000 Subject: [PATCH 01/16] Add CAPI call to upload warranty serial CSV file. #704 --- config/api_acl.yaml | 5 + config/routes/capi.yaml | 6 + .../CAPI/WarrantySerialController.php | 267 ++++++++++++++++++ src/Entity/WarrantySerialLoadLog.php | 120 ++++++++ src/Entity/WarrantySerialUploadLog.php | 101 +++++++ src/Service/WarrantySerialLoadLogger.php | 37 +++ src/Service/WarrantySerialUploadLogger.php | 35 +++ 7 files changed, 571 insertions(+) create mode 100644 src/Controller/CAPI/WarrantySerialController.php create mode 100644 src/Entity/WarrantySerialLoadLog.php create mode 100644 src/Entity/WarrantySerialUploadLog.php create mode 100644 src/Service/WarrantySerialLoadLogger.php create mode 100644 src/Service/WarrantySerialUploadLogger.php diff --git a/config/api_acl.yaml b/config/api_acl.yaml index ae1a9bfb..5d5dcfce 100644 --- a/config/api_acl.yaml +++ b/config/api_acl.yaml @@ -69,3 +69,8 @@ access_keys: acls: - id: dealer.list label: List + - id: warrantyserial + label: Warranty Serial + acls: + - id: warrantyserial.upload + label: Upload diff --git a/config/routes/capi.yaml b/config/routes/capi.yaml index 472da5c2..de9bf872 100644 --- a/config/routes/capi.yaml +++ b/config/routes/capi.yaml @@ -184,3 +184,9 @@ capi_dealer_list: path: /capi/dealers controller: App\Controller\CAPI\DealerController::getAll methods: [GET] + +# warranty serial api +capi_warranty_serial_upload: + path: /capi/warranty_serial + controller: App\Controller\CAPI\WarrantySerialController::upload + methods: [POST] diff --git a/src/Controller/CAPI/WarrantySerialController.php b/src/Controller/CAPI/WarrantySerialController.php new file mode 100644 index 00000000..c5e27f33 --- /dev/null +++ b/src/Controller/CAPI/WarrantySerialController.php @@ -0,0 +1,267 @@ +acl_gen = $acl_gen; + } + + public function upload(Request $req, EntityManagerInterface $em, WarrantySerialUploadLogger $upload_logger + WarrantySerialLoadLogger $load_logger) + { + $this->denyAccessUnlessGranted('warrantyserial.upload', null, 'No access.'); + + $required_params = [ + 'serial_file', + ]; + + $user_id = $_SERVER['HTTP_X_CATA_API_KEY']; + + $res = $this->checkRequiredParams($req, $required_params, $upload_logger, $user_id); + + if (!$res) + return $res; + + // get the csv file + $csv_file = $req->files->get('serial_file'); + + $res = $this->processWarrantySerialFile($csv_file, $upload_logger, $load_logger); + if (!$res) + return $res; + } + + protected function processWarrantySerialFile($csv_file, $upload_logger, $load_logger) + { + // attempt to open file + try + { + $fh = fopen($csv_file, "r"); + } + catch (Exception $e) + { + $error = 'The file ' . $csv_file . 'could not be read.'; + $log_data = [ + 'user_id' => $user_id, + 'is_uploaded' => false, + 'error' => $error; + ]; + $this->upload_logger->logWarrantySerialUploadInfo($log_data); + return new APIResponse(false, $error); + } + + while(($row = $fgetcsv($fh)) !== false) + { + $is_valid = $this->validateRow($row[0], $load_logger, $user_id); + if ($is_valid) + { + // valid entry, we parse and insert + $serial = trim(strtoupper($row[0])); + $sku = trim(strtoupper($row[1])); + $dispatch_status = trim($row[2]); + $str_date_create = trim($row[3]); + $inventory_status = trim($row[4]); + $cat_id = trim($row[5]); + $cat_name = trim(strtoupper($row[6])); + + // we are sure that this is a valid date at this point + $created_date = $this->convertDateCreate($str_date_create); + + $meta_info = [ + 'dispatch_status' => $dispatch_status, + 'inventory_status' => $inventory_status, + 'category_id' => $cat_id, + 'category_name' => $cat_name, + ]; + + $info = json_encode($meta_info); + + // insert the data. Still need to figure out how to do this fast + + // log the successful insert + $log_data = [ + 'user_id' => $user_id, + 'is_loaded' => true, + 'serial' => $serial, + 'error' => ''; + ]; + + $this->upload_logger->logWarrantySeriaLoadInfo($log_data); + } + } + + return true; + } + + protected function validateRow($entry, $load_logger, $user_id) + { + // possible lines: + // (1) header in csv file - ignore + // SerialNumber,Sku,DispatchStatus,CreatedDate,InventoryStatus,CategoryID,CategoryName + // (2) No available data - ignore + // (3) CH2000012071,WCHD23BL-CPN00-LX,0,2020-08-11 04:05:27.090,0,4,CHAMPION MF - valid + // (4) MG2000313690,N/A,1,2021-05-14T23:47:30.6430000+08:00,0,10,GOLD - valid + // (5) Empty line - ignore + // (6) empty sku - log + + // check if the line is a header + if ($row[0] == 'SerialNumber') + { + // no need to log, just ignore and skip + return false; + } + + // check if empty line + if ($row == array(null)) + { + // no need to log, just ignore and skip + return false; + } + + // check if empty serial + if (empty($row[0])) + { + // this one, we log + $error = 'Empty serial'; + $log_data = [ + 'user_id' => $user_id, + 'is_loaded' => false, + 'serial' => '', + 'error' => $error; + ]; + $this->upload_logger->logWarrantySeriaLoadInfo($log_data); + + return false; + } + + // validate the date created + $str_date_create = trim($row[3]); + + $date_create = $this->convertDateCreate($str_date_create) + if ($date_create == null) + { + // log + $error = 'Invalid date create'; + $serial = trim($row[0]); + $log_data = [ + 'user_id' => $user_id, + 'is_loaded' => false, + 'serial' => $serial, + 'error' => $error; + ]; + + $this->upload_logger->logWarrantySeriaLoadInfo($log_data); + + return false; + } + + // valid entry + return true; + } + + protected convertDateCreate($str_date_create) + { + // since some people cannot follow simple instructions... + // check the date format on the string + // try 2021-05-15T08:35:46+08:00 format on str_date_create + $date_create = DateTime::createFromFormat('Y-m-d\TH:i:sP', $str_date_create); + + if ($date_create == false) + { + // try this format: 2021-05-15T08:47:20.3330000+08:00 + // get the date, time and timezone from str_date_create + $str_date_time = substr($str_date_create, 0, 19); + $str_timezone = substr($str_date_create, 27); + $str_datetime_tz = $str_date_time . $str_timezone; + + // create DateTime object + // sample: 2021-05-15T12:16:06+08:00 + $date_create = DateTime::createFromFormat('Y-m-d\TH:i:sP', $str_datetime_tz); + + // check if datetime object was created + // if not, someone f*cked up and we have no date create + if ($date_create == false) + { + return null; + } + } + + // if you reach this part, then date string is valid and can be converted + $created_date = $date_create->format('Y-m-d H:i:s'); + + return $created_date; + } + + protected function checkRequiredParams(Request $req, $params, $upload_logger, $user_id) + { + // check required parameters + $missing = $this->checkMissingParameters($req, $params); + if (count($missing) > 0) + { + // log the error + $miss_string = implode(', ', $missing); + $log_data = [ + 'user_id' => $user_id, + 'is_uploaded' => false, + 'error' => 'Missing parameter(s): ' . $miss_string + ]; + + $this->upload_logger->logWarrantySerialUploadInfo($log_data); + + return new APIResponse(false, 'Missing parameter(s): ' . $miss_string); + } + + return true; + } + + protected function checkMissingParameters(Request $req, $params = []) + { + $missing = []; + + // check if parameters are there + foreach ($params as $param) + { + if ($req->getMethod() == 'GET') + { + $check = $req->query->get($param); + if (empty($check)) + $missing[] = $param; + } + else if ($req->getMethod() == 'POST') + { + $check = $req->request->get($param); + if (empty($check)) + $missing[] = $param; + } + else + return $params; + } + + return $missing; + } +} diff --git a/src/Entity/WarrantySerialLoadLog.php b/src/Entity/WarrantySerialLoadLog.php new file mode 100644 index 00000000..ece58ffb --- /dev/null +++ b/src/Entity/WarrantySerialLoadLog.php @@ -0,0 +1,120 @@ +date_create = new DateTime(); + $this->api_user = ''; + $this->serial = ''; + $this->flag_loaded = false; + $this->error = null; + } + + public function getID() + { + return $this->id; + } + + public function setDateCreate(DateTime $date_create) + { + $this->date_create = $date_create; + return $this; + } + + public function getDateCreate() + { + return $this->date_create; + } + + public function setApiUser($api_user) + { + $this->api_user = $api_user; + return $this; + } + + public function getApiUser() + { + return $this->api_user; + } + + public function setSerial($serial) + { + $this->serial = $serial; + return $this; + } + + public function getSerial() + { + return $this->serial; + } + + public function setLoaded($flag_loaded = true) + { + $this->flag_loaded = $flag_loaded; + return $this; + } + + public function isLoaded() + { + return $this->flag_loaded; + } + + public function setError($error) + { + $this->error = $error; + return $this; + } + + public function getError() + { + return $this->error; + } +} diff --git a/src/Entity/WarrantySerialUploadLog.php b/src/Entity/WarrantySerialUploadLog.php new file mode 100644 index 00000000..e9e916b3 --- /dev/null +++ b/src/Entity/WarrantySerialUploadLog.php @@ -0,0 +1,101 @@ +date_create = new DateTime(); + $this->api_user = ''; + $this->flag_uploaded = false; + $this->error = null; + } + + public function getID() + { + return $this->id; + } + + public function setDateCreate(DateTime $date_create) + { + $this->date_create = $date_create; + return $this; + } + + public function getDateCreate() + { + return $this->date_create; + } + + public function setApiUser($api_user) + { + $this->api_user = $api_user; + return $this; + } + + public function getApiUser() + { + return $this->api_user; + } + + public function setUploaded($flag_uploaded = true) + { + $this->flag_uploaded = $flag_uploaded; + return $this; + } + + public function isUploaded() + { + return $this->flag_uploaded; + } + + public function setError($error) + { + $this->error = $error; + return $this; + } + + public function getError() + { + return $this->error; + } diff --git a/src/Service/WarrantySerialLoadLogger.php b/src/Service/WarrantySerialLoadLogger.php new file mode 100644 index 00000000..c20486a3 --- /dev/null +++ b/src/Service/WarrantySerialLoadLogger.php @@ -0,0 +1,37 @@ +em = $em; + } + + public function logWarrantySerialLoadInfo($log_data) + { + $log_entry = new WarrantySerialLoadLog(); + + $user_id = $log_data['user_id']; + $serial = $log_data['serial']; + $is_loaded = $log_data['is_loaded']; + $error = $log_data['error']; + + $log_entry->setApiUser($user_id) + ->setSerial($serial) + ->setLoaded($is_loaded) + ->setError($error); + + $this->em->persist($log_entry); + $this->em->flush(); + + } + +} diff --git a/src/Service/WarrantySerialUploadLogger.php b/src/Service/WarrantySerialUploadLogger.php new file mode 100644 index 00000000..4172e78d --- /dev/null +++ b/src/Service/WarrantySerialUploadLogger.php @@ -0,0 +1,35 @@ +em = $em; + } + + public function logWarrantySerialUploadInfo($log_data) + { + $log_entry = new WarrantySerialUploadLog(); + + $user_id = $log_data['user_id']; + $is_uploaded = $log_data['is_uploaded']; + $error = $log_data['error']; + + $log_entry->setApiUser($user_id) + ->setLoaded($is_uploaded) + ->setError($error); + + $this->em->persist($log_entry); + $this->em->flush(); + + } + +} From 3c9bdc8152cc093f3c84b5792fa086d2f94ada64 Mon Sep 17 00:00:00 2001 From: Korina Cordero Date: Wed, 14 Sep 2022 08:02:51 +0000 Subject: [PATCH 02/16] Add WarrantySerialQueue entity. #704 --- config/services.yaml | 10 +++ .../CAPI/WarrantySerialController.php | 16 ++-- src/Entity/WarrantySerialQueue.php | 75 +++++++++++++++++++ src/Entity/WarrantySerialUploadLog.php | 1 + 4 files changed, 95 insertions(+), 7 deletions(-) create mode 100644 src/Entity/WarrantySerialQueue.php diff --git a/config/services.yaml b/config/services.yaml index 9d1ee14e..72817508 100644 --- a/config/services.yaml +++ b/config/services.yaml @@ -306,3 +306,13 @@ services: App\Service\WarrantyBulkUploader: arguments: $em: "@doctrine.orm.entity_manager" + + # warranty serial file logger + App\Service\WarrantySerialUploadLogger: + arguments: + $em: "@doctrine.orm.entity_manager" + + # warranty serial load logger + App\Service\WarrantySerialLoadLogger: + arguments: + $em: "@doctrine.orm.entity_manager" diff --git a/src/Controller/CAPI/WarrantySerialController.php b/src/Controller/CAPI/WarrantySerialController.php index c5e27f33..61f030d7 100644 --- a/src/Controller/CAPI/WarrantySerialController.php +++ b/src/Controller/CAPI/WarrantySerialController.php @@ -32,7 +32,7 @@ class WarrantySerialController extends APIController $this->acl_gen = $acl_gen; } - public function upload(Request $req, EntityManagerInterface $em, WarrantySerialUploadLogger $upload_logger + public function upload(Request $req, EntityManagerInterface $em, WarrantySerialUploadLogger $upload_logger, WarrantySerialLoadLogger $load_logger) { $this->denyAccessUnlessGranted('warrantyserial.upload', null, 'No access.'); @@ -69,7 +69,7 @@ class WarrantySerialController extends APIController $log_data = [ 'user_id' => $user_id, 'is_uploaded' => false, - 'error' => $error; + 'error' => $error, ]; $this->upload_logger->logWarrantySerialUploadInfo($log_data); return new APIResponse(false, $error); @@ -108,7 +108,7 @@ class WarrantySerialController extends APIController 'user_id' => $user_id, 'is_loaded' => true, 'serial' => $serial, - 'error' => ''; + 'error' => '', ]; $this->upload_logger->logWarrantySeriaLoadInfo($log_data); @@ -152,7 +152,7 @@ class WarrantySerialController extends APIController 'user_id' => $user_id, 'is_loaded' => false, 'serial' => '', - 'error' => $error; + 'error' => $error, ]; $this->upload_logger->logWarrantySeriaLoadInfo($log_data); @@ -162,7 +162,7 @@ class WarrantySerialController extends APIController // validate the date created $str_date_create = trim($row[3]); - $date_create = $this->convertDateCreate($str_date_create) + $date_create = $this->convertDateCreate($str_date_create); if ($date_create == null) { // log @@ -172,7 +172,7 @@ class WarrantySerialController extends APIController 'user_id' => $user_id, 'is_loaded' => false, 'serial' => $serial, - 'error' => $error; + 'error' => $error, ]; $this->upload_logger->logWarrantySeriaLoadInfo($log_data); @@ -180,11 +180,13 @@ class WarrantySerialController extends APIController return false; } + // check if serial is a dupe + // valid entry return true; } - protected convertDateCreate($str_date_create) + protected function convertDateCreate($str_date_create) { // since some people cannot follow simple instructions... // check the date format on the string diff --git a/src/Entity/WarrantySerialQueue.php b/src/Entity/WarrantySerialQueue.php new file mode 100644 index 00000000..cc67f46d --- /dev/null +++ b/src/Entity/WarrantySerialQueue.php @@ -0,0 +1,75 @@ +date_create = new DateTime(); + $this->file_serial = null; + $this->status = ''; + } + + public function getID() + { + return $this->id; + } + + public function setFileSerial($file = null) + { + $this->file_serial = $file; + return $this; + } + + public function getFileSerial() + { + return $this->file_serial; + } + + public function setStatus($status) + { + $this->status = $status; + return $this; + } + + public function getStatus() + { + return $this->status; + } +} diff --git a/src/Entity/WarrantySerialUploadLog.php b/src/Entity/WarrantySerialUploadLog.php index e9e916b3..77792bfe 100644 --- a/src/Entity/WarrantySerialUploadLog.php +++ b/src/Entity/WarrantySerialUploadLog.php @@ -99,3 +99,4 @@ class WarrantySerialUploadLog { return $this->error; } +} From a3e9c45ec3ed0ad2119c0049589b4ec49b7c3b66 Mon Sep 17 00:00:00 2001 From: Korina Cordero Date: Wed, 14 Sep 2022 09:33:05 +0000 Subject: [PATCH 03/16] Separate the uploading of files and the processing of files. #704 --- src/Command/LoadWarrantySerialCommand.php | 205 +++++++++++++++++ .../CAPI/WarrantySerialController.php | 207 +++++------------- src/Entity/WarrantySerialUploadLog.php | 38 +++- src/Service/WarrantySerialUploadLogger.php | 14 +- 4 files changed, 306 insertions(+), 158 deletions(-) create mode 100644 src/Command/LoadWarrantySerialCommand.php diff --git a/src/Command/LoadWarrantySerialCommand.php b/src/Command/LoadWarrantySerialCommand.php new file mode 100644 index 00000000..7faf88d8 --- /dev/null +++ b/src/Command/LoadWarrantySerialCommand.php @@ -0,0 +1,205 @@ +em = $em; + + parent::__construct(); + } + + protected function configure() + { + $this->setName('warrantyserial:load') + ->setDescription('Load warranty serials from file.') + ->setHelp('Load warranty serials from file.'); + } + + protected function execute(InputInterface $input, OutputInterface $output) + { + return 0; + } + + protected function processWarrantySerialFile($csv_file, $upload_logger, $load_logger) + { + // attempt to open file + try + { + $fh = fopen($csv_file, "r"); + } + catch (Exception $e) + { + $error = 'The file ' . $csv_file . 'could not be read.'; + $log_data = [ + 'user_id' => $user_id, + 'is_uploaded' => false, + 'error' => $error, + ]; + $this->upload_logger->logWarrantySerialUploadInfo($log_data); + return new APIResponse(false, $error); + } + + while(($row = $fgetcsv($fh)) !== false) + { + $is_valid = $this->validateRow($row[0], $load_logger, $user_id); + if ($is_valid) + { + // valid entry, we parse and insert + $serial = trim(strtoupper($row[0])); + $sku = trim(strtoupper($row[1])); + $dispatch_status = trim($row[2]); + $str_date_create = trim($row[3]); + $inventory_status = trim($row[4]); + $cat_id = trim($row[5]); + $cat_name = trim(strtoupper($row[6])); + + // we are sure that this is a valid date at this point + $created_date = $this->convertDateCreate($str_date_create); + + $meta_info = [ + 'dispatch_status' => $dispatch_status, + 'inventory_status' => $inventory_status, + 'category_id' => $cat_id, + 'category_name' => $cat_name, + ]; + + $info = json_encode($meta_info); + + // insert the data. Still need to figure out how to do this fast + + // log the successful insert + $log_data = [ + 'user_id' => $user_id, + 'is_loaded' => true, + 'serial' => $serial, + 'error' => '', + ]; + + $this->upload_logger->logWarrantySeriaLoadInfo($log_data); + } + } + + return true; + } + + protected function validateRow($entry, $load_logger, $user_id) + { + // possible lines: + // (1) header in csv file - ignore + // SerialNumber,Sku,DispatchStatus,CreatedDate,InventoryStatus,CategoryID,CategoryName + // (2) No available data - ignore + // (3) CH2000012071,WCHD23BL-CPN00-LX,0,2020-08-11 04:05:27.090,0,4,CHAMPION MF - valid + // (4) MG2000313690,N/A,1,2021-05-14T23:47:30.6430000+08:00,0,10,GOLD - valid + // (5) Empty line - ignore + // (6) empty sku - log + + // check if the line is a header + if ($row[0] == 'SerialNumber') + { + // no need to log, just ignore and skip + return false; + } + + // check if empty line + if ($row == array(null)) + { + // no need to log, just ignore and skip + return false; + } + + // check if empty serial + if (empty($row[0])) + { + // this one, we log + $error = 'Empty serial'; + $log_data = [ + 'user_id' => $user_id, + 'is_loaded' => false, + 'serial' => '', + 'error' => $error, + ]; + $this->upload_logger->logWarrantySeriaLoadInfo($log_data); + + return false; + } + + // validate the date created + $str_date_create = trim($row[3]); + + $date_create = $this->convertDateCreate($str_date_create); + if ($date_create == null) + { + // log + $error = 'Invalid date create'; + $serial = trim($row[0]); + $log_data = [ + 'user_id' => $user_id, + 'is_loaded' => false, + 'serial' => $serial, + 'error' => $error, + ]; + + $this->upload_logger->logWarrantySeriaLoadInfo($log_data); + + return false; + } + + // check if serial is a dupe + + // valid entry + return true; + } + + protected function convertDateCreate($str_date_create) + { + // since some people cannot follow simple instructions... + // check the date format on the string + // try 2021-05-15T08:35:46+08:00 format on str_date_create + $date_create = DateTime::createFromFormat('Y-m-d\TH:i:sP', $str_date_create); + + if ($date_create == false) + { + // try this format: 2021-05-15T08:47:20.3330000+08:00 + // get the date, time and timezone from str_date_create + $str_date_time = substr($str_date_create, 0, 19); + $str_timezone = substr($str_date_create, 27); + $str_datetime_tz = $str_date_time . $str_timezone; + + // create DateTime object + // sample: 2021-05-15T12:16:06+08:00 + $date_create = DateTime::createFromFormat('Y-m-d\TH:i:sP', $str_datetime_tz); + + // check if datetime object was created + // if not, someone f*cked up and we have no date create + if ($date_create == false) + { + return null; + } + } + + // TODO: think about how we're going to save this, if we need string or DateTime + // if you reach this part, then date string is valid and can be converted + $created_date = $date_create->format('Y-m-d H:i:s'); + + return $created_date; + } +} diff --git a/src/Controller/CAPI/WarrantySerialController.php b/src/Controller/CAPI/WarrantySerialController.php index 61f030d7..e1226c1f 100644 --- a/src/Controller/CAPI/WarrantySerialController.php +++ b/src/Controller/CAPI/WarrantySerialController.php @@ -13,12 +13,11 @@ use Doctrine\DBAL\Exception\UniqueConstraintViolationException; use Catalyst\APIBundle\Controller\APIController; use Catalyst\APIBundle\Response\APIResponse; -use App\Entity\WarrantySerial; -use App\Entity\WarrantySerialUploadLog; -use App\Entity\WarrantySerialLoadLog; +use App\Entity\WarrantySerialiQueue; use App\Service\WarrantySerialUploadLogger; -use App\Service\WarrantySerialLoadLogger; + +use DateTime; use Catalyst\APIBundle\Access\Generator as ACLGenerator; @@ -32,8 +31,7 @@ class WarrantySerialController extends APIController $this->acl_gen = $acl_gen; } - public function upload(Request $req, EntityManagerInterface $em, WarrantySerialUploadLogger $upload_logger, - WarrantySerialLoadLogger $load_logger) + public function upload(Request $req, EntityManagerInterface $em, WarrantySerialUploadLogger $upload_logger) { $this->denyAccessUnlessGranted('warrantyserial.upload', null, 'No access.'); @@ -51,172 +49,69 @@ class WarrantySerialController extends APIController // get the csv file $csv_file = $req->files->get('serial_file'); - $res = $this->processWarrantySerialFile($csv_file, $upload_logger, $load_logger); - if (!$res) - return $res; + // process file upload + $upload_dir = $kernel->getProjectDir() . '/public/warranty_serial_uploads'; + $serial_filename = $this->handleSerialFileUpload($csv_file, $upload_dir, $upload_logger); + + // insert to warranty serial queue + $res = $this->registerWarrantySerialFile($em, $csv_file, $serial_filename, $upload_logger, $user_id); + + // flush to db + $em->flush(); + + return $res; } - protected function processWarrantySerialFile($csv_file, $upload_logger, $load_logger) + protected function registerWarrantySerialFile($em, $file, $serial_filename, $upload_logger, $user_id) { - // attempt to open file - try - { - $fh = fopen($csv_file, "r"); - } - catch (Exception $e) - { - $error = 'The file ' . $csv_file . 'could not be read.'; - $log_data = [ - 'user_id' => $user_id, - 'is_uploaded' => false, - 'error' => $error, - ]; - $this->upload_logger->logWarrantySerialUploadInfo($log_data); - return new APIResponse(false, $error); - } + $orig_filename = pathinfo($file->getClientOriginalName(), PATHINFO_FILENAME); - while(($row = $fgetcsv($fh)) !== false) + $ws_file = new WarrantySerialQueue(); + + $ws_file->setFileSerial($serial_filename) + ->setStatus('pending'); + + $em->persist($ws_file); + + // log upload + $log_data = [ + 'user_id' => $user_id, + 'is_uploaded' => true, + 'orig_file_serial' => $orig_filename, + 'uploaded_file_serial' => $serial_filename, + ]; + + $this->upload_logger->logWarrantySerialUploadInfo($log_data); + } + + protected function handleSerialFileUpload($file, $upload_dir, $upload_logger) + { + // create target dir if it doesn't exist + if (!file_exists($target_dir)) { - $is_valid = $this->validateRow($row[0], $load_logger, $user_id); - if ($is_valid) + if (!mkdir($target_dir, 0744, true)) { - // valid entry, we parse and insert - $serial = trim(strtoupper($row[0])); - $sku = trim(strtoupper($row[1])); - $dispatch_status = trim($row[2]); - $str_date_create = trim($row[3]); - $inventory_status = trim($row[4]); - $cat_id = trim($row[5]); - $cat_name = trim(strtoupper($row[6])); - - // we are sure that this is a valid date at this point - $created_date = $this->convertDateCreate($str_date_create); - - $meta_info = [ - 'dispatch_status' => $dispatch_status, - 'inventory_status' => $inventory_status, - 'category_id' => $cat_id, - 'category_name' => $cat_name, - ]; - - $info = json_encode($meta_info); - - // insert the data. Still need to figure out how to do this fast - - // log the successful insert $log_data = [ 'user_id' => $user_id, - 'is_loaded' => true, - 'serial' => $serial, - 'error' => '', + 'is_uploaded' => false, + 'error' => 'Failed to create folder for warranty serial files.' ]; - $this->upload_logger->logWarrantySeriaLoadInfo($log_data); - } - } + $this->upload_logger->logWarrantySerialUploadInfo($log_data); - return true; - } - - protected function validateRow($entry, $load_logger, $user_id) - { - // possible lines: - // (1) header in csv file - ignore - // SerialNumber,Sku,DispatchStatus,CreatedDate,InventoryStatus,CategoryID,CategoryName - // (2) No available data - ignore - // (3) CH2000012071,WCHD23BL-CPN00-LX,0,2020-08-11 04:05:27.090,0,4,CHAMPION MF - valid - // (4) MG2000313690,N/A,1,2021-05-14T23:47:30.6430000+08:00,0,10,GOLD - valid - // (5) Empty line - ignore - // (6) empty sku - log - - // check if the line is a header - if ($row[0] == 'SerialNumber') - { - // no need to log, just ignore and skip - return false; - } - - // check if empty line - if ($row == array(null)) - { - // no need to log, just ignore and skip - return false; - } - - // check if empty serial - if (empty($row[0])) - { - // this one, we log - $error = 'Empty serial'; - $log_data = [ - 'user_id' => $user_id, - 'is_loaded' => false, - 'serial' => '', - 'error' => $error, - ]; - $this->upload_logger->logWarrantySeriaLoadInfo($log_data); - - return false; - } - - // validate the date created - $str_date_create = trim($row[3]); - - $date_create = $this->convertDateCreate($str_date_create); - if ($date_create == null) - { - // log - $error = 'Invalid date create'; - $serial = trim($row[0]); - $log_data = [ - 'user_id' => $user_id, - 'is_loaded' => false, - 'serial' => $serial, - 'error' => $error, - ]; - - $this->upload_logger->logWarrantySeriaLoadInfo($log_data); - - return false; - } - - // check if serial is a dupe - - // valid entry - return true; - } - - protected function convertDateCreate($str_date_create) - { - // since some people cannot follow simple instructions... - // check the date format on the string - // try 2021-05-15T08:35:46+08:00 format on str_date_create - $date_create = DateTime::createFromFormat('Y-m-d\TH:i:sP', $str_date_create); - - if ($date_create == false) - { - // try this format: 2021-05-15T08:47:20.3330000+08:00 - // get the date, time and timezone from str_date_create - $str_date_time = substr($str_date_create, 0, 19); - $str_timezone = substr($str_date_create, 27); - $str_datetime_tz = $str_date_time . $str_timezone; - - // create DateTime object - // sample: 2021-05-15T12:16:06+08:00 - $date_create = DateTime::createFromFormat('Y-m-d\TH:i:sP', $str_datetime_tz); - - // check if datetime object was created - // if not, someone f*cked up and we have no date create - if ($date_create == false) - { return null; } } - // if you reach this part, then date string is valid and can be converted - $created_date = $date_create->format('Y-m-d H:i:s'); + // get current date + $curr_date = new DateTime(); + $str_curr_date = $curr_date->format('YmdHis'); - return $created_date; + // move file + $filename = 'warranty_serial' . '.' . $file->getClientOriginalExtension(); + $file->move($target_dir . '/' . $str_curr_date, $filename); + + return $str_curr_date . '/' . $filename; } protected function checkRequiredParams(Request $req, $params, $upload_logger, $user_id) diff --git a/src/Entity/WarrantySerialUploadLog.php b/src/Entity/WarrantySerialUploadLog.php index 77792bfe..44f5b16a 100644 --- a/src/Entity/WarrantySerialUploadLog.php +++ b/src/Entity/WarrantySerialUploadLog.php @@ -32,6 +32,18 @@ class WarrantySerialUploadLog */ protected $api_user; + // original filename of warranty serial + /** + * @ORM\Column(type="string", length=80, nullable=true) + */ + protected $orig_file_serial; + + // uploaded file name + /** + * @ORM\Column(type="string", length=80, nullable=true) + */ + protected $uploaded_file_serial; + // flag if uploaded /** * @ORM\Column(type="boolean") @@ -39,7 +51,7 @@ class WarrantySerialUploadLog protected $flag_uploaded; /** - * @ORM\Column(type="string", length=30, nullable=true) + * @ORM\Column(type="string", length=60, nullable=true) */ protected $error; @@ -47,6 +59,8 @@ class WarrantySerialUploadLog { $this->date_create = new DateTime(); $this->api_user = ''; + $this->orig_file_serial = ''; + $this->orig_file_serial = ''; $this->flag_uploaded = false; $this->error = null; } @@ -78,6 +92,28 @@ class WarrantySerialUploadLog return $this->api_user; } + public function setOrigFileSerial($file = null) + { + $this->orig_file_serial = $file; + return $this; + } + + public function getOrigFileSerial() + { + return $this->orig_file_serial; + } + + public function setUploadedFileSerial($file = null) + { + $this->uploaded_file_serial = $file; + return $this; + } + + public function getUploadedFileSerial() + { + return $this->uploaded_file_serial; + } + public function setUploaded($flag_uploaded = true) { $this->flag_uploaded = $flag_uploaded; diff --git a/src/Service/WarrantySerialUploadLogger.php b/src/Service/WarrantySerialUploadLogger.php index 4172e78d..d4c24852 100644 --- a/src/Service/WarrantySerialUploadLogger.php +++ b/src/Service/WarrantySerialUploadLogger.php @@ -21,10 +21,22 @@ class WarrantySerialUploadLogger $user_id = $log_data['user_id']; $is_uploaded = $log_data['is_uploaded']; - $error = $log_data['error']; + + $error = ''; + $orig_file_serial = ''; + $uploaded_file_serial = ''; + + if (isset($log_data['error'])) + $error = $log_data['error']; + if (isset($log_data['orig_file_serial'])) + $orig_file_serial = $log_data['orig_file_serial']; + if (isset($log_data['uploaded_file_serial'])) + $uploaded_file_serial = $log_data['uploaded_file_serial']; $log_entry->setApiUser($user_id) ->setLoaded($is_uploaded) + ->setOrigFileSerial($orig_file_serial) + ->setUploadedFileSerial($uploaded_file_serial) ->setError($error); $this->em->persist($log_entry); From 5b848a7f10f5e7e9d2aeb9a12f991b66352eb876 Mon Sep 17 00:00:00 2001 From: Korina Cordero Date: Thu, 15 Sep 2022 05:56:43 +0000 Subject: [PATCH 04/16] Fix issue when creating API user that's not a rider. Fix issues when uploading a file. #704 --- config/routes/capi.yaml | 4 +-- src/Controller/APIUserController.php | 14 ++++---- .../CAPI/WarrantySerialController.php | 33 +++++++++++-------- src/Service/WarrantySerialUploadLogger.php | 2 +- 4 files changed, 31 insertions(+), 22 deletions(-) diff --git a/config/routes/capi.yaml b/config/routes/capi.yaml index de9bf872..0ae29c7c 100644 --- a/config/routes/capi.yaml +++ b/config/routes/capi.yaml @@ -187,6 +187,6 @@ capi_dealer_list: # warranty serial api capi_warranty_serial_upload: - path: /capi/warranty_serial - controller: App\Controller\CAPI\WarrantySerialController::upload + path: /capi/warranty_serial/upload + controller: App\Controller\CAPI\WarrantySerialController::uploadWarrantySerialFile methods: [POST] diff --git a/src/Controller/APIUserController.php b/src/Controller/APIUserController.php index 06f128e9..122d3830 100644 --- a/src/Controller/APIUserController.php +++ b/src/Controller/APIUserController.php @@ -155,18 +155,20 @@ class APIUserController extends Controller // metadata $rider_id = $req->request->get('rider_id'); $rider = $em->getRepository(Rider::class)->find($rider_id); - // TODO: check for null rider + if ($rider != null) + { + $meta = ['rider_id' => $rider_id]; - $meta = ['rider_id' => $rider_id]; + // set api user in rider + $rider->setAPIUser($obj); - // set api user in rider - $rider->setAPIUser($obj); + $obj->setRider($rider) + ->setMetadata($meta); + } // set and save values $obj->setName($req->request->get('name')) ->setEnabled($req->request->get('enabled') ? true : false) - ->setMetadata($meta) - ->setRider($rider) ->clearRoles(); // set roles diff --git a/src/Controller/CAPI/WarrantySerialController.php b/src/Controller/CAPI/WarrantySerialController.php index e1226c1f..ecd706bf 100644 --- a/src/Controller/CAPI/WarrantySerialController.php +++ b/src/Controller/CAPI/WarrantySerialController.php @@ -13,7 +13,7 @@ use Doctrine\DBAL\Exception\UniqueConstraintViolationException; use Catalyst\APIBundle\Controller\APIController; use Catalyst\APIBundle\Response\APIResponse; -use App\Entity\WarrantySerialiQueue; +use App\Entity\WarrantySerialQueue; use App\Service\WarrantySerialUploadLogger; @@ -25,13 +25,15 @@ use Catalyst\APIBundle\Access\Generator as ACLGenerator; class WarrantySerialController extends APIController { protected $acl_gen; + protected $upload_logger; - public function __construct(ACLGenerator $acl_gen) + public function __construct(ACLGenerator $acl_gen, WarrantySerialUploadLogger $upload_logger) { $this->acl_gen = $acl_gen; + $this->upload_logger = $upload_logger; } - public function upload(Request $req, EntityManagerInterface $em, WarrantySerialUploadLogger $upload_logger) + public function uploadWarrantySerialFile(Request $req, EntityManagerInterface $em, KernelInterface $kernel) { $this->denyAccessUnlessGranted('warrantyserial.upload', null, 'No access.'); @@ -41,9 +43,9 @@ class WarrantySerialController extends APIController $user_id = $_SERVER['HTTP_X_CATA_API_KEY']; - $res = $this->checkRequiredParams($req, $required_params, $upload_logger, $user_id); + $res = $this->checkRequiredParamsForFiles($req, $required_params, $user_id); - if (!$res) + if ($res !== true) return $res; // get the csv file @@ -51,10 +53,10 @@ class WarrantySerialController extends APIController // process file upload $upload_dir = $kernel->getProjectDir() . '/public/warranty_serial_uploads'; - $serial_filename = $this->handleSerialFileUpload($csv_file, $upload_dir, $upload_logger); + $serial_filename = $this->handleSerialFileUpload($csv_file, $upload_dir); // insert to warranty serial queue - $res = $this->registerWarrantySerialFile($em, $csv_file, $serial_filename, $upload_logger, $user_id); + $res = $this->registerWarrantySerialFile($em, $csv_file, $serial_filename, $user_id); // flush to db $em->flush(); @@ -62,7 +64,7 @@ class WarrantySerialController extends APIController return $res; } - protected function registerWarrantySerialFile($em, $file, $serial_filename, $upload_logger, $user_id) + protected function registerWarrantySerialFile($em, $file, $serial_filename, $user_id) { $orig_filename = pathinfo($file->getClientOriginalName(), PATHINFO_FILENAME); @@ -82,9 +84,11 @@ class WarrantySerialController extends APIController ]; $this->upload_logger->logWarrantySerialUploadInfo($log_data); + + return new APIResponse(true, 'Warranty serial file uploaded.'); } - protected function handleSerialFileUpload($file, $upload_dir, $upload_logger) + protected function handleSerialFileUpload($file, $target_dir) { // create target dir if it doesn't exist if (!file_exists($target_dir)) @@ -114,10 +118,10 @@ class WarrantySerialController extends APIController return $str_curr_date . '/' . $filename; } - protected function checkRequiredParams(Request $req, $params, $upload_logger, $user_id) + protected function checkRequiredParamsForFiles(Request $req, $params, $user_id) { // check required parameters - $missing = $this->checkMissingParameters($req, $params); + $missing = $this->checkMissingParametersForFiles($req, $params); if (count($missing) > 0) { // log the error @@ -136,7 +140,7 @@ class WarrantySerialController extends APIController return true; } - protected function checkMissingParameters(Request $req, $params = []) + protected function checkMissingParametersForFiles(Request $req, $params = []) { $missing = []; @@ -151,9 +155,12 @@ class WarrantySerialController extends APIController } else if ($req->getMethod() == 'POST') { - $check = $req->request->get($param); + // get files from request. + $check = $req->files->get($param); if (empty($check)) + { $missing[] = $param; + } } else return $params; diff --git a/src/Service/WarrantySerialUploadLogger.php b/src/Service/WarrantySerialUploadLogger.php index d4c24852..d708792c 100644 --- a/src/Service/WarrantySerialUploadLogger.php +++ b/src/Service/WarrantySerialUploadLogger.php @@ -34,7 +34,7 @@ class WarrantySerialUploadLogger $uploaded_file_serial = $log_data['uploaded_file_serial']; $log_entry->setApiUser($user_id) - ->setLoaded($is_uploaded) + ->setUploaded($is_uploaded) ->setOrigFileSerial($orig_file_serial) ->setUploadedFileSerial($uploaded_file_serial) ->setError($error); From 32720ba9e7f4edef2e750825722b4ae37d7d29c8 Mon Sep 17 00:00:00 2001 From: Korina Cordero Date: Thu, 15 Sep 2022 07:38:33 +0000 Subject: [PATCH 05/16] Fix issues for the load command. #704 --- src/Command/LoadWarrantySerialCommand.php | 131 +++++++++++++----- .../CAPI/WarrantySerialController.php | 2 +- src/Entity/WarrantySerialQueue.php | 18 +++ src/Service/WarrantySerialLoadLogger.php | 2 +- 4 files changed, 118 insertions(+), 35 deletions(-) diff --git a/src/Command/LoadWarrantySerialCommand.php b/src/Command/LoadWarrantySerialCommand.php index 7faf88d8..18c1ec1b 100644 --- a/src/Command/LoadWarrantySerialCommand.php +++ b/src/Command/LoadWarrantySerialCommand.php @@ -6,6 +6,7 @@ use Symfony\Component\Console\Command\Command; use Symfony\Component\Console\Input\InputArgument; use Symfony\Component\Console\Input\InputInterface; use Symfony\Component\Console\Output\OutputInterface; +use Symfony\Component\HttpKernel\KernelInterface; use Doctrine\ORM\EntityManagerInterface; @@ -16,13 +17,23 @@ use App\Entity\WarrantySerialLoadLog; use App\Service\WarrantySerialUploadLogger; use App\Service\WarrantySerialLoadLogger; +use PDO; +use DateTime; + class LoadWarrantySerialCommand extends Command { protected $em; + protected $upload_logger; + protected $load_logger; + protected $project_dir; - public function __construct(EntityManagerInterface $em) + public function __construct(EntityManagerInterface $em, WarrantySerialUploadLogger $upload_logger, + WarrantySerialLoadLogger $load_logger, KernelInterface $kernel) { $this->em = $em; + $this->upload_logger = $upload_logger; + $this->load_logger = $load_logger; + $this->project_dir = $kernel->getProjectDir(); parent::__construct(); } @@ -36,10 +47,35 @@ class LoadWarrantySerialCommand extends Command protected function execute(InputInterface $input, OutputInterface $output) { + $em = $this->em; + + $status = 'pending'; + + // get the filenames from the queue table with status pending + $db = $em->getConnection(); + + $ws_query_sql = 'SELECT file_serial, api_user FROM warranty_serial_queue + WHERE status = :status'; + + $ws_query_stmt = $db->prepare($ws_query_sql); + $ws_query_stmt->bindValue('status', $status); + + $ws_results = $ws_query_stmt->executeQuery(); + + while ($row = $ws_results->fetchAssociative()) + { + $filename = $row['file_serial']; + $user_id = $row['api_user']; + + $fname = $this->project_dir . '/public/warranty_serial_uploads/' . $filename; + + $this->processWarrantySerialFile($fname, $user_id); + } + return 0; } - protected function processWarrantySerialFile($csv_file, $upload_logger, $load_logger) + protected function processWarrantySerialFile($csv_file, $user_id) { // attempt to open file try @@ -55,12 +91,13 @@ class LoadWarrantySerialCommand extends Command 'error' => $error, ]; $this->upload_logger->logWarrantySerialUploadInfo($log_data); - return new APIResponse(false, $error); + + error_log($error); } - while(($row = $fgetcsv($fh)) !== false) + while(($row = fgetcsv($fh)) !== false) { - $is_valid = $this->validateRow($row[0], $load_logger, $user_id); + $is_valid = $this->validateRow($row, $user_id); if ($is_valid) { // valid entry, we parse and insert @@ -84,24 +121,43 @@ class LoadWarrantySerialCommand extends Command $info = json_encode($meta_info); - // insert the data. Still need to figure out how to do this fast + // prepare the data + $source = 'motiv'; + if ($sku == 'N/A') + $sku = null; - // log the successful insert - $log_data = [ - 'user_id' => $user_id, - 'is_loaded' => true, - 'serial' => $serial, - 'error' => '', - ]; + // prepared statement + $db = $this->em->getConnection(); + $insert_stmt = $db->prepare('INSERT INTO warranty_serial (id, sku, date_create, source, meta_info) + VALUES (:serial, :sku, :date_create, :source, :meta_info)'); - $this->upload_logger->logWarrantySeriaLoadInfo($log_data); + $res = $insert_stmt->execute([ + ':serial' => $serial, + ':sku' => $sku, + ':date_create' => $created_date, + ':source' => $source, + ':meta_info' => $info, + ]); + + if (!$res) + { + // log the not successful insert + $err = $insert_stmt->errorInfo(); + $error = $err[2]; + $this->logLoadInfo($user_id, false, $serial, $error); + } + else + { + // log the successful insert + $this->logLoadInfo($user_id, true, $serial, ''); + } } } return true; } - protected function validateRow($entry, $load_logger, $user_id) + protected function validateRow($row, $user_id) { // possible lines: // (1) header in csv file - ignore @@ -131,18 +187,13 @@ class LoadWarrantySerialCommand extends Command { // this one, we log $error = 'Empty serial'; - $log_data = [ - 'user_id' => $user_id, - 'is_loaded' => false, - 'serial' => '', - 'error' => $error, - ]; - $this->upload_logger->logWarrantySeriaLoadInfo($log_data); + $this->logLoadInfo($user_id, false, '', $error); return false; } // validate the date created + $serial = trim($row[0]); $str_date_create = trim($row[3]); $date_create = $this->convertDateCreate($str_date_create); @@ -150,20 +201,21 @@ class LoadWarrantySerialCommand extends Command { // log $error = 'Invalid date create'; - $serial = trim($row[0]); - $log_data = [ - 'user_id' => $user_id, - 'is_loaded' => false, - 'serial' => $serial, - 'error' => $error, - ]; - - $this->upload_logger->logWarrantySeriaLoadInfo($log_data); + $this->logLoadInfo($user_id, false, $serial, $error); return false; } // check if serial is a dupe + $existing_serial = $this->em->getRepository(WarrantySerial::class)->find($serial); + if ($existing_serial != null) + { + // log + $error = 'Duplicate serial'; + $this->logLoadInfo($user_id, false, $serial, $error); + + return false; + } // valid entry return true; @@ -196,10 +248,23 @@ class LoadWarrantySerialCommand extends Command } } - // TODO: think about how we're going to save this, if we need string or DateTime - // if you reach this part, then date string is valid and can be converted + // if you reach this part, then date string is valid. Since we'll be using + // sql to insert the entries, we return the string $created_date = $date_create->format('Y-m-d H:i:s'); + // $created_date = DateTime::createFromFormat('Y-m-d H:i:s', $str_created_date); return $created_date; } + + protected function logLoadInfo($user_id, $is_loaded, $serial, $error) + { + $log_data = [ + 'user_id' => $user_id, + 'is_loaded' => $is_loaded, + 'serial' => $serial, + 'error' => $error, + ]; + + $this->load_logger->logWarrantySerialLoadInfo($log_data); + } } diff --git a/src/Controller/CAPI/WarrantySerialController.php b/src/Controller/CAPI/WarrantySerialController.php index ecd706bf..db447dd5 100644 --- a/src/Controller/CAPI/WarrantySerialController.php +++ b/src/Controller/CAPI/WarrantySerialController.php @@ -66,7 +66,7 @@ class WarrantySerialController extends APIController protected function registerWarrantySerialFile($em, $file, $serial_filename, $user_id) { - $orig_filename = pathinfo($file->getClientOriginalName(), PATHINFO_FILENAME); + $orig_filename = pathinfo($file->getClientOriginalName(), PATHINFO_FILENAME) . '.' . $file->getClientOriginalExtension(); $ws_file = new WarrantySerialQueue(); diff --git a/src/Entity/WarrantySerialQueue.php b/src/Entity/WarrantySerialQueue.php index cc67f46d..585be163 100644 --- a/src/Entity/WarrantySerialQueue.php +++ b/src/Entity/WarrantySerialQueue.php @@ -39,11 +39,18 @@ class WarrantySerialQueue */ protected $status; + // user that uploaded the serials + /** + * @ORM\Column(type="string", length=32) + */ + protected $api_user; + public function __construct() { $this->date_create = new DateTime(); $this->file_serial = null; $this->status = ''; + $this->api_user = ''; } public function getID() @@ -72,4 +79,15 @@ class WarrantySerialQueue { return $this->status; } + + public function setApiUser($api_user) + { + $this->api_user = $api_user; + return $this; + } + + public function getApiUser() + { + return $this->api_user; + } } diff --git a/src/Service/WarrantySerialLoadLogger.php b/src/Service/WarrantySerialLoadLogger.php index c20486a3..336b1dcb 100644 --- a/src/Service/WarrantySerialLoadLogger.php +++ b/src/Service/WarrantySerialLoadLogger.php @@ -20,9 +20,9 @@ class WarrantySerialLoadLogger $log_entry = new WarrantySerialLoadLog(); $user_id = $log_data['user_id']; - $serial = $log_data['serial']; $is_loaded = $log_data['is_loaded']; $error = $log_data['error']; + $serial = $log_data['serial']; $log_entry->setApiUser($user_id) ->setSerial($serial) From cbc0b290adea4118fb93eccd708282de9c5e2395 Mon Sep 17 00:00:00 2001 From: Korina Cordero Date: Thu, 15 Sep 2022 09:51:07 +0000 Subject: [PATCH 06/16] Add processing for output. #704 --- src/Command/LoadWarrantySerialCommand.php | 162 ++++++++++++++-------- 1 file changed, 107 insertions(+), 55 deletions(-) diff --git a/src/Command/LoadWarrantySerialCommand.php b/src/Command/LoadWarrantySerialCommand.php index 18c1ec1b..026ff251 100644 --- a/src/Command/LoadWarrantySerialCommand.php +++ b/src/Command/LoadWarrantySerialCommand.php @@ -54,7 +54,7 @@ class LoadWarrantySerialCommand extends Command // get the filenames from the queue table with status pending $db = $em->getConnection(); - $ws_query_sql = 'SELECT file_serial, api_user FROM warranty_serial_queue + $ws_query_sql = 'SELECT id, file_serial, api_user FROM warranty_serial_queue WHERE status = :status'; $ws_query_stmt = $db->prepare($ws_query_sql); @@ -62,14 +62,22 @@ class LoadWarrantySerialCommand extends Command $ws_results = $ws_query_stmt->executeQuery(); + $output_info = []; while ($row = $ws_results->fetchAssociative()) { $filename = $row['file_serial']; $user_id = $row['api_user']; + $ws_file_id = $row['id']; $fname = $this->project_dir . '/public/warranty_serial_uploads/' . $filename; - $this->processWarrantySerialFile($fname, $user_id); + // TODO: need to consider what happens if file cannot be read. + // if file can be read, output_info will have an array of objects + // which can be used to transform into data + // if file cannot be read, output_info will have an array of strings + $output_info[] = $this->processWarrantySerialFile($fname, $user_id); + + $this->updateWarrantySerialQueue($ws_file_id); } return 0; @@ -77,6 +85,8 @@ class LoadWarrantySerialCommand extends Command protected function processWarrantySerialFile($csv_file, $user_id) { + $output_info = []; + // attempt to open file try { @@ -92,73 +102,83 @@ class LoadWarrantySerialCommand extends Command ]; $this->upload_logger->logWarrantySerialUploadInfo($log_data); - error_log($error); + $output_info = $this->setOutputInfo('', 'not_uploaded', true, $error); + + return $output_info; } while(($row = fgetcsv($fh)) !== false) { - $is_valid = $this->validateRow($row, $user_id); - if ($is_valid) + $validation_result = $this->validateRow($row, $user_id); + if (!empty($validation_result)) { - // valid entry, we parse and insert - $serial = trim(strtoupper($row[0])); - $sku = trim(strtoupper($row[1])); - $dispatch_status = trim($row[2]); - $str_date_create = trim($row[3]); - $inventory_status = trim($row[4]); - $cat_id = trim($row[5]); - $cat_name = trim(strtoupper($row[6])); + $output_info[] = $validation_result; + continue; + } - // we are sure that this is a valid date at this point - $created_date = $this->convertDateCreate($str_date_create); + // valid entry, we parse and insert + $serial = trim(strtoupper($row[0])); + $sku = trim(strtoupper($row[1])); + $dispatch_status = trim($row[2]); + $str_date_create = trim($row[3]); + $inventory_status = trim($row[4]); + $cat_id = trim($row[5]); + $cat_name = trim(strtoupper($row[6])); - $meta_info = [ - 'dispatch_status' => $dispatch_status, - 'inventory_status' => $inventory_status, - 'category_id' => $cat_id, - 'category_name' => $cat_name, - ]; + // we are sure that this is a valid date at this point + $created_date = $this->convertDateCreate($str_date_create); - $info = json_encode($meta_info); + $meta_info = [ + 'dispatch_status' => $dispatch_status, + 'inventory_status' => $inventory_status, + 'category_id' => $cat_id, + 'category_name' => $cat_name, + ]; - // prepare the data - $source = 'motiv'; - if ($sku == 'N/A') - $sku = null; + $info = json_encode($meta_info); - // prepared statement - $db = $this->em->getConnection(); - $insert_stmt = $db->prepare('INSERT INTO warranty_serial (id, sku, date_create, source, meta_info) - VALUES (:serial, :sku, :date_create, :source, :meta_info)'); + // prepare the data + $source = 'motiv'; + if ($sku == 'N/A') + $sku = null; - $res = $insert_stmt->execute([ - ':serial' => $serial, - ':sku' => $sku, - ':date_create' => $created_date, - ':source' => $source, - ':meta_info' => $info, - ]); + // prepared statement + $db = $this->em->getConnection(); + $insert_stmt = $db->prepare('INSERT INTO warranty_serial (id, sku, date_create, source, meta_info) + VALUES (:serial, :sku, :date_create, :source, :meta_info)'); - if (!$res) - { - // log the not successful insert - $err = $insert_stmt->errorInfo(); - $error = $err[2]; - $this->logLoadInfo($user_id, false, $serial, $error); - } - else - { - // log the successful insert - $this->logLoadInfo($user_id, true, $serial, ''); - } + $res = $insert_stmt->execute([ + ':serial' => $serial, + ':sku' => $sku, + ':date_create' => $created_date, + ':source' => $source, + ':meta_info' => $info, + ]); + + if (!$res) + { + // log the not successful insert + $err = $insert_stmt->errorInfo(); + $error = $err[2]; + $this->logLoadInfo($user_id, false, $serial, $error); + + $output_info = $this->setOutputInfo($serial, 'not uploaded', true, $error); + } + else + { + // log the successful insert + $this->logLoadInfo($user_id, true, $serial, ''); + + $output_info = $this->setOutputInfo($serial, 'uploaded', false, ''); } } - return true; + return $output_info; } protected function validateRow($row, $user_id) { + $data = []; // possible lines: // (1) header in csv file - ignore // SerialNumber,Sku,DispatchStatus,CreatedDate,InventoryStatus,CategoryID,CategoryName @@ -189,7 +209,9 @@ class LoadWarrantySerialCommand extends Command $error = 'Empty serial'; $this->logLoadInfo($user_id, false, '', $error); - return false; + $data = $this->setOutputInfo('', 'not_uploaded', true, $error); + + return $data; } // validate the date created @@ -203,7 +225,9 @@ class LoadWarrantySerialCommand extends Command $error = 'Invalid date create'; $this->logLoadInfo($user_id, false, $serial, $error); - return false; + $data = $this->setOutputInfo($serial, 'not_uploaded', true, $error); + + return $data; } // check if serial is a dupe @@ -214,11 +238,13 @@ class LoadWarrantySerialCommand extends Command $error = 'Duplicate serial'; $this->logLoadInfo($user_id, false, $serial, $error); - return false; + $data = $this->setOutputInfo($serial, 'not_uploaded', true, $error) + + return $data; } - // valid entry - return true; + // valid entry, return empty + return $data; } protected function convertDateCreate($str_date_create) @@ -267,4 +293,30 @@ class LoadWarrantySerialCommand extends Command $this->load_logger->logWarrantySerialLoadInfo($log_data); } + + protected function updateWarrantySerialQueue($id) + { + // prepared statement + $db = $this->em->getConnection(); + $update_stmt = $db->prepare('UPDATE warranty_serial_queue SET status = :status + WHERE id = :id'); + + $res = $update_stmt->execute([ + ':status' => 'done', + ':id' => $id, + ]); + } + + protected function setOutputInfo($serial, $status, $has_error, $error_message) + { + $info = [ + 'serial' => $serial, + 'status' => $status, + 'has_error' => $has_error, + 'error_message' => $error_message + ]; + + return $info; + } + } From 70e9b933d97315b1938d9de01153f1d21ca57632 Mon Sep 17 00:00:00 2001 From: Korina Cordero Date: Fri, 16 Sep 2022 05:22:04 +0000 Subject: [PATCH 07/16] Add output to command. #704 --- src/Command/LoadWarrantySerialCommand.php | 111 +++++++++++++++++----- src/Entity/WarrantySerialUploadLog.php | 4 +- 2 files changed, 89 insertions(+), 26 deletions(-) diff --git a/src/Command/LoadWarrantySerialCommand.php b/src/Command/LoadWarrantySerialCommand.php index 026ff251..c4e35af5 100644 --- a/src/Command/LoadWarrantySerialCommand.php +++ b/src/Command/LoadWarrantySerialCommand.php @@ -69,22 +69,21 @@ class LoadWarrantySerialCommand extends Command $user_id = $row['api_user']; $ws_file_id = $row['id']; - $fname = $this->project_dir . '/public/warranty_serial_uploads/' . $filename; - - // TODO: need to consider what happens if file cannot be read. - // if file can be read, output_info will have an array of objects - // which can be used to transform into data - // if file cannot be read, output_info will have an array of strings - $output_info[] = $this->processWarrantySerialFile($fname, $user_id); + $output_info[] = $this->processWarrantySerialFile($filename, $user_id); $this->updateWarrantySerialQueue($ws_file_id); + + // send results back to third party + $this->sendResults($output_info); } return 0; } - protected function processWarrantySerialFile($csv_file, $user_id) + protected function processWarrantySerialFile($filename, $user_id) { + $csv_file = $this->project_dir . '/public/warranty_serial_uploads/' . $filename; + $output_info = []; // attempt to open file @@ -102,22 +101,26 @@ class LoadWarrantySerialCommand extends Command ]; $this->upload_logger->logWarrantySerialUploadInfo($log_data); - $output_info = $this->setOutputInfo('', 'not_uploaded', true, $error); + $output_info = $this->setOutputInfo($filename, true, $error, $data); return $output_info; } + $data = []; while(($row = fgetcsv($fh)) !== false) { $validation_result = $this->validateRow($row, $user_id); if (!empty($validation_result)) { - $output_info[] = $validation_result; + $data[] = $validation_result; continue; } // valid entry, we parse and insert $serial = trim(strtoupper($row[0])); + + error_log('Processing ' . $serial); + $sku = trim(strtoupper($row[1])); $dispatch_status = trim($row[2]); $str_date_create = trim($row[3]); @@ -162,17 +165,30 @@ class LoadWarrantySerialCommand extends Command $error = $err[2]; $this->logLoadInfo($user_id, false, $serial, $error); - $output_info = $this->setOutputInfo($serial, 'not uploaded', true, $error); + $data = [ + 'serial' => $serial, + 'status' => 'error', + 'has_error' => true, + 'error_message' => $error, + ]; } else { // log the successful insert $this->logLoadInfo($user_id, true, $serial, ''); - $output_info = $this->setOutputInfo($serial, 'uploaded', false, ''); + $data = [ + 'serial' => $serial, + 'status' => 'success', + 'has_error' => false, + 'error_message' => '', + ]; } } + // form what we output + $output_info = $this->setOutputInfo($filename, false, '', $data); + return $output_info; } @@ -191,15 +207,31 @@ class LoadWarrantySerialCommand extends Command // check if the line is a header if ($row[0] == 'SerialNumber') { - // no need to log, just ignore and skip - return false; + // no need to log, but send back error + $error = 'Invalid information.'; + $data = [ + 'serial' => '', + 'status' => 'error', + 'has_error' => true, + 'error_message' => $error, + ]; + + return $data; } // check if empty line if ($row == array(null)) { - // no need to log, just ignore and skip - return false; + // no need to log, but send back error + $error = 'Empty line'; + $data = [ + 'serial' => '', + 'status' => 'error', + 'has_error' => true, + 'error_message' => $error, + ]; + + return $data; } // check if empty serial @@ -209,7 +241,12 @@ class LoadWarrantySerialCommand extends Command $error = 'Empty serial'; $this->logLoadInfo($user_id, false, '', $error); - $data = $this->setOutputInfo('', 'not_uploaded', true, $error); + $data = [ + 'serial' => '', + 'status' => 'error', + 'has_error' => true, + 'error_message' => $error, + ]; return $data; } @@ -222,10 +259,15 @@ class LoadWarrantySerialCommand extends Command if ($date_create == null) { // log - $error = 'Invalid date create'; + $error = 'Invalid date create.'; $this->logLoadInfo($user_id, false, $serial, $error); - $data = $this->setOutputInfo($serial, 'not_uploaded', true, $error); + $data = [ + 'serial' => $serial, + 'status' => 'error', + 'has_error' => true, + 'error_message' => $error, + ]; return $data; } @@ -235,10 +277,15 @@ class LoadWarrantySerialCommand extends Command if ($existing_serial != null) { // log - $error = 'Duplicate serial'; + $error = 'Serial already exists.'; $this->logLoadInfo($user_id, false, $serial, $error); - $data = $this->setOutputInfo($serial, 'not_uploaded', true, $error) + $data = [ + 'serial' => $serial, + 'status' => 'error', + 'has_error' => true, + 'error_message' => $error, + ]; return $data; } @@ -307,16 +354,30 @@ class LoadWarrantySerialCommand extends Command ]); } - protected function setOutputInfo($serial, $status, $has_error, $error_message) + protected function setOutputInfo($filename, $has_error, $error_message, $entries) { + // need to get the original filename from warranty_serial_upload_log + // use the uploaded_file_serial which has the saved csv file + $upload_entry = $this->em->getRepository(WarrantySerialUploadLog::class)->findOneBy(['uploaded_file_serial' => $filename]); + $original_filename = $upload_entry->getOrigFileSerial(); + $info = [ - 'serial' => $serial, - 'status' => $status, + 'filename' => $original_filename, 'has_error' => $has_error, - 'error_message' => $error_message + 'error_message' => $error_message, + 'data' => $entries, ]; return $info; } + protected function sendResults($output_info) + { + $json_output = json_encode($output_info); + + error_log(print_r($json_output, true)); + + // TODO: sent json output somewhere + } + } diff --git a/src/Entity/WarrantySerialUploadLog.php b/src/Entity/WarrantySerialUploadLog.php index 44f5b16a..12b97d67 100644 --- a/src/Entity/WarrantySerialUploadLog.php +++ b/src/Entity/WarrantySerialUploadLog.php @@ -8,7 +8,9 @@ use DateTime; /** * @ORM\Entity - * @ORM\Table(name="warranty_serial_upload_log") + * @ORM\Table(name="warranty_serial_upload_log", indexes={ + * @ORM\Index(name="uploaded_file_idx", columns={"uploaded_file_serial"}), + * }) */ class WarrantySerialUploadLog { From d9aaea928c5479303749e7e7f55c04f08f8ee0e2 Mon Sep 17 00:00:00 2001 From: Korina Cordero Date: Mon, 19 Sep 2022 06:27:03 +0000 Subject: [PATCH 08/16] Add call to curl to command. #704 --- src/Command/LoadWarrantySerialCommand.php | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/src/Command/LoadWarrantySerialCommand.php b/src/Command/LoadWarrantySerialCommand.php index c4e35af5..41fcb15d 100644 --- a/src/Command/LoadWarrantySerialCommand.php +++ b/src/Command/LoadWarrantySerialCommand.php @@ -378,6 +378,26 @@ class LoadWarrantySerialCommand extends Command error_log(print_r($json_output, true)); // TODO: sent json output somewhere + + $curl = curl_init(); + + $options = [ + CURLOPT_URL => $this->base_url . '/' . $url, + CURLOPT_POST => true, + CURLOPT_RETURNTRANSFER => true, + CURLOPT_POSTFIELDS => $body, + CURLOPT_HTTPHEADER => [ + 'Content-Type: application/json', + 'Ocp-Apim-Subscription-Key: ' . $this->sub_key, + 'Authorization: Bearer ' . $this->token, + ], + ]; + + curl_setopt_array($curl, $options); + $res = curl_exec($curl); + + curl_close($curl); + } } From 0ffbf10879be855de450af61679fa92195b3db48 Mon Sep 17 00:00:00 2001 From: Korina Cordero Date: Mon, 19 Sep 2022 07:15:55 +0000 Subject: [PATCH 09/16] Fix conflict. #704 --- src/Controller/APIUserController.php | 6 ------ 1 file changed, 6 deletions(-) diff --git a/src/Controller/APIUserController.php b/src/Controller/APIUserController.php index dd93d7b8..122d3830 100644 --- a/src/Controller/APIUserController.php +++ b/src/Controller/APIUserController.php @@ -162,15 +162,9 @@ class APIUserController extends Controller // set api user in rider $rider->setAPIUser($obj); -<<<<<<< HEAD $obj->setRider($rider) ->setMetadata($meta); } -======= - // set api user in rider - if ($rider != null) - $rider->setAPIUser($obj); ->>>>>>> 7416be842559728bbb3743c0035a6d26605eb1be // set and save values $obj->setName($req->request->get('name')) From fc15b514bf6ad7257b131a0b16237ae40cc7e49b Mon Sep 17 00:00:00 2001 From: Korina Cordero Date: Mon, 19 Sep 2022 08:21:29 +0000 Subject: [PATCH 10/16] Add file id. #704 --- src/Command/LoadWarrantySerialCommand.php | 4 -- .../CAPI/WarrantySerialController.php | 20 +++++++--- src/Entity/WarrantySerialQueue.php | 38 ++++++++++++++++++- 3 files changed, 52 insertions(+), 10 deletions(-) diff --git a/src/Command/LoadWarrantySerialCommand.php b/src/Command/LoadWarrantySerialCommand.php index 41fcb15d..fc861a20 100644 --- a/src/Command/LoadWarrantySerialCommand.php +++ b/src/Command/LoadWarrantySerialCommand.php @@ -377,8 +377,6 @@ class LoadWarrantySerialCommand extends Command error_log(print_r($json_output, true)); - // TODO: sent json output somewhere - $curl = curl_init(); $options = [ @@ -388,8 +386,6 @@ class LoadWarrantySerialCommand extends Command CURLOPT_POSTFIELDS => $body, CURLOPT_HTTPHEADER => [ 'Content-Type: application/json', - 'Ocp-Apim-Subscription-Key: ' . $this->sub_key, - 'Authorization: Bearer ' . $this->token, ], ]; diff --git a/src/Controller/CAPI/WarrantySerialController.php b/src/Controller/CAPI/WarrantySerialController.php index db447dd5..7595d3ff 100644 --- a/src/Controller/CAPI/WarrantySerialController.php +++ b/src/Controller/CAPI/WarrantySerialController.php @@ -66,12 +66,17 @@ class WarrantySerialController extends APIController protected function registerWarrantySerialFile($em, $file, $serial_filename, $user_id) { + // parse the serial filename to get the file id + $parts = explode('/', $serial_filename); + $file_id = $parts[0]; $orig_filename = pathinfo($file->getClientOriginalName(), PATHINFO_FILENAME) . '.' . $file->getClientOriginalExtension(); $ws_file = new WarrantySerialQueue(); $ws_file->setFileSerial($serial_filename) - ->setStatus('pending'); + ->setStatus('pending') + ->setOrigFileSerial($orig_filename) + ->setFileID($file_id); $em->persist($ws_file); @@ -85,7 +90,11 @@ class WarrantySerialController extends APIController $this->upload_logger->logWarrantySerialUploadInfo($log_data); - return new APIResponse(true, 'Warranty serial file uploaded.'); + $data = [ + 'id' => $file_id, + ]; + + return new APIResponse(true, 'Warranty serial file uploaded.', $data); } protected function handleSerialFileUpload($file, $target_dir) @@ -109,13 +118,14 @@ class WarrantySerialController extends APIController // get current date $curr_date = new DateTime(); - $str_curr_date = $curr_date->format('YmdHis'); + $str_curr_date = $curr_date->format('Ymd'); + $file_id = $str_curr_date . uniqid(); // move file $filename = 'warranty_serial' . '.' . $file->getClientOriginalExtension(); - $file->move($target_dir . '/' . $str_curr_date, $filename); + $file->move($target_dir . '/' . $file_id, $filename); - return $str_curr_date . '/' . $filename; + return $file_id . '/' . $filename; } protected function checkRequiredParamsForFiles(Request $req, $params, $user_id) diff --git a/src/Entity/WarrantySerialQueue.php b/src/Entity/WarrantySerialQueue.php index 585be163..d139a173 100644 --- a/src/Entity/WarrantySerialQueue.php +++ b/src/Entity/WarrantySerialQueue.php @@ -27,7 +27,13 @@ class WarrantySerialQueue */ protected $date_create; - // warranty serial file + // original filename of warranty serial + /** + * @ORM\Column(type="string", length=80, nullable=true) + */ + protected $orig_file_serial; + + // warranty serial file that we created /** * @ORM\Column(type="string", length=80, nullable=true) */ @@ -45,12 +51,20 @@ class WarrantySerialQueue */ protected $api_user; + // file identifier that we send back + /** + * @ORM\Column(type="string", length=21) + */ + protected $file_id; + public function __construct() { $this->date_create = new DateTime(); $this->file_serial = null; + $this->orig_file_serial = null; $this->status = ''; $this->api_user = ''; + $this->file_id = ''; } public function getID() @@ -90,4 +104,26 @@ class WarrantySerialQueue { return $this->api_user; } + + public function setFileID($file_id) + { + $this->file_id = $file_id; + return $this; + } + + public function getFileID() + { + return $this->file_id; + } + + public function setOrigFileSerial($file = null) + { + $this->orig_file_serial = $file; + return $this; + } + + public function getOrigFileSerial() + { + return $this->orig_file_serial; + } } From f8910860a6314df033f087c964550fb951ee8d8b Mon Sep 17 00:00:00 2001 From: Korina Cordero Date: Mon, 19 Sep 2022 09:28:12 +0000 Subject: [PATCH 11/16] Fix json output for command. #704 --- config/services.yaml | 4 ++ src/Command/LoadWarrantySerialCommand.php | 49 +++++++++++-------- .../CAPI/WarrantySerialController.php | 3 +- src/Entity/WarrantySerialUploadLog.php | 4 +- 4 files changed, 36 insertions(+), 24 deletions(-) diff --git a/config/services.yaml b/config/services.yaml index 72817508..c130b986 100644 --- a/config/services.yaml +++ b/config/services.yaml @@ -111,6 +111,10 @@ services: $cvu_mfg_id: "%env(CVU_MFG_ID)%" $cvu_brand_id: "%env(CVU_BRAND_ID)%" + App\Command\LoadWarrantySerialCommand: + arguments: + $callback_url: "%env(WARRANTY_SERIAL_CALLBACK_URL)%" + # rider tracker service App\Service\RiderTracker: arguments: diff --git a/src/Command/LoadWarrantySerialCommand.php b/src/Command/LoadWarrantySerialCommand.php index fc861a20..cd3c5342 100644 --- a/src/Command/LoadWarrantySerialCommand.php +++ b/src/Command/LoadWarrantySerialCommand.php @@ -11,6 +11,7 @@ use Symfony\Component\HttpKernel\KernelInterface; use Doctrine\ORM\EntityManagerInterface; use App\Entity\WarrantySerial; +use App\Entity\WarrantySerialQueue; use App\Entity\WarrantySerialUploadLog; use App\Entity\WarrantySerialLoadLog; @@ -26,14 +27,16 @@ class LoadWarrantySerialCommand extends Command protected $upload_logger; protected $load_logger; protected $project_dir; + protected $callback_url; public function __construct(EntityManagerInterface $em, WarrantySerialUploadLogger $upload_logger, - WarrantySerialLoadLogger $load_logger, KernelInterface $kernel) + WarrantySerialLoadLogger $load_logger, KernelInterface $kernel, $callback_url) { $this->em = $em; $this->upload_logger = $upload_logger; $this->load_logger = $load_logger; $this->project_dir = $kernel->getProjectDir(); + $this->callback_url = $callback_url; parent::__construct(); } @@ -54,7 +57,7 @@ class LoadWarrantySerialCommand extends Command // get the filenames from the queue table with status pending $db = $em->getConnection(); - $ws_query_sql = 'SELECT id, file_serial, api_user FROM warranty_serial_queue + $ws_query_sql = 'SELECT id, file_serial, file_id, api_user FROM warranty_serial_queue WHERE status = :status'; $ws_query_stmt = $db->prepare($ws_query_sql); @@ -67,20 +70,21 @@ class LoadWarrantySerialCommand extends Command { $filename = $row['file_serial']; $user_id = $row['api_user']; - $ws_file_id = $row['id']; + $id = $row['id']; + $file_id = $row['file_id']; - $output_info[] = $this->processWarrantySerialFile($filename, $user_id); + $output_info[] = $this->processWarrantySerialFile($filename, $user_id, $file_id); - $this->updateWarrantySerialQueue($ws_file_id); - - // send results back to third party - $this->sendResults($output_info); + $this->updateWarrantySerialQueue($id); } + // send results back to third party + $this->sendResults($output_info); + return 0; } - protected function processWarrantySerialFile($filename, $user_id) + protected function processWarrantySerialFile($filename, $user_id, $file_id) { $csv_file = $this->project_dir . '/public/warranty_serial_uploads/' . $filename; @@ -101,7 +105,7 @@ class LoadWarrantySerialCommand extends Command ]; $this->upload_logger->logWarrantySerialUploadInfo($log_data); - $output_info = $this->setOutputInfo($filename, true, $error, $data); + $output_info = $this->setOutputInfo($filename, $file_id, true, $error, $data); return $output_info; } @@ -119,7 +123,7 @@ class LoadWarrantySerialCommand extends Command // valid entry, we parse and insert $serial = trim(strtoupper($row[0])); - error_log('Processing ' . $serial); + // error_log('Processing ' . $serial); $sku = trim(strtoupper($row[1])); $dispatch_status = trim($row[2]); @@ -165,7 +169,7 @@ class LoadWarrantySerialCommand extends Command $error = $err[2]; $this->logLoadInfo($user_id, false, $serial, $error); - $data = [ + $data[] = [ 'serial' => $serial, 'status' => 'error', 'has_error' => true, @@ -177,7 +181,7 @@ class LoadWarrantySerialCommand extends Command // log the successful insert $this->logLoadInfo($user_id, true, $serial, ''); - $data = [ + $data[] = [ 'serial' => $serial, 'status' => 'success', 'has_error' => false, @@ -187,7 +191,7 @@ class LoadWarrantySerialCommand extends Command } // form what we output - $output_info = $this->setOutputInfo($filename, false, '', $data); + $output_info = $this->setOutputInfo($filename, $file_id, false, '', $data); return $output_info; } @@ -354,14 +358,15 @@ class LoadWarrantySerialCommand extends Command ]); } - protected function setOutputInfo($filename, $has_error, $error_message, $entries) + protected function setOutputInfo($filename, $file_id, $has_error, $error_message, $entries) { - // need to get the original filename from warranty_serial_upload_log + // need to get the original filename from warranty_serial_queue // use the uploaded_file_serial which has the saved csv file - $upload_entry = $this->em->getRepository(WarrantySerialUploadLog::class)->findOneBy(['uploaded_file_serial' => $filename]); + $upload_entry = $this->em->getRepository(WarrantySerialQueue::class)->findOneBy(['file_id' => $file_id]); $original_filename = $upload_entry->getOrigFileSerial(); $info = [ + 'id' => $file_id, 'filename' => $original_filename, 'has_error' => $has_error, 'error_message' => $error_message, @@ -375,12 +380,16 @@ class LoadWarrantySerialCommand extends Command { $json_output = json_encode($output_info); - error_log(print_r($json_output, true)); + // error_log(print_r($json_output, true)); + // TODO: make a test controller + // send json there + // log what it gets + /* $curl = curl_init(); $options = [ - CURLOPT_URL => $this->base_url . '/' . $url, + CURLOPT_URL => $this->callback_url, CURLOPT_POST => true, CURLOPT_RETURNTRANSFER => true, CURLOPT_POSTFIELDS => $body, @@ -392,7 +401,7 @@ class LoadWarrantySerialCommand extends Command curl_setopt_array($curl, $options); $res = curl_exec($curl); - curl_close($curl); + curl_close($curl); */ } diff --git a/src/Controller/CAPI/WarrantySerialController.php b/src/Controller/CAPI/WarrantySerialController.php index 7595d3ff..c845e367 100644 --- a/src/Controller/CAPI/WarrantySerialController.php +++ b/src/Controller/CAPI/WarrantySerialController.php @@ -76,7 +76,8 @@ class WarrantySerialController extends APIController $ws_file->setFileSerial($serial_filename) ->setStatus('pending') ->setOrigFileSerial($orig_filename) - ->setFileID($file_id); + ->setFileID($file_id) + ->setApiUser($user_id); $em->persist($ws_file); diff --git a/src/Entity/WarrantySerialUploadLog.php b/src/Entity/WarrantySerialUploadLog.php index 12b97d67..44f5b16a 100644 --- a/src/Entity/WarrantySerialUploadLog.php +++ b/src/Entity/WarrantySerialUploadLog.php @@ -8,9 +8,7 @@ use DateTime; /** * @ORM\Entity - * @ORM\Table(name="warranty_serial_upload_log", indexes={ - * @ORM\Index(name="uploaded_file_idx", columns={"uploaded_file_serial"}), - * }) + * @ORM\Table(name="warranty_serial_upload_log") */ class WarrantySerialUploadLog { From 138c60a61b909ac154dea9acfda41d3c3d0245ec Mon Sep 17 00:00:00 2001 From: Korina Cordero Date: Tue, 20 Sep 2022 02:38:32 +0000 Subject: [PATCH 12/16] Comment out sending to callback URL. #704 --- src/Command/LoadWarrantySerialCommand.php | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/src/Command/LoadWarrantySerialCommand.php b/src/Command/LoadWarrantySerialCommand.php index cd3c5342..ac40cfb1 100644 --- a/src/Command/LoadWarrantySerialCommand.php +++ b/src/Command/LoadWarrantySerialCommand.php @@ -79,7 +79,7 @@ class LoadWarrantySerialCommand extends Command } // send results back to third party - $this->sendResults($output_info); + // $this->sendResults($output_info); return 0; } @@ -208,6 +208,8 @@ class LoadWarrantySerialCommand extends Command // (5) Empty line - ignore // (6) empty sku - log + // TODO: count the number of fields + // check if the line is a header if ($row[0] == 'SerialNumber') { @@ -378,14 +380,12 @@ class LoadWarrantySerialCommand extends Command protected function sendResults($output_info) { - $json_output = json_encode($output_info); + $body = json_encode($output_info); - // error_log(print_r($json_output, true)); + // error_log(print_r($body, true)); + + error_log('Sending json output to ' . $this->callback_url); - // TODO: make a test controller - // send json there - // log what it gets - /* $curl = curl_init(); $options = [ @@ -401,8 +401,7 @@ class LoadWarrantySerialCommand extends Command curl_setopt_array($curl, $options); $res = curl_exec($curl); - curl_close($curl); */ - + curl_close($curl); } } From 6314b4097d8e1edcaaf4694270d7be28bf053351 Mon Sep 17 00:00:00 2001 From: Korina Cordero Date: Tue, 20 Sep 2022 04:20:36 +0000 Subject: [PATCH 13/16] Add test capi endpoint. #704 --- config/packages/security.yaml | 4 ++++ config/routes/capi.yaml | 4 ++++ src/Command/LoadWarrantySerialCommand.php | 5 ++++- src/Controller/CAPI/TestController.php | 15 +++++++++++++++ 4 files changed, 27 insertions(+), 1 deletion(-) diff --git a/config/packages/security.yaml b/config/packages/security.yaml index 60518b75..1b0f6824 100644 --- a/config/packages/security.yaml +++ b/config/packages/security.yaml @@ -43,6 +43,10 @@ security: pattern: ^\/rapi\/ security: false + test_capi: + pattern: ^\/test_capi\/ + security: false + warranty_api: pattern: ^\/capi\/ stateless: true diff --git a/config/routes/capi.yaml b/config/routes/capi.yaml index 0ae29c7c..44f57c99 100644 --- a/config/routes/capi.yaml +++ b/config/routes/capi.yaml @@ -3,6 +3,10 @@ capi_test: path: /capi/test controller: App\Controller\CAPI\TestController::test +capi_test_warranty_serial: + path: /test_capi/test/warranty_serial + controller: App\Controller\CAPI\TestController::warrantySerial + # battery api diff --git a/src/Command/LoadWarrantySerialCommand.php b/src/Command/LoadWarrantySerialCommand.php index ac40cfb1..e0da8b11 100644 --- a/src/Command/LoadWarrantySerialCommand.php +++ b/src/Command/LoadWarrantySerialCommand.php @@ -79,7 +79,7 @@ class LoadWarrantySerialCommand extends Command } // send results back to third party - // $this->sendResults($output_info); + $this->sendResults($output_info); return 0; } @@ -402,6 +402,9 @@ class LoadWarrantySerialCommand extends Command $res = curl_exec($curl); curl_close($curl); + + // check result + error_log('Result ' . $res); } } diff --git a/src/Controller/CAPI/TestController.php b/src/Controller/CAPI/TestController.php index e1a5ea08..c7bdc404 100644 --- a/src/Controller/CAPI/TestController.php +++ b/src/Controller/CAPI/TestController.php @@ -17,4 +17,19 @@ class TestController extends APIController ]; return new APIResponse(true, 'Test successful.', $data); } + + public function warrantySerial(Request $req) + { + error_log('Got request'); + + $res = json_decode($req->getContent(), true); + + // return $res; + + $data = [ + 'status' => 'Test successful.', + ]; + return new APIResponse(true, 'Test successful.', $data); + } + } From f40155e1f51c292482de40465585e02ea59aa6ac Mon Sep 17 00:00:00 2001 From: Korina Cordero Date: Tue, 20 Sep 2022 08:19:43 +0000 Subject: [PATCH 14/16] Change the logging of serial. #704 --- src/Command/LoadWarrantySerialCommand.php | 26 ++++++++++----- src/Controller/CAPI/TestController.php | 2 +- src/Service/WarrantySerialLoadLogger.php | 40 ++++++++++++++++------- 3 files changed, 46 insertions(+), 22 deletions(-) diff --git a/src/Command/LoadWarrantySerialCommand.php b/src/Command/LoadWarrantySerialCommand.php index e0da8b11..3ef4d849 100644 --- a/src/Command/LoadWarrantySerialCommand.php +++ b/src/Command/LoadWarrantySerialCommand.php @@ -28,6 +28,7 @@ class LoadWarrantySerialCommand extends Command protected $load_logger; protected $project_dir; protected $callback_url; + protected $log_data; public function __construct(EntityManagerInterface $em, WarrantySerialUploadLogger $upload_logger, WarrantySerialLoadLogger $load_logger, KernelInterface $kernel, $callback_url) @@ -51,6 +52,7 @@ class LoadWarrantySerialCommand extends Command protected function execute(InputInterface $input, OutputInterface $output) { $em = $this->em; + $this->log_data = []; $status = 'pending'; @@ -78,6 +80,10 @@ class LoadWarrantySerialCommand extends Command $this->updateWarrantySerialQueue($id); } + // error_log(print_r($this->log_data, true)); + // load log data into db + $this->load_logger->logWarrantySerialLoadInfo($this->log_data); + // send results back to third party $this->sendResults($output_info); @@ -337,14 +343,16 @@ class LoadWarrantySerialCommand extends Command protected function logLoadInfo($user_id, $is_loaded, $serial, $error) { - $log_data = [ - 'user_id' => $user_id, - 'is_loaded' => $is_loaded, - 'serial' => $serial, - 'error' => $error, - ]; + $date_create = new DateTime(); + $str_date_create = $date_create->format('Y-m-d H:i:s'); - $this->load_logger->logWarrantySerialLoadInfo($log_data); + $this->log_data[] = [ + $str_date_create, + $user_id, + $serial, + $is_loaded, + $error, + ]; } protected function updateWarrantySerialQueue($id) @@ -384,7 +392,7 @@ class LoadWarrantySerialCommand extends Command // error_log(print_r($body, true)); - error_log('Sending json output to ' . $this->callback_url); + // error_log('Sending json output to ' . $this->callback_url); $curl = curl_init(); @@ -404,7 +412,7 @@ class LoadWarrantySerialCommand extends Command curl_close($curl); // check result - error_log('Result ' . $res); + // error_log('Result ' . $res); } } diff --git a/src/Controller/CAPI/TestController.php b/src/Controller/CAPI/TestController.php index c7bdc404..9604c056 100644 --- a/src/Controller/CAPI/TestController.php +++ b/src/Controller/CAPI/TestController.php @@ -27,7 +27,7 @@ class TestController extends APIController // return $res; $data = [ - 'status' => 'Test successful.', + 'result' => $res, ]; return new APIResponse(true, 'Test successful.', $data); } diff --git a/src/Service/WarrantySerialLoadLogger.php b/src/Service/WarrantySerialLoadLogger.php index 336b1dcb..904a6535 100644 --- a/src/Service/WarrantySerialLoadLogger.php +++ b/src/Service/WarrantySerialLoadLogger.php @@ -4,7 +4,7 @@ namespace App\Service; use Doctrine\ORM\EntityManagerInterface; -use App\Entity\WarrantySerialLoadLog; +use DateTime; class WarrantySerialLoadLogger { @@ -17,21 +17,37 @@ class WarrantySerialLoadLogger public function logWarrantySerialLoadInfo($log_data) { - $log_entry = new WarrantySerialLoadLog(); + // cache directory + $cache_dir = __DIR__ . '/../../var/cache'; - $user_id = $log_data['user_id']; - $is_loaded = $log_data['is_loaded']; - $error = $log_data['error']; - $serial = $log_data['serial']; + $file = $cache_dir . '/warranty_serial_load_log.tab'; + error_log('opening file for warranty serial load log - ' . $file); - $log_entry->setApiUser($user_id) - ->setSerial($serial) - ->setLoaded($is_loaded) - ->setError($error); + $fp = fopen($file, 'w'); + if ($fp === false) + { + error_log('could not open file for load data infile - ' . $file); + } + else + { + foreach ($log_data as $key => $data) + { + $line = implode('|', $data) . "\r\n"; + fwrite($fp, $line); + } + } - $this->em->persist($log_entry); - $this->em->flush(); + fclose($fp); + // prepared statement + $db = $this->em->getConnection(); + $stmt = $db->prepare('LOAD DATA LOCAL INFILE \''. $file . '\' INTO TABLE warranty_serial_load_log FIELDS TERMINATED BY \'|\' LINES TERMINATED BY \'\\r\\n\' (date_create, api_user, serial, flag_loaded, error)'); + + $result = $stmt->execute(); + if (!$result) + error_log('Failed loading data.'); + + // TODO: delete file? } } From 9efe73e8b246400b825a9206481c0e6f5a808e24 Mon Sep 17 00:00:00 2001 From: Korina Cordero Date: Wed, 21 Sep 2022 04:54:21 +0000 Subject: [PATCH 15/16] Add validation for invalid number of fields. #704 --- src/Command/LoadWarrantySerialCommand.php | 32 ++++++++++++++++------- 1 file changed, 23 insertions(+), 9 deletions(-) diff --git a/src/Command/LoadWarrantySerialCommand.php b/src/Command/LoadWarrantySerialCommand.php index 3ef4d849..e9fc6718 100644 --- a/src/Command/LoadWarrantySerialCommand.php +++ b/src/Command/LoadWarrantySerialCommand.php @@ -23,6 +23,8 @@ use DateTime; class LoadWarrantySerialCommand extends Command { + const FIELD_COUNT = 7; + protected $em; protected $upload_logger; protected $load_logger; @@ -214,13 +216,11 @@ class LoadWarrantySerialCommand extends Command // (5) Empty line - ignore // (6) empty sku - log - // TODO: count the number of fields - - // check if the line is a header - if ($row[0] == 'SerialNumber') + // check if empty line + if ($row == array(null)) { // no need to log, but send back error - $error = 'Invalid information.'; + $error = 'Empty line'; $data = [ 'serial' => '', 'status' => 'error', @@ -231,11 +231,25 @@ class LoadWarrantySerialCommand extends Command return $data; } - // check if empty line - if ($row == array(null)) + // check the number of fields + if (count($row) != self::FIELD_COUNT) + { + $error = 'Invalid number of fields.'; + $data = [ + 'serial' => '', + 'status' => 'error', + 'has_error' => true, + 'error_message' => $error, + ]; + + return $data; + } + + // check if the line is a header + if ($row[0] == 'SerialNumber') { // no need to log, but send back error - $error = 'Empty line'; + $error = 'Invalid information.'; $data = [ 'serial' => '', 'status' => 'error', @@ -412,7 +426,7 @@ class LoadWarrantySerialCommand extends Command curl_close($curl); // check result - // error_log('Result ' . $res); + error_log('Result ' . $res); } } From 6ee6a7d9a1c73c50001c26df8297862caa473726 Mon Sep 17 00:00:00 2001 From: Korina Cordero Date: Wed, 21 Sep 2022 05:36:39 +0000 Subject: [PATCH 16/16] Comment debug logs. #704 --- src/Command/LoadWarrantySerialCommand.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Command/LoadWarrantySerialCommand.php b/src/Command/LoadWarrantySerialCommand.php index e9fc6718..07ba261a 100644 --- a/src/Command/LoadWarrantySerialCommand.php +++ b/src/Command/LoadWarrantySerialCommand.php @@ -426,7 +426,7 @@ class LoadWarrantySerialCommand extends Command curl_close($curl); // check result - error_log('Result ' . $res); + // error_log('Result ' . $res); } }