From 21231358d650b35eb5f9951994e0b1a6222e9d21 Mon Sep 17 00:00:00 2001 From: Korina Cordero Date: Wed, 28 Sep 2022 02:43:27 +0000 Subject: [PATCH 1/6] Delete entry from warranty_serial_queue when done processing. #708 --- src/Command/LoadWarrantySerialCommand.php | 25 +++++++++-------------- 1 file changed, 10 insertions(+), 15 deletions(-) diff --git a/src/Command/LoadWarrantySerialCommand.php b/src/Command/LoadWarrantySerialCommand.php index af587c3d..ac5d505d 100644 --- a/src/Command/LoadWarrantySerialCommand.php +++ b/src/Command/LoadWarrantySerialCommand.php @@ -62,7 +62,7 @@ class LoadWarrantySerialCommand extends Command // get the filenames from the queue table with status pending $db = $em->getConnection(); - $ws_query_sql = 'SELECT id, file_serial, file_id, api_user FROM warranty_serial_queue + $ws_query_sql = 'SELECT id, file_serial, file_id, api_user, orig_file_serial FROM warranty_serial_queue WHERE status = :status'; $ws_query_stmt = $db->prepare($ws_query_sql); @@ -77,8 +77,9 @@ class LoadWarrantySerialCommand extends Command $user_id = $row['api_user']; $id = $row['id']; $file_id = $row['file_id']; + $orig_filename = $row['orig_file_serial']; - $output_info[] = $this->processWarrantySerialFile($filename, $user_id, $file_id); + $output_info[] = $this->processWarrantySerialFile($filename, $user_id, $file_id, $orig_filename); $this->updateWarrantySerialQueue($id); } @@ -93,7 +94,7 @@ class LoadWarrantySerialCommand extends Command return 0; } - protected function processWarrantySerialFile($filename, $user_id, $file_id) + protected function processWarrantySerialFile($filename, $user_id, $file_id, $orig_filename) { $csv_file = $this->project_dir . '/public/warranty_serial_uploads/' . $filename; @@ -114,7 +115,7 @@ class LoadWarrantySerialCommand extends Command ]; $this->upload_logger->logWarrantySerialUploadInfo($log_data); - $output_info = $this->setOutputInfo($filename, $file_id, true, $error, $data); + $output_info = $this->setOutputInfo($filename, $file_id, true, $error, $data, $orig_filename); return $output_info; } @@ -200,7 +201,7 @@ class LoadWarrantySerialCommand extends Command } // form what we output - $output_info = $this->setOutputInfo($filename, $file_id, false, '', $data); + $output_info = $this->setOutputInfo($filename, $file_id, false, '', $data, $orig_filename, $orig_filename); return $output_info; } @@ -391,25 +392,19 @@ class LoadWarrantySerialCommand extends Command { // prepared statement $db = $this->em->getConnection(); - $update_stmt = $db->prepare('UPDATE warranty_serial_queue SET status = :status + $delete_stmt = $db->prepare('DELETE FROM warranty_serial_queue WHERE id = :id'); - $res = $update_stmt->execute([ - ':status' => 'done', + $res = $delete_stmt->execute([ ':id' => $id, ]); } - protected function setOutputInfo($filename, $file_id, $has_error, $error_message, $entries) + protected function setOutputInfo($filename, $file_id, $has_error, $error_message, $entries, $orig_filename) { - // need to get the original filename from warranty_serial_queue - // use the uploaded_file_serial which has the saved csv file - $upload_entry = $this->em->getRepository(WarrantySerialQueue::class)->findOneBy(['file_id' => $file_id]); - $original_filename = $upload_entry->getOrigFileSerial(); - $info = [ 'id' => $file_id, - 'filename' => $original_filename, + 'filename' => $orig_filename, 'has_error' => $has_error, 'error_message' => $error_message, 'data' => $entries, From 815c25a894427ea0e916282a81ce5d5516fc2b0b Mon Sep 17 00:00:00 2001 From: Korina Cordero Date: Wed, 28 Sep 2022 06:19:41 +0000 Subject: [PATCH 2/6] Add locking and unlocking for warranty serial queue table when deleting an entry. #708 --- src/Command/LoadWarrantySerialCommand.php | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/Command/LoadWarrantySerialCommand.php b/src/Command/LoadWarrantySerialCommand.php index ac5d505d..f7bd5797 100644 --- a/src/Command/LoadWarrantySerialCommand.php +++ b/src/Command/LoadWarrantySerialCommand.php @@ -392,12 +392,18 @@ class LoadWarrantySerialCommand extends Command { // prepared statement $db = $this->em->getConnection(); + + // lock the warranty serial queue table + $db->exec('LOCK TABLES warranty_serial_queue WRITE;'); + $delete_stmt = $db->prepare('DELETE FROM warranty_serial_queue WHERE id = :id'); $res = $delete_stmt->execute([ ':id' => $id, ]); + + $db->exec('UNLOCK TABLES;'); } protected function setOutputInfo($filename, $file_id, $has_error, $error_message, $entries, $orig_filename) From 97d058c3d3aa041a176f66fb30139942566ba38a Mon Sep 17 00:00:00 2001 From: Korina Cordero Date: Wed, 28 Sep 2022 09:43:04 +0000 Subject: [PATCH 3/6] Modify the load warranty serial command to process one file. Add a command to get the number of pending files. Add bash script to call the load warranty serial command. #708 --- ...TotalPendingWarrantySerialFilesCommand.php | 54 +++++++++++++++++++ src/Command/LoadWarrantySerialCommand.php | 2 +- src/Service/WarrantySerialLoadLogger.php | 2 +- .../warranty_serial_load.sh | 10 ++++ 4 files changed, 66 insertions(+), 2 deletions(-) create mode 100644 src/Command/CountTotalPendingWarrantySerialFilesCommand.php create mode 100755 utils/warranty_serial_load/warranty_serial_load.sh diff --git a/src/Command/CountTotalPendingWarrantySerialFilesCommand.php b/src/Command/CountTotalPendingWarrantySerialFilesCommand.php new file mode 100644 index 00000000..2ff96d5b --- /dev/null +++ b/src/Command/CountTotalPendingWarrantySerialFilesCommand.php @@ -0,0 +1,54 @@ +em = $em; + + parent::__construct(); + } + + protected function configure() + { + $this->setName('warrantyserial:count') + ->setDescription('Count number of pending warranty serial files.') + ->setHelp('Count number of pending warranty serial files.'); + } + + protected function execute(InputInterface $input, OutputInterface $output) + { + $em = $this->em; + + $status = 'pending'; + + $db = $em->getConnection(); + + $ws_query_sql = 'SELECT COUNT(*) AS total FROM warranty_serial_queue + WHERE status = :status'; + + $ws_query_stmt = $db->prepare($ws_query_sql); + $ws_query_stmt->bindValue('status', $status); + + $ws_results = $ws_query_stmt->executeQuery(); + + $results = $ws_results->fetchAssociative(); + + $output->write($results['total']); + + return 0; + } +} diff --git a/src/Command/LoadWarrantySerialCommand.php b/src/Command/LoadWarrantySerialCommand.php index f7bd5797..6dfe53d8 100644 --- a/src/Command/LoadWarrantySerialCommand.php +++ b/src/Command/LoadWarrantySerialCommand.php @@ -63,7 +63,7 @@ class LoadWarrantySerialCommand extends Command $db = $em->getConnection(); $ws_query_sql = 'SELECT id, file_serial, file_id, api_user, orig_file_serial FROM warranty_serial_queue - WHERE status = :status'; + WHERE status = :status LIMIT 1'; $ws_query_stmt = $db->prepare($ws_query_sql); $ws_query_stmt->bindValue('status', $status); diff --git a/src/Service/WarrantySerialLoadLogger.php b/src/Service/WarrantySerialLoadLogger.php index 904a6535..b66b4db9 100644 --- a/src/Service/WarrantySerialLoadLogger.php +++ b/src/Service/WarrantySerialLoadLogger.php @@ -21,7 +21,7 @@ class WarrantySerialLoadLogger $cache_dir = __DIR__ . '/../../var/cache'; $file = $cache_dir . '/warranty_serial_load_log.tab'; - error_log('opening file for warranty serial load log - ' . $file); + // error_log('opening file for warranty serial load log - ' . $file); $fp = fopen($file, 'w'); if ($fp === false) diff --git a/utils/warranty_serial_load/warranty_serial_load.sh b/utils/warranty_serial_load/warranty_serial_load.sh new file mode 100755 index 00000000..af79f1af --- /dev/null +++ b/utils/warranty_serial_load/warranty_serial_load.sh @@ -0,0 +1,10 @@ +#!/bin/bash +# get the number fo warranty serial files to process +declare -i num_files + +num_files=`/var/www/resq/bin/console warrantyserial:count` + +for (( i = 1; i <= num_files; i++ )) +do + /usr/bin/php /var/www/resq/bin/console warrantyserial:load +done From 6794db5f6a341de0f5fbef0eb89f054a4be38dab Mon Sep 17 00:00:00 2001 From: Korina Cordero Date: Thu, 29 Sep 2022 05:50:46 +0000 Subject: [PATCH 4/6] Add checking if any file was processed. #708 --- src/Command/LoadWarrantySerialCommand.php | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/src/Command/LoadWarrantySerialCommand.php b/src/Command/LoadWarrantySerialCommand.php index 6dfe53d8..9abb600e 100644 --- a/src/Command/LoadWarrantySerialCommand.php +++ b/src/Command/LoadWarrantySerialCommand.php @@ -63,7 +63,7 @@ class LoadWarrantySerialCommand extends Command $db = $em->getConnection(); $ws_query_sql = 'SELECT id, file_serial, file_id, api_user, orig_file_serial FROM warranty_serial_queue - WHERE status = :status LIMIT 1'; + WHERE status = :status ORDER BY id LIMIT 1'; $ws_query_stmt = $db->prepare($ws_query_sql); $ws_query_stmt->bindValue('status', $status); @@ -84,12 +84,15 @@ class LoadWarrantySerialCommand extends Command $this->updateWarrantySerialQueue($id); } - // error_log(print_r($this->log_data, true)); - // load log data into db - $this->load_logger->logWarrantySerialLoadInfo($this->log_data); + if (count($output_info) > 0) + { + // error_log(print_r($this->log_data, true)); + // load log data into db + $this->load_logger->logWarrantySerialLoadInfo($this->log_data); - // send results back to third party - $this->sendResults($output_info); + // send results back to third party + $this->sendResults($output_info); + } return 0; } @@ -133,7 +136,7 @@ class LoadWarrantySerialCommand extends Command // valid entry, we parse and insert $serial = trim(strtoupper($row[0])); - // error_log('Processing ' . $serial); + error_log('Processing ' . $serial); $sku = trim(strtoupper($row[1])); $dispatch_status = trim($row[2]); @@ -445,7 +448,7 @@ class LoadWarrantySerialCommand extends Command curl_close($curl); // check result - // error_log('Result ' . $res); + error_log('Result ' . $res); } } From b4a6e233f1b020869234e79787380e9561a5e219 Mon Sep 17 00:00:00 2001 From: Korina Cordero Date: Thu, 29 Sep 2022 07:04:53 +0000 Subject: [PATCH 5/6] Delete csv file and directory after processing file. #708 --- src/Command/LoadWarrantySerialCommand.php | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/src/Command/LoadWarrantySerialCommand.php b/src/Command/LoadWarrantySerialCommand.php index 9abb600e..40c89620 100644 --- a/src/Command/LoadWarrantySerialCommand.php +++ b/src/Command/LoadWarrantySerialCommand.php @@ -7,6 +7,7 @@ use Symfony\Component\Console\Input\InputArgument; use Symfony\Component\Console\Input\InputInterface; use Symfony\Component\Console\Output\OutputInterface; use Symfony\Component\HttpKernel\KernelInterface; +use Symfony\Component\Filesystem\Filesystem; use Doctrine\ORM\EntityManagerInterface; @@ -32,15 +33,18 @@ class LoadWarrantySerialCommand extends Command protected $project_dir; protected $callback_url; protected $log_data; + protected $filesystem; public function __construct(EntityManagerInterface $em, WarrantySerialUploadLogger $upload_logger, - WarrantySerialLoadLogger $load_logger, KernelInterface $kernel, $callback_url) + WarrantySerialLoadLogger $load_logger, KernelInterface $kernel, $callback_url, + FileSystem $filesystem) { $this->em = $em; $this->upload_logger = $upload_logger; $this->load_logger = $load_logger; $this->project_dir = $kernel->getProjectDir(); $this->callback_url = $callback_url; + $this->filesystem = $filesystem; parent::__construct(); } @@ -81,7 +85,11 @@ class LoadWarrantySerialCommand extends Command $output_info[] = $this->processWarrantySerialFile($filename, $user_id, $file_id, $orig_filename); + // remove entry from queue table $this->updateWarrantySerialQueue($id); + + // delete the uploaded csv file and directory + $this->deleteDirectoryAndFile($file_id); } if (count($output_info) > 0) @@ -409,6 +417,13 @@ class LoadWarrantySerialCommand extends Command $db->exec('UNLOCK TABLES;'); } + protected function deleteDirectoryAndFile($filedir) + { + $csv_filedir = $this->project_dir . '/public/warranty_serial_uploads/' . $filedir; + + $this->filesystem->remove($csv_filedir); + } + protected function setOutputInfo($filename, $file_id, $has_error, $error_message, $entries, $orig_filename) { $info = [ From b3678168da2b71162dc0cbdac711fb2856ba871f Mon Sep 17 00:00:00 2001 From: Korina Cordero Date: Fri, 30 Sep 2022 04:57:54 +0000 Subject: [PATCH 6/6] Comment debug logs. #708 --- src/Command/LoadWarrantySerialCommand.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Command/LoadWarrantySerialCommand.php b/src/Command/LoadWarrantySerialCommand.php index 40c89620..f70504e7 100644 --- a/src/Command/LoadWarrantySerialCommand.php +++ b/src/Command/LoadWarrantySerialCommand.php @@ -144,7 +144,7 @@ class LoadWarrantySerialCommand extends Command // valid entry, we parse and insert $serial = trim(strtoupper($row[0])); - error_log('Processing ' . $serial); + // error_log('Processing ' . $serial); $sku = trim(strtoupper($row[1])); $dispatch_status = trim($row[2]);