From 645e8a63fcc0a058d3a163052f66455860a27a07 Mon Sep 17 00:00:00 2001 From: Korina Cordero Date: Tue, 26 Sep 2023 16:28:40 +0800 Subject: [PATCH] Add deletion. #762 --- src/Command/GetJobOrderArchiveDataCommand.php | 66 ++++++++++++------- 1 file changed, 43 insertions(+), 23 deletions(-) diff --git a/src/Command/GetJobOrderArchiveDataCommand.php b/src/Command/GetJobOrderArchiveDataCommand.php index d6ba4582..cc0fab18 100644 --- a/src/Command/GetJobOrderArchiveDataCommand.php +++ b/src/Command/GetJobOrderArchiveDataCommand.php @@ -39,6 +39,10 @@ class GetJobOrderArchiveDataCommand extends Command protected function execute(InputInterface $input, OutputInterface $output) { + $current_datetime = new DateTime('now'); + + error_log('Archive start time ' . $current_datetime->format('Y-m-d H:i')); + // get year to archive $year = $input->getArgument('year'); @@ -61,20 +65,26 @@ class GetJobOrderArchiveDataCommand extends Command $db = $this->em->getConnection(); - // TODO: improve performance. out of memory exception + // set the pdo connection to use unbuffered query so we don't run out of memory + // when processing the job order related tables + $db->getWrappedConnection()->setAttribute(PDO::MYSQL_ATTR_USE_BUFFERED_QUERY, false); + $query_sql = 'SELECT * FROM job_order WHERE YEAR(date_create) = :year - ORDER BY date_create - LIMIT 100000'; + ORDER BY date_create'; $query_stmt = $db->prepare($query_sql); $query_stmt->bindValue('year', $year, PDO::PARAM_STR); - $callback = ['App\Command\GetJobOrderArchiveDataCommand', 'getRelatedArchiveData']; + $callback = [$this, 'getRelatedArchiveData']; $this->getArchiveData($query_stmt, $callback, 'job_order', $year); + $current_datetime = new DateTime('now'); + + error_log('Archive end time ' . $current_datetime->format('Y-m-d H:i')); + return 0; } @@ -87,11 +97,11 @@ class GetJobOrderArchiveDataCommand extends Command // delete the related data files foreach ($related_tables as $tname) { - $this->deleteDataFiles($tname); + $this->deleteDataFiles($tname, $year); } // special since this is not directly related to JO but to invoice - $this->deleteDataFiles('invoice_item'); + $this->deleteDataFiles('invoice_item', $year); $archive_files = []; @@ -114,18 +124,21 @@ class GetJobOrderArchiveDataCommand extends Command { foreach ($related_tables as $table_name) { - if (($table_name == 'jo_event') || - ($table_name == 'invoice') || - ($table_name == 'ticket')) - { - $query_sql = 'SELECT * FROM ' . $table_name . ' WHERE job_order_id = :id'; - } - else - { - $query_sql = 'SELECT * FROM ' . $table_name . ' WHERE jo_id = :id'; + switch ($table_name) { + case 'jo_event': + case 'invoice': + case 'ticket': + $query_sql = 'SELECT * FROM ' . $table_name . ' WHERE job_order_id = :id'; + break; + case 'jo_rejection': + case 'rider_rating': + $query_sql = 'SELECT * FROM ' . $table_name . ' WHERE jo_id = :id'; + default: + break; } $db = $this->em->getConnection(); + $db->getWrappedConnection()->setAttribute(PDO::MYSQL_ATTR_USE_BUFFERED_QUERY, false); $query_stmt = $db->prepare($query_sql); $query_stmt->bindValue('id', $jo_id); @@ -142,7 +155,7 @@ class GetJobOrderArchiveDataCommand extends Command } // write the array into the file - $file = $this->createDataFileRelatedArchiveData($jo_data, $jo_tname, 'w'); + $file = $this->createDataFileRelatedArchiveData($jo_data, $jo_tname, $year, 'w'); if ($file != null) { @@ -155,7 +168,7 @@ class GetJobOrderArchiveDataCommand extends Command // need to get the list of invoice ids for deletion for invoice items $invoice_id_list = $this->getInvoiceIds($jo_id_list); - error_log(print_r($invoice_id_list, true)); + // error_log(print_r($invoice_id_list, true)); // at this point, all the job order and related data have been archived into the database $this->deleteData($jo_id_list, $invoice_id_list); @@ -199,7 +212,7 @@ class GetJobOrderArchiveDataCommand extends Command } // write the array into the file - $file = $this->createDataFileRelatedArchiveData($data, $table_name, 'a'); + $file = $this->createDataFileRelatedArchiveData($data, $table_name, $year, 'a'); if ($file != null) $files[$table_name] = $file; @@ -210,6 +223,7 @@ class GetJobOrderArchiveDataCommand extends Command protected function getInvoiceItemArchiveData($id, $year) { $db = $this->em->getConnection(); + $db->getWrappedConnection()->setAttribute(PDO::MYSQL_ATTR_USE_BUFFERED_QUERY, false); $query_sql = 'SELECT * FROM invoice_item WHERE invoice_id = :id'; @@ -238,23 +252,23 @@ class GetJobOrderArchiveDataCommand extends Command ]; } - $file = $this->createDataFileRelatedArchiveData($ii_data, 'invoice_item', 'a'); + $file = $this->createDataFileRelatedArchiveData($ii_data, 'invoice_item', $year, 'a'); return $file; } - protected function deleteDataFiles($tname) + protected function deleteDataFiles($tname, $year) { // cache directory $cache_dir = __DIR__ . '/../../var/cache'; - $file = $cache_dir . '/' . $tname . '_archive.tab'; + $file = $cache_dir . '/' . $tname . '_archive_' . $year .'.tab'; if (file_exists($file)) unlink($file); } - protected function createDataFileRelatedArchiveData($archive_data, $table_name, $option) + protected function createDataFileRelatedArchiveData($archive_data, $table_name, $year, $option) { if (isset($archive_data[$table_name])) { @@ -263,7 +277,7 @@ class GetJobOrderArchiveDataCommand extends Command // cache directory $cache_dir = __DIR__ . '/../../var/cache'; - $file = $cache_dir . '/' . $table_name . '_archive.tab'; + $file = $cache_dir . '/' . $table_name . '_archive_'. $year . '.tab'; // error_log('opening file for archive - ' . $file); $fp = fopen($file, $option); @@ -766,6 +780,12 @@ class GetJobOrderArchiveDataCommand extends Command $db = $this->em->getConnection(); // delete the invoice items first + $inv_ids = str_repeat('?,', count($invoice_id_list) - 1) . '?'; + + $ii_del_sql = 'DELETE FROM invoice_item WHERE invoice_id IN ($inv_ids)'; + $ii_stmt = $db->prepare($ii_del_sql); + + $ii_stmt->execute($invoice_id_list); } protected function getInvoiceIds($jo_id_list)