rgutierrez/resq
1
0
Fork 0
Code Issues 126 Pull requests 49 Projects Releases 4 Packages Wiki Activity Actions

Resolve "Data archiving" #1687

Open
korina.cordero wants to merge 139 commits from 762-data-archiving into 746-resq-2-0-final
pull from: 762-data-archiving
merge into: rgutierrez:746-resq-2-0-final
Conversation 30 Commits 139 Files changed 69 +43 -23
1 changed files with 43 additions and 23 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files
Showing only changes of commit 22985cde51 - Show all commits

66
src/Command/GetJobOrderArchiveDataCommand.php
View file

@ -39,6 +39,10 @@ class GetJobOrderArchiveDataCommand extends Command
protected function execute(InputInterface $input, OutputInterface $output)
{
$current_datetime = new DateTime('now');
error_log('Archive start time ' . $current_datetime->format('Y-m-d H:i'));
// get year to archive
$year = $input->getArgument('year');
@ -61,20 +65,26 @@ class GetJobOrderArchiveDataCommand extends Command
$db = $this->em->getConnection();
// TODO: improve performance. out of memory exception
// set the pdo connection to use unbuffered query so we don't run out of memory
// when processing the job order related tables
$db->getWrappedConnection()->setAttribute(PDO::MYSQL_ATTR_USE_BUFFERED_QUERY, false);
$query_sql = 'SELECT *
FROM job_order
WHERE YEAR(date_create) = :year
ORDER BY date_create
LIMIT 100000';
ORDER BY date_create';
$query_stmt = $db->prepare($query_sql);
$query_stmt->bindValue('year', $year, PDO::PARAM_STR);
$callback = ['App\Command\GetJobOrderArchiveDataCommand', 'getRelatedArchiveData'];
$callback = [$this, 'getRelatedArchiveData'];
$this->getArchiveData($query_stmt, $callback, 'job_order', $year);
$current_datetime = new DateTime('now');
error_log('Archive end time ' . $current_datetime->format('Y-m-d H:i'));
return 0;
}
@ -87,11 +97,11 @@ class GetJobOrderArchiveDataCommand extends Command
// delete the related data files
foreach ($related_tables as $tname)
{
$this->deleteDataFiles($tname);
$this->deleteDataFiles($tname, $year);
}
// special since this is not directly related to JO but to invoice
$this->deleteDataFiles('invoice_item');
$this->deleteDataFiles('invoice_item', $year);
$archive_files = [];
@ -114,18 +124,21 @@ class GetJobOrderArchiveDataCommand extends Command
{
foreach ($related_tables as $table_name)
{
if (($table_name == 'jo_event') ||
($table_name == 'invoice') ||
($table_name == 'ticket'))
{
$query_sql = 'SELECT * FROM ' . $table_name . ' WHERE job_order_id = :id';
}
else
{
$query_sql = 'SELECT * FROM ' . $table_name . ' WHERE jo_id = :id';
switch ($table_name) {
case 'jo_event':
case 'invoice':
case 'ticket':
$query_sql = 'SELECT * FROM ' . $table_name . ' WHERE job_order_id = :id';
break;
case 'jo_rejection':
case 'rider_rating':
$query_sql = 'SELECT * FROM ' . $table_name . ' WHERE jo_id = :id';
default:
break;
}
$db = $this->em->getConnection();
$db->getWrappedConnection()->setAttribute(PDO::MYSQL_ATTR_USE_BUFFERED_QUERY, false);
$query_stmt = $db->prepare($query_sql);
$query_stmt->bindValue('id', $jo_id);
@ -142,7 +155,7 @@ class GetJobOrderArchiveDataCommand extends Command
}
// write the array into the file
$file = $this->createDataFileRelatedArchiveData($jo_data, $jo_tname, 'w');
$file = $this->createDataFileRelatedArchiveData($jo_data, $jo_tname, $year, 'w');
if ($file != null)
{
@ -155,7 +168,7 @@ class GetJobOrderArchiveDataCommand extends Command
// need to get the list of invoice ids for deletion for invoice items
$invoice_id_list = $this->getInvoiceIds($jo_id_list);
error_log(print_r($invoice_id_list, true));
// error_log(print_r($invoice_id_list, true));
// at this point, all the job order and related data have been archived into the database
$this->deleteData($jo_id_list, $invoice_id_list);
@ -199,7 +212,7 @@ class GetJobOrderArchiveDataCommand extends Command
}
// write the array into the file
$file = $this->createDataFileRelatedArchiveData($data, $table_name, 'a');
$file = $this->createDataFileRelatedArchiveData($data, $table_name, $year, 'a');
if ($file != null)
$files[$table_name] = $file;
@ -210,6 +223,7 @@ class GetJobOrderArchiveDataCommand extends Command
protected function getInvoiceItemArchiveData($id, $year)
{
$db = $this->em->getConnection();
$db->getWrappedConnection()->setAttribute(PDO::MYSQL_ATTR_USE_BUFFERED_QUERY, false);
$query_sql = 'SELECT * FROM invoice_item WHERE invoice_id = :id';
@ -238,23 +252,23 @@ class GetJobOrderArchiveDataCommand extends Command
];
}
$file = $this->createDataFileRelatedArchiveData($ii_data, 'invoice_item', 'a');
$file = $this->createDataFileRelatedArchiveData($ii_data, 'invoice_item', $year, 'a');
return $file;
}
protected function deleteDataFiles($tname)
protected function deleteDataFiles($tname, $year)
{
// cache directory
$cache_dir = __DIR__ . '/../../var/cache';
$file = $cache_dir . '/' . $tname . '_archive.tab';
$file = $cache_dir . '/' . $tname . '_archive_' . $year .'.tab';
if (file_exists($file))
unlink($file);
}
protected function createDataFileRelatedArchiveData($archive_data, $table_name, $option)
protected function createDataFileRelatedArchiveData($archive_data, $table_name, $year, $option)
{
if (isset($archive_data[$table_name]))
{
@ -263,7 +277,7 @@ class GetJobOrderArchiveDataCommand extends Command
// cache directory
$cache_dir = __DIR__ . '/../../var/cache';
$file = $cache_dir . '/' . $table_name . '_archive.tab';
$file = $cache_dir . '/' . $table_name . '_archive_'. $year . '.tab';
// error_log('opening file for archive - ' . $file);
$fp = fopen($file, $option);
@ -766,6 +780,12 @@ class GetJobOrderArchiveDataCommand extends Command
$db = $this->em->getConnection();
// delete the invoice items first
$inv_ids = str_repeat('?,', count($invoice_id_list) - 1) . '?';
$ii_del_sql = 'DELETE FROM invoice_item WHERE invoice_id IN ($inv_ids)';
$ii_stmt = $db->prepare($ii_del_sql);
$ii_stmt->execute($invoice_id_list);
}
protected function getInvoiceIds($jo_id_list)