From 22d4201097af0295ceb6c30705899c49d4d679a5 Mon Sep 17 00:00:00 2001 From: Korina Cordero Date: Thu, 7 Sep 2023 16:29:27 +0800 Subject: [PATCH 01/27] Add command to archive job order data. #762 --- src/Command/GetJobOrderArchiveDataCommand.php | 344 ++++++++++++++++++ 1 file changed, 344 insertions(+) create mode 100644 src/Command/GetJobOrderArchiveDataCommand.php diff --git a/src/Command/GetJobOrderArchiveDataCommand.php b/src/Command/GetJobOrderArchiveDataCommand.php new file mode 100644 index 00000000..ec5266e6 --- /dev/null +++ b/src/Command/GetJobOrderArchiveDataCommand.php @@ -0,0 +1,344 @@ +em = $em; + $this->project_dir = $kernel->getProjectDir(); + $this->filesystem = $filesystem; + + parent::__construct(); + } + + protected function configure() + { + $this->setName('joborder:archive') + ->setDescription('Get job order data to archive.') + ->setHelp('Get job order data to archive.') + ->addArgument('cutoff_date', InputArgument::REQUIRED, 'cutoff_date') + ->addArgument('period', InputArgument::REQUIRED, 'period'); + } + + protected function execute(InputInterface $input, OutputInterface $output) + { + // get table to back up + $period = $input->getArgument('period'); + $cutoff_date = $input->getArgument('cutoff_date'); + + $em = $this->em; + + $db = $em->getConnection(); + + // create the archive table + $archive_table_name = $this->createArchiveTable($cutoff_date, $period); + + // sql query to retrieve the rows or entries for backup + $query_sql = 'SELECT id, + customer_id, + cvehicle_id, + rider_id, + date_create, + date_schedule, + date_fulfill, + coordinates, + flag_advance, + service_type, + source, + date_cancel, + status, + delivery_instructions, + delivery_address, + create_user_id, + assign_user_id, + date_assign, + warranty_class, + process_user_id, + hub_id, + cancel_reason, + ref_jo_id, + tier1_notes, + tier2_notes, + mode_of_payment, + or_name, + landmark, + promo_detail, + or_num, + trade_in_type, + flag_rider_rating, + flag_coolant, + facilitated_hub_id, + facilitated_type, + coord_long, + coord_lat, + priority, + meta, + status_autoassign, + first_name, + last_name, + plate_number, + phone_mobile, + no_trade_in_reason, + will_wait, + reason_not_waiting, + not_waiting_notes, + delivery_status, + emergency_type_id, + ownership_type_id, + cust_location_id, + source_of_awareness, + remarks, + initial_concern, + initial_concern_notes, + gender, + caller_classification, + inventory_count + FROM job_order + WHERE DATE_CREATE < DATE_SUB(:cutoff_date, INTERVAL ' . $period . ' YEAR)'; + + $query_stmt = $db->prepare($query_sql); + $query_stmt->bindValue('cutoff_date', $cutoff_date, PDO::PARAM_STR); + + $results = $query_stmt->executeQuery(); + + $backup_data = []; + + while ($row = $results->fetchAssociative()) + { + $backup_data[] = $this->createBackupData($row); + } + + // create the load file for the backup data + $this->createLoadDataFileForBackupData($backup_data); + + return 0; + } + + protected function createArchiveTable($str_date, $period) + { + // form the archive table name _archive_ + $modifier = '-' . $period. 'year'; + + // convert the date string into DateTime + $date = new DateTime($str_date); + $year = $date->modify($modifier)->format('Y'); + + $archive_table_name = 'job_order_archive_' . $year; + + // create the table if it doesn't exist + $db = $this->em->getConnection(); + + $create_sql = 'CREATE TABLE IF NOT EXISTS `' . $archive_table_name . '` ( + `id` int(11) NOT NULL, + `customer_id` int(11) DEFAULT NULL, + `cvehicle_id` int(11) DEFAULT NULL, + `rider_id` int(11) DEFAULT NULL, + `date_create` datetime NOT NULL, + `date_schedule` datetime NOT NULL, + `date_fulfill` datetime DEFAULT NULL, + `coordinates` point NOT NULL COMMENT \'(DC2Type:point)\', + `flag_advance` tinyint(1) NOT NULL, + `service_type` varchar(25) COLLATE utf8_unicode_ci NOT NULL, + `source` varchar(30) COLLATE utf8_unicode_ci NOT NULL, + `date_cancel` datetime DEFAULT NULL, + `status` varchar(15) COLLATE utf8_unicode_ci NOT NULL, + `delivery_instructions` longtext COLLATE utf8_unicode_ci DEFAULT NULL, + `delivery_address` longtext COLLATE utf8_unicode_ci NOT NULL, + `create_user_id` int(11) DEFAULT NULL, + `assign_user_id` int(11) DEFAULT NULL, + `date_assign` datetime DEFAULT NULL, + `warranty_class` varchar(25) COLLATE utf8_unicode_ci NOT NULL, + `process_user_id` int(11) DEFAULT NULL, + `hub_id` int(11) DEFAULT NULL, + `cancel_reason` varchar(200) COLLATE utf8_unicode_ci DEFAULT NULL, + `ref_jo_id` int(11) DEFAULT NULL, + `tier1_notes` longtext COLLATE utf8_unicode_ci DEFAULT NULL, + `tier2_notes` longtext COLLATE utf8_unicode_ci DEFAULT NULL, + `mode_of_payment` varchar(50) COLLATE utf8_unicode_ci NOT NULL, + `or_name` varchar(80) COLLATE utf8_unicode_ci NOT NULL, + `landmark` longtext COLLATE utf8_unicode_ci NOT NULL, + `promo_detail` varchar(80) COLLATE utf8_unicode_ci NOT NULL, + `or_num` varchar(80) COLLATE utf8_unicode_ci DEFAULT NULL, + `trade_in_type` varchar(25) COLLATE utf8_unicode_ci DEFAULT NULL, + `flag_rider_rating` tinyint(1) DEFAULT NULL, + `flag_coolant` tinyint(1) NOT NULL, + `facilitated_hub_id` int(11) DEFAULT NULL, + `facilitated_type` varchar(8) COLLATE utf8_unicode_ci DEFAULT NULL, + `coord_long` decimal(11,8) NOT NULL, + `coord_lat` decimal(11,8) NOT NULL, + `priority` int(11) NOT NULL DEFAULT 0, + `meta` longtext CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL, + `status_autoassign` varchar(30) COLLATE utf8_unicode_ci DEFAULT NULL, + `first_name` varchar(80) COLLATE utf8_unicode_ci NOT NULL, + `last_name` varchar(80) COLLATE utf8_unicode_ci NOT NULL, + `plate_number` varchar(100) COLLATE utf8_unicode_ci NOT NULL, + `phone_mobile` varchar(30) COLLATE utf8_unicode_ci NOT NULL, + `no_trade_in_reason` varchar(80) COLLATE utf8_unicode_ci DEFAULT NULL, + `will_wait` varchar(30) COLLATE utf8_unicode_ci NOT NULL, + `reason_not_waiting` varchar(80) COLLATE utf8_unicode_ci DEFAULT NULL, + `not_waiting_notes` longtext COLLATE utf8_unicode_ci DEFAULT NULL, + `delivery_status` varchar(30) COLLATE utf8_unicode_ci DEFAULT NULL, + `emergency_type_id` int(11) DEFAULT NULL, + `ownership_type_id` int(11) DEFAULT NULL, + `cust_location_id` int(11) DEFAULT NULL, + `source_of_awareness` varchar(80) COLLATE utf8_unicode_ci DEFAULT NULL, + `remarks` longtext COLLATE utf8_unicode_ci DEFAULT NULL, + `initial_concern` varchar(80) COLLATE utf8_unicode_ci DEFAULT NULL, + `initial_concern_notes` longtext COLLATE utf8_unicode_ci DEFAULT NULL, + `gender` varchar(80) COLLATE utf8_unicode_ci DEFAULT NULL, + `caller_classification` varchar(80) COLLATE utf8_unicode_ci DEFAULT NULL, + `inventory_count` smallint(6) NOT NULL, PRIMARY KEY (`id`))'; + + $create_stmt = $db->prepare($create_sql); + + $result = $create_stmt->execute(); + + return $archive_table_name; + } + + protected function createBackupData($row) + { + // get job order data + // check for nulls. check the ff fields since these can be null: date_fulfill, date_cancel, date_assign, create_user_id, + // assign_user_id, proces_user_id, hub_id, rider_id, cancel_reason, ref_jo_id, or_num, trade_in_type, + // flag_rider_rating, facilitated_type, facilitated_hub_id, status_autoassign, reason_not_waiting, + // not_waiting_notes, no_trade_in_reason, delivery_status, source_of_awareness, remarks, initial_concern, + // initial_concern_notes, gender, caller_classifications, emergency_type_id, ownership_type_id, cust_location_id + + $id = $row['id']; + $cust_id = $row['customer_id']; + $cv_id = $row['cvehicle_id']; + + $rider_id = '\N'; + if ($row['rider_id'] != NULL) + $rider_id = $row]'rider_id']; + + $date_create = $row['date_create']; + $date_schedule = $row['date_schedule']; + + $date_fulfill = '\N'; + if ($row['date_fulfill'] != NULL) + $date_fulfill = $row['date_schedule']; + + $coordinates = $row['coordinates']; + $flag_advance = $row['flag_advance']; + $service_type = $row['service_type']; + $source = $row['source']; + + $date_cancel = '\N'; + if ($row['date_cancel'] != NULL) + $date_cancel = $row['date_cancel']; + + $status = $row['status']; + $del_instructions = $row['delivery_instructions']; + $del_address = $row['delivery_address']; + + $create_user_id = '\N'; + if ($row['create_user_id'] != NULL) + $create_user_id = $row['create_user_id']; + + $assign_user_id = '\N'; + if ($row['assign_user_id'] != NULL) + $assign_user_id = $row['assign_user_id']; + + $date_assign = '\N'; + if ($row['date_assign'] != NULL) + $date_assign = $row['date_assign']; + + $warr_class = $row['warranty_class']; + + $process_user_id = '\N'; + if ($row['process_user_id'] != NULL) + $process_user_id = $row['process_user_id']; + + $hub_id = '\N'; + if ($row['hub_id'] != NULL) + $hub_id = $row['hub_id']; + + $cancel_reason = '\N'; + if ($row['cancel_reason'] != NULL) + $cancel_reason = $row['cancel_reason']; + + $ref_jo_id = '\N'; + if ($row['ref_jo_id'] != NULL) + $ref_jo_id = $row['ref_jo_id']; + + $tier1_notes = $row['tier1_notes']; + $tier2_notes = $row['tier2_notes']; + $mode_of_payment = $row['mode_of_payment']; + $or_name = $row['or_name']; + $landmark = $row['landmark']; + $promo_details = $row['promo_detail']; + + $or_num = '\N'; + if ($row['or_num'] != NULL) + $or_num = $row['or_num']; + + $trade_in_type = '\N'; + if ($row['trade_in_type'] != NULL) + $trade_in_type = $row['trade_in_type']; + + $flag_rider_rating = '\N'; + if ($row['flag_rider_rating'] != NULL) + $flag_rider_rating = $row['flag_rider_rating']; + + $flag_coolant = $row['flag_coolant']; + + $fac_hub_id = '\N'; + if ($row['facilitated_hub_id'] != NULL) + $fac_hub_id = $row['facilitated_hub_id']; + + $fac_type = '\N'; + if ($row['facilitated_type'] != NULL) + $fac_type = $row['facilitated_type']; + + $coord_long = $row['coord_long']; + $coord_lat = $row['coord_lat']; + $priority = $row['priority']; + $meta = $row['meta']; + $status_autoassign = $row['status_autoassign']; + $first_name = $row['first_name']; + $last_name = $row['last_name']; + $plate_number = $row['plate_number']; + $phone_mobile = $row['phone_mobile']; + $no_trade_in_reason = $row[' no_trade_in_reason']; + $will_wait = $row['will_wait']; + $reason_not_waiting = $row['reason_not_waiting']; + $not_waiting_notes = $row['not_waiting_notes']; + $del_status = $row['delivery_status']; + $emergency_type_id = $row['emergency_type_id']; + $owner_type_id = $row['ownership_type_id']; + $cust_location_id = $row['cust_location_id']; + $source_of_awareness = $row['source_of_awareness']; + $remarks = $row['remarks']; + $initial_concern = $row['initial_concern']; + $initial_concern_notes = $row['initial_concern_notes']; + $gender = $row['gender']; + $caller_class = $row['caller_classification']; + $inv_count = $row['inventory_count']; + + // check for nulls. check the ff fields since these can be null: date_fulfill, date_cancel, date_assign, create_user_id, + // assign_user_id, proces_user_id, hub_id, rider_id, cancel_reason, ref_jo_id, or_num, trade_in_type, + // flag_rider_rating, facilitated_type, facilitated_hub_id, status_autoassign, reason_not_waiting, + // not_waiting_notes, no_trade_in_reason, delivery_status, source_of_awareness, remarks, initial_concern, + // initial_concern_notes, gender, caller_classifications, emergency_type_id, ownership_type_id, cust_location_id + } +} From c318c471ff5c59d87ea285009f2405a1ab313b8d Mon Sep 17 00:00:00 2001 From: Korina Cordero Date: Thu, 7 Sep 2023 18:01:13 +0800 Subject: [PATCH 02/27] Add checking for nulls when creating the data for backup. #762 --- src/Command/GetJobOrderArchiveDataCommand.php | 89 ++++++++++++++----- 1 file changed, 69 insertions(+), 20 deletions(-) diff --git a/src/Command/GetJobOrderArchiveDataCommand.php b/src/Command/GetJobOrderArchiveDataCommand.php index ec5266e6..704c509f 100644 --- a/src/Command/GetJobOrderArchiveDataCommand.php +++ b/src/Command/GetJobOrderArchiveDataCommand.php @@ -311,34 +311,83 @@ class GetJobOrderArchiveDataCommand extends Command $fac_type = $row['facilitated_type']; $coord_long = $row['coord_long']; - $coord_lat = $row['coord_lat']; $priority = $row['priority']; $meta = $row['meta']; - $status_autoassign = $row['status_autoassign']; + + $status_autoassign = '\N'; + if ($row['status_autoassign'] != NULL) + $status_autoassign = $row['status_autoassign']; + $first_name = $row['first_name']; $last_name = $row['last_name']; $plate_number = $row['plate_number']; $phone_mobile = $row['phone_mobile']; - $no_trade_in_reason = $row[' no_trade_in_reason']; + + $no_trade_in_reason = '\N'; + if ($row['no_trade_in_reason'] != NULL) + $no_trade_in_reason = $row[' no_trade_in_reason']; + $will_wait = $row['will_wait']; - $reason_not_waiting = $row['reason_not_waiting']; - $not_waiting_notes = $row['not_waiting_notes']; - $del_status = $row['delivery_status']; - $emergency_type_id = $row['emergency_type_id']; - $owner_type_id = $row['ownership_type_id']; - $cust_location_id = $row['cust_location_id']; - $source_of_awareness = $row['source_of_awareness']; - $remarks = $row['remarks']; - $initial_concern = $row['initial_concern']; - $initial_concern_notes = $row['initial_concern_notes']; - $gender = $row['gender']; - $caller_class = $row['caller_classification']; + + $reason_not_waiting = '\N'; + if ($row['reason_not_waiting'] != NULL) + $reason_not_waiting = $row['reason_not_waiting']; + + $not_waiting_notes = '\N'; + if ($row['not_waiting_notes'] != NULL) + $not_waiting_notes = $row['not_waiting_notes']; + + $del_status = '\N'; + if ($row['delivery_status'] != NULL) + $del_status = $row['delivery_status']; + + $emergency_type_id = '\N'; + if ($row['emergency_type_id'] != NULL) + $emergency_type_id = $row['emergency_type_id']; + + $owner_type_id = '\N'; + if ($row['ownership_type_id'] != NULL) + $owner_type_id = $row['ownership_type_id']; + + $cust_location_id = '\N'; + if ($row['cust_location_id'] != NULL) + $cust_location_id = $row['cust_location_id']; + + $source_of_awareness = '\N'; + if ($row['source_of_awareness'] != NULL) + $source_of_awareness = $row['source_of_awareness']; + + $remarks = '\N'; + if ($row['remarks'] != NULL) + $remarks = $row['remarks']; + + $initial_concern = '\N'; + if ($row['initial_concern'] != NULL) + $initial_concern = $row['initial_concern']; + + $initial_concern_notes = '\N'; + if ($row['initial_concern_notes'] != NULL) + $initial_concern_notes = $row['initial_concern_notes']; + + $gender = '\N'; + if ($row['gender'] != NULL) + $gender = $row['gender']; + + $caller_class = '\N'; + if ($row['caller_classification'] != NULL) + $caller_class = $row['caller_classification']; + $inv_count = $row['inventory_count']; - // check for nulls. check the ff fields since these can be null: date_fulfill, date_cancel, date_assign, create_user_id, - // assign_user_id, proces_user_id, hub_id, rider_id, cancel_reason, ref_jo_id, or_num, trade_in_type, - // flag_rider_rating, facilitated_type, facilitated_hub_id, status_autoassign, reason_not_waiting, - // not_waiting_notes, no_trade_in_reason, delivery_status, source_of_awareness, remarks, initial_concern, - // initial_concern_notes, gender, caller_classifications, emergency_type_id, ownership_type_id, cust_location_id + // create the array for the file + $data = [ + $id, + $cust_id, + // TODO: finish writing to array + ]; + + return $data; } + + // TODO: write the load data file function } From 4c0460bdb5b5a97fa8d3eee82b4306eb70ef7104 Mon Sep 17 00:00:00 2001 From: Korina Cordero Date: Fri, 8 Sep 2023 15:27:19 +0800 Subject: [PATCH 03/27] Add loading of data into database. #762 --- src/Command/GetJobOrderArchiveDataCommand.php | 115 +++++++++++++++++- 1 file changed, 110 insertions(+), 5 deletions(-) diff --git a/src/Command/GetJobOrderArchiveDataCommand.php b/src/Command/GetJobOrderArchiveDataCommand.php index 704c509f..47c3e174 100644 --- a/src/Command/GetJobOrderArchiveDataCommand.php +++ b/src/Command/GetJobOrderArchiveDataCommand.php @@ -127,7 +127,7 @@ class GetJobOrderArchiveDataCommand extends Command } // create the load file for the backup data - $this->createLoadDataFileForBackupData($backup_data); + $this->createLoadDataFileForBackupData($backup_data, $archive_table_name); return 0; } @@ -229,7 +229,7 @@ class GetJobOrderArchiveDataCommand extends Command $rider_id = '\N'; if ($row['rider_id'] != NULL) - $rider_id = $row]'rider_id']; + $rider_id = $row['rider_id']; $date_create = $row['date_create']; $date_schedule = $row['date_schedule']; @@ -311,6 +311,7 @@ class GetJobOrderArchiveDataCommand extends Command $fac_type = $row['facilitated_type']; $coord_long = $row['coord_long']; + $coord_lat = $row['coord_lat']; $priority = $row['priority']; $meta = $row['meta']; @@ -325,7 +326,7 @@ class GetJobOrderArchiveDataCommand extends Command $no_trade_in_reason = '\N'; if ($row['no_trade_in_reason'] != NULL) - $no_trade_in_reason = $row[' no_trade_in_reason']; + $no_trade_in_reason = $row['no_trade_in_reason']; $will_wait = $row['will_wait']; @@ -383,11 +384,115 @@ class GetJobOrderArchiveDataCommand extends Command $data = [ $id, $cust_id, - // TODO: finish writing to array + $cv_id, + $rider_id, + $date_create, + $date_schedule, + $date_fulfill, + $coordinates, + $flag_advance, + $service_type, + $source, + $date_cancel, + $status, + $del_instructions, + $del_address, + $create_user_id, + $assign_user_id, + $date_assign, + $warr_class, + $process_user_id, + $hub_id, + $cancel_reason, + $ref_jo_id, + $tier1_notes, + $tier2_notes, + $mode_of_payment, + $or_name, + $landmark, + $promo_details, + $or_num, + $trade_in_type, + $flag_rider_rating, + $flag_coolant, + $fac_hub_id, + $fac_type, + $coord_long, + $coord_lat, + $priority, + $meta, + $status_autoassign, + $first_name, + $last_name, + $plate_number, + $phone_mobile, + $no_trade_in_reason, + $will_wait, + $reason_not_waiting, + $not_waiting_notes, + $del_status, + $emergency_type_id, + $owner_type_id, + $cust_location_id, + $source_of_awareness, + $remarks, + $initial_concern, + $initial_concern_notes, + $gender, + $caller_class, + $inv_count ]; return $data; } - // TODO: write the load data file function + protected function createLoadDataFileForBackupData($backup_data, $table_name) + { + // cache directory + $cache_dir = __DIR__ . '/../../var/cache'; + + $file = $cache_dir . '/jo_archive.tab'; + error_log('opening file for jo archive - ' . $file); + + $fp = fopen($file, 'w'); + if ($fp === false) + { + error_log('could not open file for load data infile - ' . $file); + } + else + { + foreach ($backup_data as $key => $data) + { + $line = implode('|', $data) . "\r\n"; + fwrite($fp, $line); + } + } + + fclose($fp); + + $conn = $this->em->getConnection(); + $stmt = $conn->prepare('LOAD DATA LOCAL INFILE \'' . $file . '\' INTO TABLE ' . $table_name . ' + FIELDS TERMINATED BY \'|\' + LINES TERMINATED BY \'\\r\\n\' + (id, customer_id, cvehicle_id, rider_id, date_create, + date_schedule, date_fulfill, coordinates, flag_advance, service_type, + source, date_cancel, status, delivery_instructions, delivery_address, + create_user_id, assign_user_id, date_assign, warranty_class, process_user_id, + hub_id, cancel_reason, ref_jo_id, tier1_notes, tier2_notes, + mode_of_payment, or_name, landmark, promo_detail, or_num, + trade_in_type, flag_rider_rating, flag_coolant, facilitated_hub_id, facilitated_type, + coord_long, coord_lat, priority, meta, status_autoassign, + first_name, last_name, plate_number, phone_mobile, no_trade_in_reason, + will_wait, reason_not_waiting, not_waiting_notes, delivery_status, emergency_type_id, + ownership_type_id, cust_location_id, source_of_awareness, remarks, initial_concern, + initial_concern_notes, gender, caller_classification, inventory_count)'); + + $result = $stmt->execute(); + + if (!$result) + error_log('Failed loading data.'); + + // TODO: delete file? + } + } From 4eca65cc59bbd9ac4f7075cf83b96d7a70a42568 Mon Sep 17 00:00:00 2001 From: Korina Cordero Date: Fri, 8 Sep 2023 17:24:43 +0800 Subject: [PATCH 04/27] Add saving of data to archive table. #762 --- src/Command/GetJobOrderArchiveDataCommand.php | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/src/Command/GetJobOrderArchiveDataCommand.php b/src/Command/GetJobOrderArchiveDataCommand.php index 47c3e174..7e9f969e 100644 --- a/src/Command/GetJobOrderArchiveDataCommand.php +++ b/src/Command/GetJobOrderArchiveDataCommand.php @@ -126,6 +126,8 @@ class GetJobOrderArchiveDataCommand extends Command $backup_data[] = $this->createBackupData($row); } + error_log('count ' . count($backup_data)); + // create the load file for the backup data $this->createLoadDataFileForBackupData($backup_data, $archive_table_name); @@ -238,7 +240,6 @@ class GetJobOrderArchiveDataCommand extends Command if ($row['date_fulfill'] != NULL) $date_fulfill = $row['date_schedule']; - $coordinates = $row['coordinates']; $flag_advance = $row['flag_advance']; $service_type = $row['service_type']; $source = $row['source']; @@ -248,6 +249,8 @@ class GetJobOrderArchiveDataCommand extends Command $date_cancel = $row['date_cancel']; $status = $row['status']; + + // TODO: might need to clean delivery address and delivery instructions $del_instructions = $row['delivery_instructions']; $del_address = $row['delivery_address']; @@ -312,6 +315,10 @@ class GetJobOrderArchiveDataCommand extends Command $coord_long = $row['coord_long']; $coord_lat = $row['coord_lat']; + + // coordinates needs special handling since it's a spatial column + $coordinates = 'POINT(' . $coord_lat . ' ' . $coord_long .')'; + $priority = $row['priority']; $meta = $row['meta']; @@ -471,11 +478,11 @@ class GetJobOrderArchiveDataCommand extends Command fclose($fp); $conn = $this->em->getConnection(); - $stmt = $conn->prepare('LOAD DATA LOCAL INFILE \'' . $file . '\' INTO TABLE ' . $table_name . ' + $stmt = $conn->prepare('LOAD DATA LOCAL INFILE \'' . $file . '\' INTO TABLE ' . $table_name . ' FIELDS TERMINATED BY \'|\' LINES TERMINATED BY \'\\r\\n\' (id, customer_id, cvehicle_id, rider_id, date_create, - date_schedule, date_fulfill, coordinates, flag_advance, service_type, + date_schedule, date_fulfill, @coordinates, flag_advance, service_type, source, date_cancel, status, delivery_instructions, delivery_address, create_user_id, assign_user_id, date_assign, warranty_class, process_user_id, hub_id, cancel_reason, ref_jo_id, tier1_notes, tier2_notes, @@ -485,7 +492,9 @@ class GetJobOrderArchiveDataCommand extends Command first_name, last_name, plate_number, phone_mobile, no_trade_in_reason, will_wait, reason_not_waiting, not_waiting_notes, delivery_status, emergency_type_id, ownership_type_id, cust_location_id, source_of_awareness, remarks, initial_concern, - initial_concern_notes, gender, caller_classification, inventory_count)'); + initial_concern_notes, gender, caller_classification, inventory_count) + SET coordinates=ST_GeomFromText(@coordinates)' + ); $result = $stmt->execute(); From 7e3c392c03b6c8b0e464dacc3a823cf081bbacad Mon Sep 17 00:00:00 2001 From: Korina Cordero Date: Mon, 11 Sep 2023 15:10:08 +0800 Subject: [PATCH 05/27] Fix issues found when saving archive data. #762 --- src/Command/GetJobOrderArchiveDataCommand.php | 31 ++++++++++++++----- 1 file changed, 23 insertions(+), 8 deletions(-) diff --git a/src/Command/GetJobOrderArchiveDataCommand.php b/src/Command/GetJobOrderArchiveDataCommand.php index 7e9f969e..4a42849f 100644 --- a/src/Command/GetJobOrderArchiveDataCommand.php +++ b/src/Command/GetJobOrderArchiveDataCommand.php @@ -44,6 +44,8 @@ class GetJobOrderArchiveDataCommand extends Command $period = $input->getArgument('period'); $cutoff_date = $input->getArgument('cutoff_date'); + $cutoff_date_time = $cutoff_date . ' 23:59:59'; + $em = $this->em; $db = $em->getConnection(); @@ -112,7 +114,8 @@ class GetJobOrderArchiveDataCommand extends Command caller_classification, inventory_count FROM job_order - WHERE DATE_CREATE < DATE_SUB(:cutoff_date, INTERVAL ' . $period . ' YEAR)'; + WHERE DATE_CREATE <= DATE_SUB(:cutoff_date, INTERVAL ' . $period . ' YEAR) + ORDER BY date_create'; $query_stmt = $db->prepare($query_sql); $query_stmt->bindValue('cutoff_date', $cutoff_date, PDO::PARAM_STR); @@ -250,9 +253,8 @@ class GetJobOrderArchiveDataCommand extends Command $status = $row['status']; - // TODO: might need to clean delivery address and delivery instructions - $del_instructions = $row['delivery_instructions']; - $del_address = $row['delivery_address']; + $del_instructions = $this->cleanData($row['delivery_instructions']); + $del_address = $this->cleanData($row['delivery_address']); $create_user_id = '\N'; if ($row['create_user_id'] != NULL) @@ -284,11 +286,13 @@ class GetJobOrderArchiveDataCommand extends Command if ($row['ref_jo_id'] != NULL) $ref_jo_id = $row['ref_jo_id']; - $tier1_notes = $row['tier1_notes']; - $tier2_notes = $row['tier2_notes']; + $tier1_notes = $this->cleanData($row['tier1_notes']); + $tier2_notes = $this->cleanData($row['tier2_notes']); + $mode_of_payment = $row['mode_of_payment']; $or_name = $row['or_name']; - $landmark = $row['landmark']; + + $landmark = $this->cleanData($row['landmark']); $promo_details = $row['promo_detail']; $or_num = '\N'; @@ -367,7 +371,7 @@ class GetJobOrderArchiveDataCommand extends Command $remarks = '\N'; if ($row['remarks'] != NULL) - $remarks = $row['remarks']; + $remarks = $this->cleanData(row['remarks']); $initial_concern = '\N'; if ($row['initial_concern'] != NULL) @@ -504,4 +508,15 @@ class GetJobOrderArchiveDataCommand extends Command // TODO: delete file? } + protected function cleanData($text) + { + $clean_text = ''; + + // replace the new lines with whitespace + $clean_text = preg_replace("/[\n\r]/", ' ', $text); + + return $clean_text; + + } + } From 986181c7807585e8f67b0c8195fcc76ecbb57599 Mon Sep 17 00:00:00 2001 From: Korina Cordero Date: Tue, 12 Sep 2023 11:35:31 +0800 Subject: [PATCH 06/27] Modify code according to code review. #762 --- src/Command/GetJobOrderArchiveDataCommand.php | 229 ++++-------------- 1 file changed, 44 insertions(+), 185 deletions(-) diff --git a/src/Command/GetJobOrderArchiveDataCommand.php b/src/Command/GetJobOrderArchiveDataCommand.php index 4a42849f..2e8e6a41 100644 --- a/src/Command/GetJobOrderArchiveDataCommand.php +++ b/src/Command/GetJobOrderArchiveDataCommand.php @@ -34,91 +34,32 @@ class GetJobOrderArchiveDataCommand extends Command $this->setName('joborder:archive') ->setDescription('Get job order data to archive.') ->setHelp('Get job order data to archive.') - ->addArgument('cutoff_date', InputArgument::REQUIRED, 'cutoff_date') - ->addArgument('period', InputArgument::REQUIRED, 'period'); + ->addArgument('year', InputArgument::REQUIRED, 'year'); } protected function execute(InputInterface $input, OutputInterface $output) { // get table to back up - $period = $input->getArgument('period'); - $cutoff_date = $input->getArgument('cutoff_date'); - - $cutoff_date_time = $cutoff_date . ' 23:59:59'; + $year = $input->getArgument('year'); $em = $this->em; $db = $em->getConnection(); // create the archive table - $archive_table_name = $this->createArchiveTable($cutoff_date, $period); + $archive_table_name = $this->createArchiveTable($year); + // TODO: move the creation of the query_sql to a function or + // set this as a preset or a variable or something and load specific query + // according to what table is to be archived. // sql query to retrieve the rows or entries for backup - $query_sql = 'SELECT id, - customer_id, - cvehicle_id, - rider_id, - date_create, - date_schedule, - date_fulfill, - coordinates, - flag_advance, - service_type, - source, - date_cancel, - status, - delivery_instructions, - delivery_address, - create_user_id, - assign_user_id, - date_assign, - warranty_class, - process_user_id, - hub_id, - cancel_reason, - ref_jo_id, - tier1_notes, - tier2_notes, - mode_of_payment, - or_name, - landmark, - promo_detail, - or_num, - trade_in_type, - flag_rider_rating, - flag_coolant, - facilitated_hub_id, - facilitated_type, - coord_long, - coord_lat, - priority, - meta, - status_autoassign, - first_name, - last_name, - plate_number, - phone_mobile, - no_trade_in_reason, - will_wait, - reason_not_waiting, - not_waiting_notes, - delivery_status, - emergency_type_id, - ownership_type_id, - cust_location_id, - source_of_awareness, - remarks, - initial_concern, - initial_concern_notes, - gender, - caller_classification, - inventory_count + $query_sql = 'SELECT * FROM job_order - WHERE DATE_CREATE <= DATE_SUB(:cutoff_date, INTERVAL ' . $period . ' YEAR) + WHERE YEAR(date_create) = :year ORDER BY date_create'; $query_stmt = $db->prepare($query_sql); - $query_stmt->bindValue('cutoff_date', $cutoff_date, PDO::PARAM_STR); + $query_stmt->bindValue('year', $year, PDO::PARAM_STR); $results = $query_stmt->executeQuery(); @@ -137,20 +78,15 @@ class GetJobOrderArchiveDataCommand extends Command return 0; } - protected function createArchiveTable($str_date, $period) + protected function createArchiveTable($year) { // form the archive table name _archive_ - $modifier = '-' . $period. 'year'; - - // convert the date string into DateTime - $date = new DateTime($str_date); - $year = $date->modify($modifier)->format('Y'); - $archive_table_name = 'job_order_archive_' . $year; // create the table if it doesn't exist $db = $this->em->getConnection(); + // TODO: What if table already exists? $create_sql = 'CREATE TABLE IF NOT EXISTS `' . $archive_table_name . '` ( `id` int(11) NOT NULL, `customer_id` int(11) DEFAULT NULL, @@ -223,7 +159,7 @@ class GetJobOrderArchiveDataCommand extends Command { // get job order data // check for nulls. check the ff fields since these can be null: date_fulfill, date_cancel, date_assign, create_user_id, - // assign_user_id, proces_user_id, hub_id, rider_id, cancel_reason, ref_jo_id, or_num, trade_in_type, + // assign_user_id, process_user_id, hub_id, rider_id, cancel_reason, ref_jo_id, or_num, trade_in_type, // flag_rider_rating, facilitated_type, facilitated_hub_id, status_autoassign, reason_not_waiting, // not_waiting_notes, no_trade_in_reason, delivery_status, source_of_awareness, remarks, initial_concern, // initial_concern_notes, gender, caller_classifications, emergency_type_id, ownership_type_id, cust_location_id @@ -232,59 +168,34 @@ class GetJobOrderArchiveDataCommand extends Command $cust_id = $row['customer_id']; $cv_id = $row['cvehicle_id']; - $rider_id = '\N'; - if ($row['rider_id'] != NULL) - $rider_id = $row['rider_id']; + $rider_id = $row['rider_id'] ?? '\N'; $date_create = $row['date_create']; $date_schedule = $row['date_schedule']; - $date_fulfill = '\N'; - if ($row['date_fulfill'] != NULL) - $date_fulfill = $row['date_schedule']; + $date_fulfill = $row['date_fulfill'] ?? '\N'; $flag_advance = $row['flag_advance']; $service_type = $row['service_type']; $source = $row['source']; - $date_cancel = '\N'; - if ($row['date_cancel'] != NULL) - $date_cancel = $row['date_cancel']; + $date_cancel = $row['date_cancel'] ?? '\N'; $status = $row['status']; $del_instructions = $this->cleanData($row['delivery_instructions']); $del_address = $this->cleanData($row['delivery_address']); - $create_user_id = '\N'; - if ($row['create_user_id'] != NULL) - $create_user_id = $row['create_user_id']; - - $assign_user_id = '\N'; - if ($row['assign_user_id'] != NULL) - $assign_user_id = $row['assign_user_id']; - - $date_assign = '\N'; - if ($row['date_assign'] != NULL) - $date_assign = $row['date_assign']; + $create_user_id = $row['create_user_id'] ?? '\N'; + $assign_user_id = $row['assign_user_id'] ?? '\N'; + $date_assign = $row['date_assign'] ?? '\N'; $warr_class = $row['warranty_class']; - $process_user_id = '\N'; - if ($row['process_user_id'] != NULL) - $process_user_id = $row['process_user_id']; - - $hub_id = '\N'; - if ($row['hub_id'] != NULL) - $hub_id = $row['hub_id']; - - $cancel_reason = '\N'; - if ($row['cancel_reason'] != NULL) - $cancel_reason = $row['cancel_reason']; - - $ref_jo_id = '\N'; - if ($row['ref_jo_id'] != NULL) - $ref_jo_id = $row['ref_jo_id']; + $process_user_id = $row['process_user_id'] ?? '\N'; + $hub_id = $row['hub_id'] ?? '\N'; + $cancel_reason = $row['cancel_reason'] ?? '\N'; + $ref_jo_id = $row['ref_jo_id'] ?? '\N'; $tier1_notes = $this->cleanData($row['tier1_notes']); $tier2_notes = $this->cleanData($row['tier2_notes']); @@ -295,99 +206,47 @@ class GetJobOrderArchiveDataCommand extends Command $landmark = $this->cleanData($row['landmark']); $promo_details = $row['promo_detail']; - $or_num = '\N'; - if ($row['or_num'] != NULL) - $or_num = $row['or_num']; - - $trade_in_type = '\N'; - if ($row['trade_in_type'] != NULL) - $trade_in_type = $row['trade_in_type']; - - $flag_rider_rating = '\N'; - if ($row['flag_rider_rating'] != NULL) - $flag_rider_rating = $row['flag_rider_rating']; + $or_num = $row['or_num'] ?? '\N'; + $trade_in_type = $row['trade_in_type'] ?? '\N'; + $flag_rider_rating = $row['flag_rider_rating'] ?? '\N'; $flag_coolant = $row['flag_coolant']; - $fac_hub_id = '\N'; - if ($row['facilitated_hub_id'] != NULL) - $fac_hub_id = $row['facilitated_hub_id']; - - $fac_type = '\N'; - if ($row['facilitated_type'] != NULL) - $fac_type = $row['facilitated_type']; + $fac_hub_id = $row['facilitated_hub_id'] ?? '\N'; + $fac_type = $row['facilitated_type'] ?? '\N'; $coord_long = $row['coord_long']; $coord_lat = $row['coord_lat']; // coordinates needs special handling since it's a spatial column - $coordinates = 'POINT(' . $coord_lat . ' ' . $coord_long .')'; + $geo_coordinates = 'POINT(' . $coord_lat . ' ' . $coord_long .')'; $priority = $row['priority']; $meta = $row['meta']; - $status_autoassign = '\N'; - if ($row['status_autoassign'] != NULL) - $status_autoassign = $row['status_autoassign']; + $status_autoassign = $row['status_autoassign'] ?? '\N'; $first_name = $row['first_name']; $last_name = $row['last_name']; $plate_number = $row['plate_number']; $phone_mobile = $row['phone_mobile']; - $no_trade_in_reason = '\N'; - if ($row['no_trade_in_reason'] != NULL) - $no_trade_in_reason = $row['no_trade_in_reason']; + $no_trade_in_reason = $row['no_trade_in_reason'] ?? '\N'; $will_wait = $row['will_wait']; - $reason_not_waiting = '\N'; - if ($row['reason_not_waiting'] != NULL) - $reason_not_waiting = $row['reason_not_waiting']; - - $not_waiting_notes = '\N'; - if ($row['not_waiting_notes'] != NULL) - $not_waiting_notes = $row['not_waiting_notes']; - - $del_status = '\N'; - if ($row['delivery_status'] != NULL) - $del_status = $row['delivery_status']; - - $emergency_type_id = '\N'; - if ($row['emergency_type_id'] != NULL) - $emergency_type_id = $row['emergency_type_id']; - - $owner_type_id = '\N'; - if ($row['ownership_type_id'] != NULL) - $owner_type_id = $row['ownership_type_id']; - - $cust_location_id = '\N'; - if ($row['cust_location_id'] != NULL) - $cust_location_id = $row['cust_location_id']; - - $source_of_awareness = '\N'; - if ($row['source_of_awareness'] != NULL) - $source_of_awareness = $row['source_of_awareness']; - - $remarks = '\N'; - if ($row['remarks'] != NULL) - $remarks = $this->cleanData(row['remarks']); - - $initial_concern = '\N'; - if ($row['initial_concern'] != NULL) - $initial_concern = $row['initial_concern']; - - $initial_concern_notes = '\N'; - if ($row['initial_concern_notes'] != NULL) - $initial_concern_notes = $row['initial_concern_notes']; - - $gender = '\N'; - if ($row['gender'] != NULL) - $gender = $row['gender']; - - $caller_class = '\N'; - if ($row['caller_classification'] != NULL) - $caller_class = $row['caller_classification']; + $reason_not_waiting = $row['reason_not_waiting'] ?? '\N'; + $not_waiting_notes = $this->cleanData($row['not_waiting_notes']) ?? '\N'; + $del_status = $row['delivery_status'] ?? '\N'; + $emergency_type_id = $row['emergency_type_id'] ?? '\N'; + $owner_type_id = $row['ownership_type_id'] ?? '\N'; + $cust_location_id = $row['cust_location_id'] ?? '\N'; + $source_of_awareness = $row['source_of_awareness'] ?? '\N'; + $remarks = $this->cleanData($row['remarks']) ?? '\N'; + $initial_concern = $row['initial_concern'] ?? '\N'; + $initial_concern_notes = $this->cleanData($row['initial_concern_notes']) ?? '\N'; + $gender = $row['gender'] ?? '\N'; + $caller_class = $row['caller_classification'] ?? '\N'; $inv_count = $row['inventory_count']; @@ -400,7 +259,7 @@ class GetJobOrderArchiveDataCommand extends Command $date_create, $date_schedule, $date_fulfill, - $coordinates, + $geo_coordinates, $flag_advance, $service_type, $source, @@ -497,7 +356,7 @@ class GetJobOrderArchiveDataCommand extends Command will_wait, reason_not_waiting, not_waiting_notes, delivery_status, emergency_type_id, ownership_type_id, cust_location_id, source_of_awareness, remarks, initial_concern, initial_concern_notes, gender, caller_classification, inventory_count) - SET coordinates=ST_GeomFromText(@coordinates)' + SET coordinates=ST_GeomFromText(@geo_coordinates)' ); $result = $stmt->execute(); From 35ed917336f3112a3a54153e65e5b3dac69d3cea Mon Sep 17 00:00:00 2001 From: Korina Cordero Date: Tue, 12 Sep 2023 18:18:37 +0800 Subject: [PATCH 07/27] Refactor the retrieval of job order data to include the other associated tables. #762 --- src/Command/GetJobOrderArchiveDataCommand.php | 154 ++++++++++++++++-- 1 file changed, 136 insertions(+), 18 deletions(-) diff --git a/src/Command/GetJobOrderArchiveDataCommand.php b/src/Command/GetJobOrderArchiveDataCommand.php index 2e8e6a41..e1bcabbd 100644 --- a/src/Command/GetJobOrderArchiveDataCommand.php +++ b/src/Command/GetJobOrderArchiveDataCommand.php @@ -42,43 +42,134 @@ class GetJobOrderArchiveDataCommand extends Command // get table to back up $year = $input->getArgument('year'); - $em = $this->em; + // get the data to archive for the following tables: + // (1) job_order + // (2) invoice (has jo foreign key) + // (3) invoice_item (has invoice foreign key) + // (4) ticket (has jo foreign key) + // (5) jo_rejection (has jo foreign key) + // (6) rider_rating (has jo foreign key) - $db = $em->getConnection(); + $jo_backup_data = $this->getJobOrderData($year); - // create the archive table - $archive_table_name = $this->createArchiveTable($year); + // create the archive tables + // TODO: create the other archive tables + $archive_table_name = $this->createJobOrderArchiveTable($year); + + error_log('count ' . count($jo_backup_data)); + + // create the load file for the backup data + // $this->createLoadDataFileForBackupData($jo_backup_data, $archive_table_name); + + return 0; + } + + protected function getJobOrderData($year) + { + $db = $this->em->getConnection(); - // TODO: move the creation of the query_sql to a function or - // set this as a preset or a variable or something and load specific query - // according to what table is to be archived. - // sql query to retrieve the rows or entries for backup $query_sql = 'SELECT * FROM job_order WHERE YEAR(date_create) = :year - ORDER BY date_create'; + ORDER BY date_create'; $query_stmt = $db->prepare($query_sql); $query_stmt->bindValue('year', $year, PDO::PARAM_STR); $results = $query_stmt->executeQuery(); - $backup_data = []; + $jo_data = []; + $jo_event_data = []; + $invoice_data = []; + $ticket_data = []; + $jo_rejection_data = []; + $rider_rating_data = []; while ($row = $results->fetchAssociative()) { - $backup_data[] = $this->createBackupData($row); + $jo_data[] = $this->createJobOrderArchiveData($row); + + // get the data to archive for jo_event, invoice, ticket, jo_rejection, rider_rating + // using the job order id + $id = $row['id']; + + $jo_event_data = $this->getJORelatedData($id, 'jo_event'); + $ticket_data = $this->getJORelatedData($id, 'ticket'); + $jo_rejection_data = $this->getJORelatedData($id, 'jo_rejection'); + $rider_rating_data = $this->getJORelatedData($id, 'rider_rating'); + + // TODO: separate the invoice and invoice item data } - error_log('count ' . count($backup_data)); + $backup_data = [ + 'jo' => $jo_data, + 'jo_event' => $jo_event_data, + 'invoice' => $invoice_data, + 'ticket' => $ticket_data, + 'jo_rejection' => $jo_rejection_data, + 'rider_rating' => $rider_rating_data, + ]; - // create the load file for the backup data - $this->createLoadDataFileForBackupData($backup_data, $archive_table_name); - - return 0; + return $backup_data; } - protected function createArchiveTable($year) + protected function getJORelatedData($id, $table_name) + { + $db = $this->em->getConnection(); + + if (($table_name == 'jo_event') || + ($table_name == 'invoice') || + ($table_name == 'ticket')) + { + $query_sql = 'SELECT * + FROM ' . $table_name . ' WHERE job_order_id = :id'; + } + else + { + $query_sql = 'SELECT * + FROM ' . $table_name . ' WHERE jo_id = :id'; + } + + $query_stmt = $db->prepare($query_sql); + $query_stmt->bindValue('id', $id); + + $results = $query_stmt->executeQuery(); + + $jo_related_data = []; + + while ($row = $results->fetchAssociative()) + { + if ($table_name == 'jo_event') + { + // create the jo event archive data + $jo_related_data = $this->createJOEventArchiveData($row); + } + if ($table_name == 'invoice') + { + // get the data to archive for invoice item + // create the invoice archive data + } + if ($table_name == 'ticket') + { + // create the ticket archive data + } + if ($table_name == 'jo_rejection') + { + // create the jo rejection archive data + } + if ($table_name == 'rider_rating') + { + // create the rider rating archive data + } + } + + return $jo_related_data; + } + + // TODO: make this so you just call one function to create all the tables + // pass the year and the table name + // set the create sql as a constant or something + protected function createJobOrderArchiveTable($year) { // form the archive table name _archive_ $archive_table_name = 'job_order_archive_' . $year; @@ -155,7 +246,7 @@ class GetJobOrderArchiveDataCommand extends Command return $archive_table_name; } - protected function createBackupData($row) + protected function createJobOrderArchiveData($row) { // get job order data // check for nulls. check the ff fields since these can be null: date_fulfill, date_cancel, date_assign, create_user_id, @@ -316,6 +407,33 @@ class GetJobOrderArchiveDataCommand extends Command return $data; } + protected function createJOEventArchiveData($row) + { + // fields that can be null: rider_id, create_user_id + $id = $row['id']; + + $create_user_id = $row['create_user_id'] ?? '\N'; + + $job_order_id = $row['job_order_id']; + $date_create = $row['date_create']; + $date_happen = $row['date_happen']; + $type_id = $row['type_id']; + + $rider_id = $row['rider_id'] ?? '\N'; + + $data = [ + $id, + $create_user_id, + $job_order_id, + $date_create, + $date_happen, + $type_id, + $rider_id, + ]; + + return $data; + } + protected function createLoadDataFileForBackupData($backup_data, $table_name) { // cache directory From d7a2b25e51d4fab4640abda8de249d6f7c82df79 Mon Sep 17 00:00:00 2001 From: Korina Cordero Date: Wed, 13 Sep 2023 17:18:33 +0800 Subject: [PATCH 08/27] Add data files for the related job order tables. #762 --- src/Command/GetJobOrderArchiveDataCommand.php | 190 +++++++++--------- 1 file changed, 96 insertions(+), 94 deletions(-) diff --git a/src/Command/GetJobOrderArchiveDataCommand.php b/src/Command/GetJobOrderArchiveDataCommand.php index e1bcabbd..de602ab5 100644 --- a/src/Command/GetJobOrderArchiveDataCommand.php +++ b/src/Command/GetJobOrderArchiveDataCommand.php @@ -39,7 +39,7 @@ class GetJobOrderArchiveDataCommand extends Command protected function execute(InputInterface $input, OutputInterface $output) { - // get table to back up + // get year to archive $year = $input->getArgument('year'); // get the data to archive for the following tables: @@ -50,70 +50,59 @@ class GetJobOrderArchiveDataCommand extends Command // (5) jo_rejection (has jo foreign key) // (6) rider_rating (has jo foreign key) - $jo_backup_data = $this->getJobOrderData($year); - // create the archive tables // TODO: create the other archive tables $archive_table_name = $this->createJobOrderArchiveTable($year); - - error_log('count ' . count($jo_backup_data)); - - // create the load file for the backup data - // $this->createLoadDataFileForBackupData($jo_backup_data, $archive_table_name); - - return 0; - } - - protected function getJobOrderData($year) - { $db = $this->em->getConnection(); $query_sql = 'SELECT * FROM job_order WHERE YEAR(date_create) = :year - ORDER BY date_create'; + ORDER BY date_create'; $query_stmt = $db->prepare($query_sql); $query_stmt->bindValue('year', $year, PDO::PARAM_STR); - $results = $query_stmt->executeQuery(); + $callback = ['App\Command\GetJobOrderArchiveDataCommand', 'getRelatedArchiveData']; - $jo_data = []; - $jo_event_data = []; - $invoice_data = []; - $ticket_data = []; - $jo_rejection_data = []; - $rider_rating_data = []; + $this->getArchiveData($query_stmt, $callback); + + return 0; + } + + protected function getArchiveData($stmt, $callbackJO) + { + $results = $stmt->executeQuery(); + + $related_tables = ['jo_event', 'invoice', 'ticket', 'jo_rejection', 'rider_rating']; + + // delete the related data files + foreach ($related_tables as $tname) + { + $this->deleteDataFiles($tname); + } while ($row = $results->fetchAssociative()) { $jo_data[] = $this->createJobOrderArchiveData($row); - // get the data to archive for jo_event, invoice, ticket, jo_rejection, rider_rating - // using the job order id - $id = $row['id']; + // foreach job order id we got from the first query, we get the JO related + // data for that id from jo_event, invoice, ticket, jo_rejection, rider_rating - $jo_event_data = $this->getJORelatedData($id, 'jo_event'); - $ticket_data = $this->getJORelatedData($id, 'ticket'); - $jo_rejection_data = $this->getJORelatedData($id, 'jo_rejection'); - $rider_rating_data = $this->getJORelatedData($id, 'rider_rating'); - - // TODO: separate the invoice and invoice item data + if (is_callable($callbackJO)) + { + foreach ($related_tables as $table_name) + { + call_user_func($callbackJO, $row, $table_name); + } + } } - $backup_data = [ - 'jo' => $jo_data, - 'jo_event' => $jo_event_data, - 'invoice' => $invoice_data, - 'ticket' => $ticket_data, - 'jo_rejection' => $jo_rejection_data, - 'rider_rating' => $rider_rating_data, - ]; + // TODO: load data infile for job order - return $backup_data; } - protected function getJORelatedData($id, $table_name) + protected function getRelatedArchiveData($row, $table_name) { $db = $this->em->getConnection(); @@ -131,39 +120,77 @@ class GetJobOrderArchiveDataCommand extends Command } $query_stmt = $db->prepare($query_sql); - $query_stmt->bindValue('id', $id); + $query_stmt->bindValue('id', $row['id']); $results = $query_stmt->executeQuery(); - $jo_related_data = []; + $data = []; - while ($row = $results->fetchAssociative()) + while ($q_row = $results->fetchAssociative()) { - if ($table_name == 'jo_event') + $fields = []; + + foreach ($q_row as $key => $value) { - // create the jo event archive data - $jo_related_data = $this->createJOEventArchiveData($row); - } - if ($table_name == 'invoice') - { - // get the data to archive for invoice item - // create the invoice archive data - } - if ($table_name == 'ticket') - { - // create the ticket archive data - } - if ($table_name == 'jo_rejection') - { - // create the jo rejection archive data - } - if ($table_name == 'rider_rating') - { - // create the rider rating archive data + $cleaned_value = $this->cleanData(($value) ?? '\N'); + $fields[] = $cleaned_value; } + + $data[$table_name][$q_row['id']] = $fields; } - return $jo_related_data; + // write the array into the file + $file = $this->createDataFileRelatedArchiveData($data, $table_name); + + if ($file != null) + { + // call load data infile + } + } + + protected function deleteDataFiles($tname) + { + // cache directory + $cache_dir = __DIR__ . '/../../var/cache'; + + $file = $cache_dir . '/' . $tname . '_archive.tab'; + + if (file_exists($file)) + unlink($file); + } + + protected function createDataFileRelatedArchiveData($archive_data, $table_name) + { + if (isset($archive_data[$table_name])) + { + $adata = $archive_data[$table_name]; + + // cache directory + $cache_dir = __DIR__ . '/../../var/cache'; + + $file = $cache_dir . '/' . $table_name . '_archive.tab'; + error_log('opening file for archive - ' . $file); + + $fp = fopen($file, 'a'); + if ($fp === false) + { + error_log('could not open file for load data infile - ' . $file); + } + else + { + foreach ($adata as $key => $data) + { + $line = implode('|', $data) . "\r\n"; + fwrite($fp, $line); + } + } + + fclose($fp); + + return $file; + } + + return null; } // TODO: make this so you just call one function to create all the tables @@ -248,6 +275,8 @@ class GetJobOrderArchiveDataCommand extends Command protected function createJobOrderArchiveData($row) { + // TODO: this could be shrunk further + // get job order data // check for nulls. check the ff fields since these can be null: date_fulfill, date_cancel, date_assign, create_user_id, // assign_user_id, process_user_id, hub_id, rider_id, cancel_reason, ref_jo_id, or_num, trade_in_type, @@ -407,34 +436,7 @@ class GetJobOrderArchiveDataCommand extends Command return $data; } - protected function createJOEventArchiveData($row) - { - // fields that can be null: rider_id, create_user_id - $id = $row['id']; - - $create_user_id = $row['create_user_id'] ?? '\N'; - - $job_order_id = $row['job_order_id']; - $date_create = $row['date_create']; - $date_happen = $row['date_happen']; - $type_id = $row['type_id']; - - $rider_id = $row['rider_id'] ?? '\N'; - - $data = [ - $id, - $create_user_id, - $job_order_id, - $date_create, - $date_happen, - $type_id, - $rider_id, - ]; - - return $data; - } - - protected function createLoadDataFileForBackupData($backup_data, $table_name) + protected function createLoadDataFileForArchiveData($archive_data, $table_name) { // cache directory $cache_dir = __DIR__ . '/../../var/cache'; @@ -449,7 +451,7 @@ class GetJobOrderArchiveDataCommand extends Command } else { - foreach ($backup_data as $key => $data) + foreach ($archive_data as $key => $data) { $line = implode('|', $data) . "\r\n"; fwrite($fp, $line); From c1578eee1f9c47f052907ae548dd5ecd0875bd0e Mon Sep 17 00:00:00 2001 From: Korina Cordero Date: Fri, 15 Sep 2023 18:19:11 +0800 Subject: [PATCH 09/27] Add archiving for invoice item. Add creation of other archive tables. #762 --- src/Command/GetJobOrderArchiveDataCommand.php | 284 ++++++++++++++---- 1 file changed, 229 insertions(+), 55 deletions(-) diff --git a/src/Command/GetJobOrderArchiveDataCommand.php b/src/Command/GetJobOrderArchiveDataCommand.php index de602ab5..81cd795d 100644 --- a/src/Command/GetJobOrderArchiveDataCommand.php +++ b/src/Command/GetJobOrderArchiveDataCommand.php @@ -51,14 +51,19 @@ class GetJobOrderArchiveDataCommand extends Command // (6) rider_rating (has jo foreign key) // create the archive tables - // TODO: create the other archive tables - $archive_table_name = $this->createJobOrderArchiveTable($year); + $this->createJobOrderArchiveTables($year); + $this->createInvoiceArchiveTable($year); + $this->createInvoiceItemArchiveTable($year); + $this->createTicketArchiveTable($year); + $this->createJORejectionArchiveTable($year); + $this->createRiderRatingArchiveTable($year); + $db = $this->em->getConnection(); $query_sql = 'SELECT * FROM job_order WHERE YEAR(date_create) = :year - ORDER BY date_create'; + ORDER BY date_create'; $query_stmt = $db->prepare($query_sql); $query_stmt->bindValue('year', $year, PDO::PARAM_STR); @@ -82,52 +87,66 @@ class GetJobOrderArchiveDataCommand extends Command $this->deleteDataFiles($tname); } + // special since this is not directly related to JO but to invoice + $this->deleteDataFiles('invoice_item'); + while ($row = $results->fetchAssociative()) { - $jo_data[] = $this->createJobOrderArchiveData($row); + $jo_data['job_order'][$row['id']] = $this->createJobOrderArchiveData($row); // foreach job order id we got from the first query, we get the JO related // data for that id from jo_event, invoice, ticket, jo_rejection, rider_rating if (is_callable($callbackJO)) - { + { foreach ($related_tables as $table_name) { - call_user_func($callbackJO, $row, $table_name); + if (($table_name == 'jo_event') || + ($table_name == 'invoice') || + ($table_name == 'ticket')) + { + $query_sql = 'SELECT * FROM ' . $table_name . ' WHERE job_order_id = :id'; + } + else + { + $query_sql = 'SELECT * FROM ' . $table_name . ' WHERE jo_id = :id'; + } + + $db = $this->em->getConnection(); + + $query_stmt = $db->prepare($query_sql); + $query_stmt->bindValue('id', $row['id']); + + call_user_func($callbackJO, $row, $query_stmt, $table_name); } } } - // TODO: load data infile for job order - + // write the array into the file + $file = $this->createDataFileRelatedArchiveData($jo_data, 'job_order', 'w'); + + if ($file != null) + { + // call load data infile + } } - protected function getRelatedArchiveData($row, $table_name) + protected function getRelatedArchiveData($row, $query_stmt, $table_name) { - $db = $this->em->getConnection(); - - if (($table_name == 'jo_event') || - ($table_name == 'invoice') || - ($table_name == 'ticket')) - { - $query_sql = 'SELECT * - FROM ' . $table_name . ' WHERE job_order_id = :id'; - } - else - { - $query_sql = 'SELECT * - FROM ' . $table_name . ' WHERE jo_id = :id'; - } - - $query_stmt = $db->prepare($query_sql); - $query_stmt->bindValue('id', $row['id']); - $results = $query_stmt->executeQuery(); $data = []; while ($q_row = $results->fetchAssociative()) { + // check if table name is invoice because we need to get + // all invoice items for a specific invoice too + if ($table_name == 'invoice') + { + // get invoice item related data + $this->getInvoiceItemArchiveData($q_row); + } + $fields = []; foreach ($q_row as $key => $value) @@ -140,7 +159,46 @@ class GetJobOrderArchiveDataCommand extends Command } // write the array into the file - $file = $this->createDataFileRelatedArchiveData($data, $table_name); + $file = $this->createDataFileRelatedArchiveData($data, $table_name, 'a'); + + if ($file != null) + { + // call load data infile + } + } + + protected function getInvoiceItemArchiveData($row) + { + $db = $this->em->getConnection(); + + $query_sql = 'SELECT * FROM invoice_item WHERE invoice_id = :id'; + + $query_stmt = $db->prepare($query_sql); + $query_stmt->bindValue('id', $row['id']); + + $results = $query_stmt->executeQuery(); + + $ii_data = []; + while ($ii_row = $results->fetchAssociative()) + { + $id = $ii_row['id']; + $invoice_id = $ii_row['invoice_id']; + $title = $ii_row['title']; + $qty = $ii_row['qty']; + $price = $ii_row['price']; + $battery_id = $ii_row['battery_id'] ?? '\N'; + + $ii_data['invoice_item'][$id] = [ + $id, + $invoice_id, + $title, + $qty, + $price, + $battery_id + ]; + } + + $file = $this->createDataFileRelatedArchiveData($ii_data, 'invoice_item', 'a'); if ($file != null) { @@ -159,7 +217,7 @@ class GetJobOrderArchiveDataCommand extends Command unlink($file); } - protected function createDataFileRelatedArchiveData($archive_data, $table_name) + protected function createDataFileRelatedArchiveData($archive_data, $table_name, $option) { if (isset($archive_data[$table_name])) { @@ -171,7 +229,7 @@ class GetJobOrderArchiveDataCommand extends Command $file = $cache_dir . '/' . $table_name . '_archive.tab'; error_log('opening file for archive - ' . $file); - $fp = fopen($file, 'a'); + $fp = fopen($file, $option); if ($fp === false) { error_log('could not open file for load data infile - ' . $file); @@ -193,11 +251,7 @@ class GetJobOrderArchiveDataCommand extends Command return null; } - // TODO: make this so you just call one function to create all the tables - // pass the year and the table name - // set the create sql as a constant or something - protected function createJobOrderArchiveTable($year) - { + protected function createJobOrderArchiveTables($year) { // form the archive table name _archive_ $archive_table_name = 'job_order_archive_' . $year; @@ -269,10 +323,142 @@ class GetJobOrderArchiveDataCommand extends Command $create_stmt = $db->prepare($create_sql); $result = $create_stmt->execute(); - - return $archive_table_name; } + protected function createInvoiceArchiveTable($year) + { + // form the archive table name _archive_ + $archive_table_name = 'invoice_archive_' . $year; + + // create the table if it doesn't exist + $db = $this->em->getConnection(); + + // TODO: What if table already exists? + $create_sql = 'CREATE TABLE IF NOT EXISTS `' . $archive_table_name . '` ( + `id` int(11) NOT NULL, + `user_id` int(11) DEFAULT NULL, + `job_order_id` int(11) DEFAULT NULL, + `date_create` datetime NOT NULL, + `date_paid` datetime DEFAULT NULL, + `date_cancel` datetime DEFAULT NULL, + `discount` decimal(9,2) NOT NULL, + `trade_in` decimal(9,2) NOT NULL, + `vat` decimal(9,2) NOT NULL, + `vat_exclusive_price` decimal(9,2) NOT NULL, + `total_price` decimal(9,2) NOT NULL, + `status` varchar(40) COLLATE utf8_unicode_ci NOT NULL, + `promo_id` int(11) DEFAULT NULL, + `used_customer_tag_id` varchar(80) COLLATE utf8_unicode_ci DEFAULT NULL, PRIMARY KEY (`id`))'; + + $create_stmt = $db->prepare($create_sql); + + $result = $create_stmt->execute(); + } + + protected function createInvoiceItemArchiveTable($year) + { + // form the archive table name _archive_ + $archive_table_name = 'invoice_item_archive_' . $year; + + // create the table if it doesn't exist + $db = $this->em->getConnection(); + + // TODO: What if table already exists? + $create_sql = 'CREATE TABLE IF NOT EXISTS `' . $archive_table_name . '` ( + `id` int(11) NOT NULL, + `invoice_id` int(11) DEFAULT NULL, + `title` varchar(80) COLLATE utf8_unicode_ci NOT NULL, + `qty` smallint(6) NOT NULL, + `price` decimal(9,2) NOT NULL, + `battery_id` int(11) DEFAULT NULL, PRIMARY KEY (`id`))'; + + $create_stmt = $db->prepare($create_sql); + + $result = $create_stmt->execute(); + } + + protected function createTicketArchiveTable($year) + { + // form the archive table name _archive_ + $archive_table_name = 'ticket_archive_' . $year; + + // create the table if it doesn't exist + $db = $this->em->getConnection(); + + // TODO: What if table already exists? + $create_sql = 'CREATE TABLE IF NOT EXISTS `' . $archive_table_name . '` ( + `id` int(11) NOT NULL, + `user_id` int(11) DEFAULT NULL, + `customer_id` int(11) DEFAULT NULL, + `date_create` datetime NOT NULL, + `status` varchar(15) COLLATE utf8_unicode_ci NOT NULL, + `ticket_type` varchar(15) COLLATE utf8_unicode_ci DEFAULT NULL, + `other_ticket_type` varchar(80) COLLATE utf8_unicode_ci DEFAULT NULL, + `first_name` varchar(80) COLLATE utf8_unicode_ci NOT NULL, + `last_name` varchar(80) COLLATE utf8_unicode_ci NOT NULL, + `contact_num` varchar(20) COLLATE utf8_unicode_ci DEFAULT NULL, + `details` longtext COLLATE utf8_unicode_ci DEFAULT NULL, + `job_order_id` int(11) DEFAULT NULL, + `plate_number` varchar(10) COLLATE utf8_unicode_ci DEFAULT NULL, + `ticket_type_id` int(11) DEFAULT NULL, + `subticket_type_id` int(11) DEFAULT NULL, + `source_of_awareness` varchar(80) COLLATE utf8_unicode_ci DEFAULT NULL, + `remarks` longtext COLLATE utf8_unicode_ci DEFAULT NULL, + `other_description` longtext COLLATE utf8_unicode_ci DEFAULT NULL, PRIMARY KEY (`id`))'; + + $create_stmt = $db->prepare($create_sql); + + $result = $create_stmt->execute(); + } + + protected function createJORejectionArchiveTable($year) + { + // form the archive table name _archive_ + $archive_table_name = 'jo_rejection_archive_' . $year; + + // create the table if it doesn't exist + $db = $this->em->getConnection(); + + // TODO: What if table already exists? + $create_sql = 'CREATE TABLE IF NOT EXISTS `' . $archive_table_name . '` ( + `id` int(11) NOT NULL, + `user_id` int(11) DEFAULT NULL, + `hub_id` int(11) DEFAULT NULL, + `jo_id` int(11) DEFAULT NULL, + `date_create` datetime NOT NULL, + `reason` varchar(255) COLLATE utf8_unicode_ci NOT NULL, + `remarks` longtext COLLATE utf8_unicode_ci DEFAULT NULL, + `contact_person` varchar(255) COLLATE utf8_unicode_ci DEFAULT NULL, PRIMARY KEY (`id`))'; + + $create_stmt = $db->prepare($create_sql); + + $result = $create_stmt->execute(); + } + + protected function createRiderRatingArchiveTable($year) + { + // form the archive table name _archive_ + $archive_table_name = 'jo_rejection_archive_' . $year; + + // create the table if it doesn't exist + $db = $this->em->getConnection(); + + // TODO: What if table already exists? + $create_sql = 'CREATE TABLE IF NOT EXISTS `' . $archive_table_name . '` ( + `id` int(11) NOT NULL, + `rider_id` int(11) DEFAULT NULL, + `customer_id` int(11) DEFAULT NULL, + `jo_id` int(11) DEFAULT NULL, + `date_create` datetime NOT NULL, + `rating` int(11) NOT NULL, + `comment` longtext COLLATE utf8_unicode_ci NOT NULL, PRIMARY KEY (`id`))'; + + $create_stmt = $db->prepare($create_sql); + + $result = $create_stmt->execute(); + } + + protected function createJobOrderArchiveData($row) { // TODO: this could be shrunk further @@ -436,31 +622,19 @@ class GetJobOrderArchiveDataCommand extends Command return $data; } - protected function createLoadDataFileForArchiveData($archive_data, $table_name) + // TODO: rewrite this to just load the file + protected function loadDataFileForArchiveData($file, $table_name) { // cache directory $cache_dir = __DIR__ . '/../../var/cache'; - $file = $cache_dir . '/jo_archive.tab'; + $file = $cache_dir . '/' $table_name . 'archive.tab'; error_log('opening file for jo archive - ' . $file); - $fp = fopen($file, 'w'); - if ($fp === false) - { - error_log('could not open file for load data infile - ' . $file); - } - else - { - foreach ($archive_data as $key => $data) - { - $line = implode('|', $data) . "\r\n"; - fwrite($fp, $line); - } - } - - fclose($fp); - $conn = $this->em->getConnection(); + + // this statement is for job order + // TODO: make for other tables $stmt = $conn->prepare('LOAD DATA LOCAL INFILE \'' . $file . '\' INTO TABLE ' . $table_name . ' FIELDS TERMINATED BY \'|\' LINES TERMINATED BY \'\\r\\n\' From 3773a664d13ffd100171e6735c81a958924ab210 Mon Sep 17 00:00:00 2001 From: Korina Cordero Date: Mon, 18 Sep 2023 18:10:06 +0800 Subject: [PATCH 10/27] Add loading of data file. #762 --- src/Command/GetJobOrderArchiveDataCommand.php | 144 +++++++++++++----- 1 file changed, 109 insertions(+), 35 deletions(-) diff --git a/src/Command/GetJobOrderArchiveDataCommand.php b/src/Command/GetJobOrderArchiveDataCommand.php index 81cd795d..2aa81c42 100644 --- a/src/Command/GetJobOrderArchiveDataCommand.php +++ b/src/Command/GetJobOrderArchiveDataCommand.php @@ -57,25 +57,27 @@ class GetJobOrderArchiveDataCommand extends Command $this->createTicketArchiveTable($year); $this->createJORejectionArchiveTable($year); $this->createRiderRatingArchiveTable($year); + $this->createJOEventArchiveTable($year); $db = $this->em->getConnection(); $query_sql = 'SELECT * FROM job_order WHERE YEAR(date_create) = :year - ORDER BY date_create'; + ORDER BY date_create + LIMIT 100'; $query_stmt = $db->prepare($query_sql); $query_stmt->bindValue('year', $year, PDO::PARAM_STR); $callback = ['App\Command\GetJobOrderArchiveDataCommand', 'getRelatedArchiveData']; - $this->getArchiveData($query_stmt, $callback); + $this->getArchiveData($query_stmt, $callback, 'job_order', $year); return 0; } - protected function getArchiveData($stmt, $callbackJO) + protected function getArchiveData($stmt, $callbackJO, $jo_tname, $year) { $results = $stmt->executeQuery(); @@ -117,21 +119,42 @@ class GetJobOrderArchiveDataCommand extends Command $query_stmt = $db->prepare($query_sql); $query_stmt->bindValue('id', $row['id']); - call_user_func($callbackJO, $row, $query_stmt, $table_name); + call_user_func($callbackJO, $row, $query_stmt, $table_name, $year); } } } // write the array into the file - $file = $this->createDataFileRelatedArchiveData($jo_data, 'job_order', 'w'); + $file = $this->createDataFileRelatedArchiveData($jo_data, $jo_tname, 'w'); if ($file != null) - { + { + $archive_tname = $jo_tname . '_archive_' . $year; + + // load statement for job order + $load_stmt = 'LOAD DATA LOCAL INFILE \'' . $file . '\' INTO TABLE ' . $archive_tname . ' + FIELDS TERMINATED BY \'|\' + LINES TERMINATED BY \'\\r\\n\' + (id, customer_id, cvehicle_id, rider_id, date_create, + date_schedule, date_fulfill, @coordinates, flag_advance, service_type, + source, date_cancel, status, delivery_instructions, delivery_address, + create_user_id, assign_user_id, date_assign, warranty_class, process_user_id, + hub_id, cancel_reason, ref_jo_id, tier1_notes, tier2_notes, + mode_of_payment, or_name, landmark, promo_detail, or_num, + trade_in_type, flag_rider_rating, flag_coolant, facilitated_hub_id, facilitated_type, + coord_long, coord_lat, priority, meta, status_autoassign, + first_name, last_name, plate_number, phone_mobile, no_trade_in_reason, + will_wait, reason_not_waiting, not_waiting_notes, delivery_status, emergency_type_id, + ownership_type_id, cust_location_id, source_of_awareness, remarks, initial_concern, + initial_concern_notes, gender, caller_classification, inventory_count) + SET coordinates=ST_GeomFromText(@geo_coordinates)'; + // call load data infile + $this->loadDataFileForArchiveData($load_stmt); } } - protected function getRelatedArchiveData($row, $query_stmt, $table_name) + protected function getRelatedArchiveData($row, $query_stmt, $table_name, $year) { $results = $query_stmt->executeQuery(); @@ -144,7 +167,7 @@ class GetJobOrderArchiveDataCommand extends Command if ($table_name == 'invoice') { // get invoice item related data - $this->getInvoiceItemArchiveData($q_row); + $this->getInvoiceItemArchiveData($q_row, $year); } $fields = []; @@ -161,13 +184,56 @@ class GetJobOrderArchiveDataCommand extends Command // write the array into the file $file = $this->createDataFileRelatedArchiveData($data, $table_name, 'a'); + // TODO: this needs to move or something because we are writing duplicate rows into the db if ($file != null) { + $archive_tname = $table_name . '_archive_' . $year; + + if ($table_name == 'jo_event') + { + $load_stmt = 'LOAD DATA LOCAL INFILE \'' . $file . '\' INTO TABLE ' . $archive_tname . ' + FIELDS TERMINATED BY \'|\' + LINES TERMINATED BY \'\\r\\n\' + (id, create_user_id, job_order_id, date_create, date_happen, type_id, rider_id)'; + } + if ($table_name == 'jo_rejection') + { + $load_stmt = 'LOAD DATA LOCAL INFILE \'' . $file . '\' INTO TABLE ' . $archive_tname . ' + FIELDS TERMINATED BY \'|\' + LINES TERMINATED BY \'\\r\\n\' + (id, user_id, hub_id, jo_id, date_create, reason, remarks, contact_person)'; + } + if ($table_name == 'invoice') + { + $load_stmt = 'LOAD DATA LOCAL INFILE \'' . $file . '\' INTO TABLE ' . $archive_tname . ' + FIELDS TERMINATED BY \'|\' + LINES TERMINATED BY \'\\r\\n\' + (id, user_id, job_order_id, date_create, date_paid, discount, trade_in, vat, vat_exclusive_price, + total_price, status, promo_id, used_customer_tag_id)'; + } + if ($table_name == 'rider_rating') + { + $load_stmt = 'LOAD DATA LOCAL INFILE \'' . $file . '\' INTO TABLE ' . $archive_tname . ' + FIELDS TERMINATED BY \'|\' + LINES TERMINATED BY \'\\r\\n\' + (id, rider_id, customer_id, jo_id, date_create, rating, comment)'; + } + if ($table_name == 'ticket') + { + $load_stmt = 'LOAD DATA LOCAL INFILE \'' . $file . '\' INTO TABLE ' . $archive_tname . ' + FIELDS TERMINATED BY \'|\' + LINES TERMINATED BY \'\\r\\n\' + (id, user_id, customer_id, date_create, status, ticket_type, other_ticket_type, first_name, last_name, + contact_num, details, job_order_id, plate_number, ticket_type_id, subticket_type_id, + source_of_awareness, remarks, other_description)'; + } + // call load data infile + $this->loadDataFileForArchiveData($load_stmt); } } - protected function getInvoiceItemArchiveData($row) + protected function getInvoiceItemArchiveData($row, $year) { $db = $this->em->getConnection(); @@ -202,6 +268,12 @@ class GetJobOrderArchiveDataCommand extends Command if ($file != null) { + $archive_tname = 'invoice_item_archive' . $year; + + $load_stmt = 'LOAD DATA LOCAL INFILE \'' . $file . '\' INTO TABLE ' . $archive_tname . ' + FIELDS TERMINATED BY \'|\' + LINES TERMINATED BY \'\\r\\n\' + (id, invoice_id, title, qty, price, battery_id)'; // call load data infile } } @@ -323,6 +395,8 @@ class GetJobOrderArchiveDataCommand extends Command $create_stmt = $db->prepare($create_sql); $result = $create_stmt->execute(); + + return $archive_table_name; } protected function createInvoiceArchiveTable($year) @@ -438,7 +512,7 @@ class GetJobOrderArchiveDataCommand extends Command protected function createRiderRatingArchiveTable($year) { // form the archive table name _archive_ - $archive_table_name = 'jo_rejection_archive_' . $year; + $archive_table_name = 'rider_rating_archive_' . $year; // create the table if it doesn't exist $db = $this->em->getConnection(); @@ -458,6 +532,29 @@ class GetJobOrderArchiveDataCommand extends Command $result = $create_stmt->execute(); } + protected function createJOEventArchiveTable($year) + { + // form the archive table name _archive_ + $archive_table_name = 'jo_event_archive_' . $year; + + // create the table if it doesn't exist + $db = $this->em->getConnection(); + + // TODO: What if table already exists? + $create_sql = 'CREATE TABLE IF NOT EXISTS `' . $archive_table_name . '` ( + `id` int(11) NOT NULL AUTO_INCREMENT, + `create_user_id` int(11) DEFAULT NULL, + `job_order_id` int(11) DEFAULT NULL, + `date_create` datetime NOT NULL, + `date_happen` datetime NOT NULL, + `type_id` varchar(30) COLLATE utf8_unicode_ci NOT NULL, + `rider_id` int(11) DEFAULT NULL, PRIMARY KEY (`id`))'; + + $create_stmt = $db->prepare($create_sql); + + $result = $create_stmt->execute(); + } + protected function createJobOrderArchiveData($row) { @@ -622,36 +719,13 @@ class GetJobOrderArchiveDataCommand extends Command return $data; } - // TODO: rewrite this to just load the file - protected function loadDataFileForArchiveData($file, $table_name) + protected function loadDataFileForArchiveData($load_stmt) { - // cache directory - $cache_dir = __DIR__ . '/../../var/cache'; - - $file = $cache_dir . '/' $table_name . 'archive.tab'; - error_log('opening file for jo archive - ' . $file); - $conn = $this->em->getConnection(); // this statement is for job order // TODO: make for other tables - $stmt = $conn->prepare('LOAD DATA LOCAL INFILE \'' . $file . '\' INTO TABLE ' . $table_name . ' - FIELDS TERMINATED BY \'|\' - LINES TERMINATED BY \'\\r\\n\' - (id, customer_id, cvehicle_id, rider_id, date_create, - date_schedule, date_fulfill, @coordinates, flag_advance, service_type, - source, date_cancel, status, delivery_instructions, delivery_address, - create_user_id, assign_user_id, date_assign, warranty_class, process_user_id, - hub_id, cancel_reason, ref_jo_id, tier1_notes, tier2_notes, - mode_of_payment, or_name, landmark, promo_detail, or_num, - trade_in_type, flag_rider_rating, flag_coolant, facilitated_hub_id, facilitated_type, - coord_long, coord_lat, priority, meta, status_autoassign, - first_name, last_name, plate_number, phone_mobile, no_trade_in_reason, - will_wait, reason_not_waiting, not_waiting_notes, delivery_status, emergency_type_id, - ownership_type_id, cust_location_id, source_of_awareness, remarks, initial_concern, - initial_concern_notes, gender, caller_classification, inventory_count) - SET coordinates=ST_GeomFromText(@geo_coordinates)' - ); + $stmt = $conn->prepare($load_stmt); $result = $stmt->execute(); From f52293caa63fbd75078ae363060f107b15e75ea5 Mon Sep 17 00:00:00 2001 From: Korina Cordero Date: Tue, 19 Sep 2023 12:48:14 +0800 Subject: [PATCH 11/27] Fix load file issue. #762 --- src/Command/GetJobOrderArchiveDataCommand.php | 76 +++++-------------- 1 file changed, 18 insertions(+), 58 deletions(-) diff --git a/src/Command/GetJobOrderArchiveDataCommand.php b/src/Command/GetJobOrderArchiveDataCommand.php index 2aa81c42..061bddc2 100644 --- a/src/Command/GetJobOrderArchiveDataCommand.php +++ b/src/Command/GetJobOrderArchiveDataCommand.php @@ -92,6 +92,7 @@ class GetJobOrderArchiveDataCommand extends Command // special since this is not directly related to JO but to invoice $this->deleteDataFiles('invoice_item'); + $related_files = []; while ($row = $results->fetchAssociative()) { $jo_data['job_order'][$row['id']] = $this->createJobOrderArchiveData($row); @@ -119,11 +120,20 @@ class GetJobOrderArchiveDataCommand extends Command $query_stmt = $db->prepare($query_sql); $query_stmt->bindValue('id', $row['id']); - call_user_func($callbackJO, $row, $query_stmt, $table_name, $year); + $files[] = call_user_func($callbackJO, $row, $query_stmt, $table_name, $year); + + // TODO: need to figure out how to get the filenames + foreach ($files as $key =>$file) + { + $related_files[$key] = $file; + } + } } } + error_log(print_r($related_files, true)); + // write the array into the file $file = $this->createDataFileRelatedArchiveData($jo_data, $jo_tname, 'w'); @@ -159,6 +169,7 @@ class GetJobOrderArchiveDataCommand extends Command $results = $query_stmt->executeQuery(); $data = []; + $files = []; while ($q_row = $results->fetchAssociative()) { @@ -167,7 +178,9 @@ class GetJobOrderArchiveDataCommand extends Command if ($table_name == 'invoice') { // get invoice item related data - $this->getInvoiceItemArchiveData($q_row, $year); + $ii_file = $this->getInvoiceItemArchiveData($q_row, $year); + + $files['invoice_item'] = $ii_file; } $fields = []; @@ -184,53 +197,9 @@ class GetJobOrderArchiveDataCommand extends Command // write the array into the file $file = $this->createDataFileRelatedArchiveData($data, $table_name, 'a'); - // TODO: this needs to move or something because we are writing duplicate rows into the db - if ($file != null) - { - $archive_tname = $table_name . '_archive_' . $year; + $files[$table_name] = $file; - if ($table_name == 'jo_event') - { - $load_stmt = 'LOAD DATA LOCAL INFILE \'' . $file . '\' INTO TABLE ' . $archive_tname . ' - FIELDS TERMINATED BY \'|\' - LINES TERMINATED BY \'\\r\\n\' - (id, create_user_id, job_order_id, date_create, date_happen, type_id, rider_id)'; - } - if ($table_name == 'jo_rejection') - { - $load_stmt = 'LOAD DATA LOCAL INFILE \'' . $file . '\' INTO TABLE ' . $archive_tname . ' - FIELDS TERMINATED BY \'|\' - LINES TERMINATED BY \'\\r\\n\' - (id, user_id, hub_id, jo_id, date_create, reason, remarks, contact_person)'; - } - if ($table_name == 'invoice') - { - $load_stmt = 'LOAD DATA LOCAL INFILE \'' . $file . '\' INTO TABLE ' . $archive_tname . ' - FIELDS TERMINATED BY \'|\' - LINES TERMINATED BY \'\\r\\n\' - (id, user_id, job_order_id, date_create, date_paid, discount, trade_in, vat, vat_exclusive_price, - total_price, status, promo_id, used_customer_tag_id)'; - } - if ($table_name == 'rider_rating') - { - $load_stmt = 'LOAD DATA LOCAL INFILE \'' . $file . '\' INTO TABLE ' . $archive_tname . ' - FIELDS TERMINATED BY \'|\' - LINES TERMINATED BY \'\\r\\n\' - (id, rider_id, customer_id, jo_id, date_create, rating, comment)'; - } - if ($table_name == 'ticket') - { - $load_stmt = 'LOAD DATA LOCAL INFILE \'' . $file . '\' INTO TABLE ' . $archive_tname . ' - FIELDS TERMINATED BY \'|\' - LINES TERMINATED BY \'\\r\\n\' - (id, user_id, customer_id, date_create, status, ticket_type, other_ticket_type, first_name, last_name, - contact_num, details, job_order_id, plate_number, ticket_type_id, subticket_type_id, - source_of_awareness, remarks, other_description)'; - } - - // call load data infile - $this->loadDataFileForArchiveData($load_stmt); - } + return $files; } protected function getInvoiceItemArchiveData($row, $year) @@ -266,16 +235,7 @@ class GetJobOrderArchiveDataCommand extends Command $file = $this->createDataFileRelatedArchiveData($ii_data, 'invoice_item', 'a'); - if ($file != null) - { - $archive_tname = 'invoice_item_archive' . $year; - - $load_stmt = 'LOAD DATA LOCAL INFILE \'' . $file . '\' INTO TABLE ' . $archive_tname . ' - FIELDS TERMINATED BY \'|\' - LINES TERMINATED BY \'\\r\\n\' - (id, invoice_id, title, qty, price, battery_id)'; - // call load data infile - } + return $file; } protected function deleteDataFiles($tname) From a7c69886079f2ecb3fd28ea8c259840052c3a661 Mon Sep 17 00:00:00 2001 From: Korina Cordero Date: Wed, 20 Sep 2023 11:54:05 +0800 Subject: [PATCH 12/27] Add loading of data files. #762 --- src/Command/GetJobOrderArchiveDataCommand.php | 119 +++++++++++++----- 1 file changed, 88 insertions(+), 31 deletions(-) diff --git a/src/Command/GetJobOrderArchiveDataCommand.php b/src/Command/GetJobOrderArchiveDataCommand.php index 061bddc2..8115be68 100644 --- a/src/Command/GetJobOrderArchiveDataCommand.php +++ b/src/Command/GetJobOrderArchiveDataCommand.php @@ -65,7 +65,7 @@ class GetJobOrderArchiveDataCommand extends Command FROM job_order WHERE YEAR(date_create) = :year ORDER BY date_create - LIMIT 100'; + LIMIT 100000'; $query_stmt = $db->prepare($query_sql); $query_stmt->bindValue('year', $year, PDO::PARAM_STR); @@ -92,7 +92,7 @@ class GetJobOrderArchiveDataCommand extends Command // special since this is not directly related to JO but to invoice $this->deleteDataFiles('invoice_item'); - $related_files = []; + $archive_files = []; while ($row = $results->fetchAssociative()) { $jo_data['job_order'][$row['id']] = $this->createJobOrderArchiveData($row); @@ -120,48 +120,27 @@ class GetJobOrderArchiveDataCommand extends Command $query_stmt = $db->prepare($query_sql); $query_stmt->bindValue('id', $row['id']); - $files[] = call_user_func($callbackJO, $row, $query_stmt, $table_name, $year); + $files = call_user_func($callbackJO, $row, $query_stmt, $table_name, $year); - // TODO: need to figure out how to get the filenames foreach ($files as $key =>$file) { - $related_files[$key] = $file; + if ($file != null) + $archive_files[$key] = $file; } - } } } - error_log(print_r($related_files, true)); - // write the array into the file $file = $this->createDataFileRelatedArchiveData($jo_data, $jo_tname, 'w'); if ($file != null) { - $archive_tname = $jo_tname . '_archive_' . $year; - - // load statement for job order - $load_stmt = 'LOAD DATA LOCAL INFILE \'' . $file . '\' INTO TABLE ' . $archive_tname . ' - FIELDS TERMINATED BY \'|\' - LINES TERMINATED BY \'\\r\\n\' - (id, customer_id, cvehicle_id, rider_id, date_create, - date_schedule, date_fulfill, @coordinates, flag_advance, service_type, - source, date_cancel, status, delivery_instructions, delivery_address, - create_user_id, assign_user_id, date_assign, warranty_class, process_user_id, - hub_id, cancel_reason, ref_jo_id, tier1_notes, tier2_notes, - mode_of_payment, or_name, landmark, promo_detail, or_num, - trade_in_type, flag_rider_rating, flag_coolant, facilitated_hub_id, facilitated_type, - coord_long, coord_lat, priority, meta, status_autoassign, - first_name, last_name, plate_number, phone_mobile, no_trade_in_reason, - will_wait, reason_not_waiting, not_waiting_notes, delivery_status, emergency_type_id, - ownership_type_id, cust_location_id, source_of_awareness, remarks, initial_concern, - initial_concern_notes, gender, caller_classification, inventory_count) - SET coordinates=ST_GeomFromText(@geo_coordinates)'; - - // call load data infile - $this->loadDataFileForArchiveData($load_stmt); + $archive_files[$jo_tname] = $file; } + + // error_log(print_r($archive_files, true)); + $this->loadArchiveFiles($archive_files, $year); } protected function getRelatedArchiveData($row, $query_stmt, $table_name, $year) @@ -197,7 +176,8 @@ class GetJobOrderArchiveDataCommand extends Command // write the array into the file $file = $this->createDataFileRelatedArchiveData($data, $table_name, 'a'); - $files[$table_name] = $file; + if ($file != null) + $files[$table_name] = $file; return $files; } @@ -679,6 +659,83 @@ class GetJobOrderArchiveDataCommand extends Command return $data; } + protected function loadArchiveFiles($archive_files, $year) + { + foreach ($archive_files as $tname => $file) + { + $archive_tname = $tname . '_archive_' . $year; + + if ($tname == 'job_order') + { + // load statement for job order + $load_stmt = 'LOAD DATA LOCAL INFILE \'' . $file . '\' INTO TABLE ' . $archive_tname . ' + FIELDS TERMINATED BY \'|\' + LINES TERMINATED BY \'\\r\\n\' + (id, customer_id, cvehicle_id, rider_id, date_create, + date_schedule, date_fulfill, @coordinates, flag_advance, service_type, + source, date_cancel, status, delivery_instructions, delivery_address, + create_user_id, assign_user_id, date_assign, warranty_class, process_user_id, + hub_id, cancel_reason, ref_jo_id, tier1_notes, tier2_notes, + mode_of_payment, or_name, landmark, promo_detail, or_num, + trade_in_type, flag_rider_rating, flag_coolant, facilitated_hub_id, facilitated_type, + coord_long, coord_lat, priority, meta, status_autoassign, + first_name, last_name, plate_number, phone_mobile, no_trade_in_reason, + will_wait, reason_not_waiting, not_waiting_notes, delivery_status, emergency_type_id, + ownership_type_id, cust_location_id, source_of_awareness, remarks, initial_concern, + initial_concern_notes, gender, caller_classification, inventory_count) + SET coordinates=ST_GeomFromText(@geo_coordinates)'; + } + if ($tname == 'jo_event') + { + $load_stmt = 'LOAD DATA LOCAL INFILE \'' . $file . '\' INTO TABLE ' . $archive_tname . ' + FIELDS TERMINATED BY \'|\' + LINES TERMINATED BY \'\\r\\n\' + (id, create_user_id, job_order_id, date_create, date_happen, type_id, rider_id)'; + } + if ($tname == 'jo_rejection') + { + $load_stmt = 'LOAD DATA LOCAL INFILE \'' . $file . '\' INTO TABLE ' . $archive_tname . ' + FIELDS TERMINATED BY \'|\' + LINES TERMINATED BY \'\\r\\n\' + (id, user_id, hub_id, jo_id, date_create, reason, remarks, contact_person)'; + } + if ($tname == 'invoice') + { + $load_stmt = 'LOAD DATA LOCAL INFILE \'' . $file . '\' INTO TABLE ' . $archive_tname . ' + FIELDS TERMINATED BY \'|\' + LINES TERMINATED BY \'\\r\\n\' + (id, user_id, job_order_id, date_create, date_paid, discount, trade_in, vat, vat_exclusive_price, + total_price, status, promo_id, used_customer_tag_id)'; + } + if ($tname == 'rider_rating') + { + $load_stmt = 'LOAD DATA LOCAL INFILE \'' . $file . '\' INTO TABLE ' . $archive_tname . ' + FIELDS TERMINATED BY \'|\' + LINES TERMINATED BY \'\\r\\n\' + (id, rider_id, customer_id, jo_id, date_create, rating, comment)'; + } + if ($tname == 'ticket') + { + $load_stmt = 'LOAD DATA LOCAL INFILE \'' . $file . '\' INTO TABLE ' . $archive_tname . ' + FIELDS TERMINATED BY \'|\' + LINES TERMINATED BY \'\\r\\n\' + (id, user_id, customer_id, date_create, status, ticket_type, other_ticket_type, first_name, last_name, + contact_num, details, job_order_id, plate_number, ticket_type_id, subticket_type_id, + source_of_awareness, remarks, other_description)'; + } + if ($tname == 'invoice_item') + { + $load_stmt = 'LOAD DATA LOCAL INFILE \'' . $file . '\' INTO TABLE ' . $archive_tname . ' + FIELDS TERMINATED BY \'|\' + LINES TERMINATED BY \'\\r\\n\' + (id, invoice_id, title, qty, price, battery_id)'; + } + + // call load data infile + $this->loadDataFileForArchiveData($load_stmt); + } + } + protected function loadDataFileForArchiveData($load_stmt) { $conn = $this->em->getConnection(); From 19840c03058affa9f71c6ebb33fad4d6c5e7758c Mon Sep 17 00:00:00 2001 From: Korina Cordero Date: Wed, 20 Sep 2023 15:31:46 +0800 Subject: [PATCH 13/27] Improve performance. #762 --- src/Command/GetJobOrderArchiveDataCommand.php | 37 ++++++++++++++----- 1 file changed, 27 insertions(+), 10 deletions(-) diff --git a/src/Command/GetJobOrderArchiveDataCommand.php b/src/Command/GetJobOrderArchiveDataCommand.php index 8115be68..c1f24cc6 100644 --- a/src/Command/GetJobOrderArchiveDataCommand.php +++ b/src/Command/GetJobOrderArchiveDataCommand.php @@ -61,11 +61,12 @@ class GetJobOrderArchiveDataCommand extends Command $db = $this->em->getConnection(); + // TODO: improve performance. out of memory exception $query_sql = 'SELECT * FROM job_order WHERE YEAR(date_create) = :year - ORDER BY date_create - LIMIT 100000'; + ORDER BY date_create + LIMIT 150000'; $query_stmt = $db->prepare($query_sql); $query_stmt->bindValue('year', $year, PDO::PARAM_STR); @@ -93,13 +94,22 @@ class GetJobOrderArchiveDataCommand extends Command $this->deleteDataFiles('invoice_item'); $archive_files = []; + + $jo_id_list = []; + while ($row = $results->fetchAssociative()) { $jo_data['job_order'][$row['id']] = $this->createJobOrderArchiveData($row); + // add jo id to jo_id_list + $jo_id_list[] = $row['id']; + } + + // get all related data for job order + foreach ($jo_id_list as $jo_id) + { // foreach job order id we got from the first query, we get the JO related // data for that id from jo_event, invoice, ticket, jo_rejection, rider_rating - if (is_callable($callbackJO)) { foreach ($related_tables as $table_name) @@ -118,7 +128,7 @@ class GetJobOrderArchiveDataCommand extends Command $db = $this->em->getConnection(); $query_stmt = $db->prepare($query_sql); - $query_stmt->bindValue('id', $row['id']); + $query_stmt->bindValue('id', $jo_id); $files = call_user_func($callbackJO, $row, $query_stmt, $table_name, $year); @@ -149,6 +159,7 @@ class GetJobOrderArchiveDataCommand extends Command $data = []; $files = []; + $invoice_id_list = []; while ($q_row = $results->fetchAssociative()) { @@ -156,10 +167,8 @@ class GetJobOrderArchiveDataCommand extends Command // all invoice items for a specific invoice too if ($table_name == 'invoice') { - // get invoice item related data - $ii_file = $this->getInvoiceItemArchiveData($q_row, $year); - - $files['invoice_item'] = $ii_file; + // add invoice id to list + $invoice_id_list[] = $q_row['id']; } $fields = []; @@ -173,6 +182,14 @@ class GetJobOrderArchiveDataCommand extends Command $data[$table_name][$q_row['id']] = $fields; } + // get the invoice items for archiving + foreach ($invoice_id_list as $i_id) + { + $ii_file = $this->getInvoiceItemArchiveData($i_id, $year); + + $files['invoice_item'] = $ii_file; + } + // write the array into the file $file = $this->createDataFileRelatedArchiveData($data, $table_name, 'a'); @@ -182,14 +199,14 @@ class GetJobOrderArchiveDataCommand extends Command return $files; } - protected function getInvoiceItemArchiveData($row, $year) + protected function getInvoiceItemArchiveData($id, $year) { $db = $this->em->getConnection(); $query_sql = 'SELECT * FROM invoice_item WHERE invoice_id = :id'; $query_stmt = $db->prepare($query_sql); - $query_stmt->bindValue('id', $row['id']); + $query_stmt->bindValue('id', $id); $results = $query_stmt->executeQuery(); From fde9b367ecbdb9c45a94b1fd2bd2f18e16dad037 Mon Sep 17 00:00:00 2001 From: Korina Cordero Date: Fri, 22 Sep 2023 14:37:54 +0800 Subject: [PATCH 14/27] Set limit to jo query results. #762 --- src/Command/GetJobOrderArchiveDataCommand.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Command/GetJobOrderArchiveDataCommand.php b/src/Command/GetJobOrderArchiveDataCommand.php index c1f24cc6..7303b79a 100644 --- a/src/Command/GetJobOrderArchiveDataCommand.php +++ b/src/Command/GetJobOrderArchiveDataCommand.php @@ -66,7 +66,7 @@ class GetJobOrderArchiveDataCommand extends Command FROM job_order WHERE YEAR(date_create) = :year ORDER BY date_create - LIMIT 150000'; + LIMIT 125000'; $query_stmt = $db->prepare($query_sql); $query_stmt->bindValue('year', $year, PDO::PARAM_STR); From e65083bc4c34844b9f86b02764e36ab65ee84312 Mon Sep 17 00:00:00 2001 From: Korina Cordero Date: Tue, 26 Sep 2023 13:03:47 +0800 Subject: [PATCH 15/27] Add retrieval of ids for deletion. #762 --- src/Command/GetJobOrderArchiveDataCommand.php | 45 ++++++++++++++++++- 1 file changed, 43 insertions(+), 2 deletions(-) diff --git a/src/Command/GetJobOrderArchiveDataCommand.php b/src/Command/GetJobOrderArchiveDataCommand.php index 7303b79a..d6ba4582 100644 --- a/src/Command/GetJobOrderArchiveDataCommand.php +++ b/src/Command/GetJobOrderArchiveDataCommand.php @@ -66,7 +66,7 @@ class GetJobOrderArchiveDataCommand extends Command FROM job_order WHERE YEAR(date_create) = :year ORDER BY date_create - LIMIT 125000'; + LIMIT 100000'; $query_stmt = $db->prepare($query_sql); $query_stmt->bindValue('year', $year, PDO::PARAM_STR); @@ -151,6 +151,14 @@ class GetJobOrderArchiveDataCommand extends Command // error_log(print_r($archive_files, true)); $this->loadArchiveFiles($archive_files, $year); + + // need to get the list of invoice ids for deletion for invoice items + $invoice_id_list = $this->getInvoiceIds($jo_id_list); + + error_log(print_r($invoice_id_list, true)); + + // at this point, all the job order and related data have been archived into the database + $this->deleteData($jo_id_list, $invoice_id_list); } protected function getRelatedArchiveData($row, $query_stmt, $table_name, $year) @@ -256,7 +264,7 @@ class GetJobOrderArchiveDataCommand extends Command $cache_dir = __DIR__ . '/../../var/cache'; $file = $cache_dir . '/' . $table_name . '_archive.tab'; - error_log('opening file for archive - ' . $file); + // error_log('opening file for archive - ' . $file); $fp = fopen($file, $option); if ($fp === false) @@ -753,6 +761,39 @@ class GetJobOrderArchiveDataCommand extends Command } } + protected function deleteData($jo_id_list, $invoice_id_list) + { + $db = $this->em->getConnection(); + + // delete the invoice items first + } + + protected function getInvoiceIds($jo_id_list) + { + $invoice_id_list = []; + + $db = $this->em->getConnection(); + + $query_sql = 'SELECT id FROM invoice WHERE job_order_id = :id'; + + $query_stmt = $db->prepare($query_sql); + + foreach ($jo_id_list as $jo_id) + { + // need to get the invoice ids for the invoice items to delete + $query_stmt->bindValue('id', $jo_id); + + $results = $query_stmt->executeQuery(); + + while ($row = $results->fetchAssociative()) + { + $invoice_id_list[] = $row['id']; + } + } + + return $invoice_id_list; + } + protected function loadDataFileForArchiveData($load_stmt) { $conn = $this->em->getConnection(); From 645e8a63fcc0a058d3a163052f66455860a27a07 Mon Sep 17 00:00:00 2001 From: Korina Cordero Date: Tue, 26 Sep 2023 16:28:40 +0800 Subject: [PATCH 16/27] Add deletion. #762 --- src/Command/GetJobOrderArchiveDataCommand.php | 66 ++++++++++++------- 1 file changed, 43 insertions(+), 23 deletions(-) diff --git a/src/Command/GetJobOrderArchiveDataCommand.php b/src/Command/GetJobOrderArchiveDataCommand.php index d6ba4582..cc0fab18 100644 --- a/src/Command/GetJobOrderArchiveDataCommand.php +++ b/src/Command/GetJobOrderArchiveDataCommand.php @@ -39,6 +39,10 @@ class GetJobOrderArchiveDataCommand extends Command protected function execute(InputInterface $input, OutputInterface $output) { + $current_datetime = new DateTime('now'); + + error_log('Archive start time ' . $current_datetime->format('Y-m-d H:i')); + // get year to archive $year = $input->getArgument('year'); @@ -61,20 +65,26 @@ class GetJobOrderArchiveDataCommand extends Command $db = $this->em->getConnection(); - // TODO: improve performance. out of memory exception + // set the pdo connection to use unbuffered query so we don't run out of memory + // when processing the job order related tables + $db->getWrappedConnection()->setAttribute(PDO::MYSQL_ATTR_USE_BUFFERED_QUERY, false); + $query_sql = 'SELECT * FROM job_order WHERE YEAR(date_create) = :year - ORDER BY date_create - LIMIT 100000'; + ORDER BY date_create'; $query_stmt = $db->prepare($query_sql); $query_stmt->bindValue('year', $year, PDO::PARAM_STR); - $callback = ['App\Command\GetJobOrderArchiveDataCommand', 'getRelatedArchiveData']; + $callback = [$this, 'getRelatedArchiveData']; $this->getArchiveData($query_stmt, $callback, 'job_order', $year); + $current_datetime = new DateTime('now'); + + error_log('Archive end time ' . $current_datetime->format('Y-m-d H:i')); + return 0; } @@ -87,11 +97,11 @@ class GetJobOrderArchiveDataCommand extends Command // delete the related data files foreach ($related_tables as $tname) { - $this->deleteDataFiles($tname); + $this->deleteDataFiles($tname, $year); } // special since this is not directly related to JO but to invoice - $this->deleteDataFiles('invoice_item'); + $this->deleteDataFiles('invoice_item', $year); $archive_files = []; @@ -114,18 +124,21 @@ class GetJobOrderArchiveDataCommand extends Command { foreach ($related_tables as $table_name) { - if (($table_name == 'jo_event') || - ($table_name == 'invoice') || - ($table_name == 'ticket')) - { - $query_sql = 'SELECT * FROM ' . $table_name . ' WHERE job_order_id = :id'; - } - else - { - $query_sql = 'SELECT * FROM ' . $table_name . ' WHERE jo_id = :id'; + switch ($table_name) { + case 'jo_event': + case 'invoice': + case 'ticket': + $query_sql = 'SELECT * FROM ' . $table_name . ' WHERE job_order_id = :id'; + break; + case 'jo_rejection': + case 'rider_rating': + $query_sql = 'SELECT * FROM ' . $table_name . ' WHERE jo_id = :id'; + default: + break; } $db = $this->em->getConnection(); + $db->getWrappedConnection()->setAttribute(PDO::MYSQL_ATTR_USE_BUFFERED_QUERY, false); $query_stmt = $db->prepare($query_sql); $query_stmt->bindValue('id', $jo_id); @@ -142,7 +155,7 @@ class GetJobOrderArchiveDataCommand extends Command } // write the array into the file - $file = $this->createDataFileRelatedArchiveData($jo_data, $jo_tname, 'w'); + $file = $this->createDataFileRelatedArchiveData($jo_data, $jo_tname, $year, 'w'); if ($file != null) { @@ -155,7 +168,7 @@ class GetJobOrderArchiveDataCommand extends Command // need to get the list of invoice ids for deletion for invoice items $invoice_id_list = $this->getInvoiceIds($jo_id_list); - error_log(print_r($invoice_id_list, true)); + // error_log(print_r($invoice_id_list, true)); // at this point, all the job order and related data have been archived into the database $this->deleteData($jo_id_list, $invoice_id_list); @@ -199,7 +212,7 @@ class GetJobOrderArchiveDataCommand extends Command } // write the array into the file - $file = $this->createDataFileRelatedArchiveData($data, $table_name, 'a'); + $file = $this->createDataFileRelatedArchiveData($data, $table_name, $year, 'a'); if ($file != null) $files[$table_name] = $file; @@ -210,6 +223,7 @@ class GetJobOrderArchiveDataCommand extends Command protected function getInvoiceItemArchiveData($id, $year) { $db = $this->em->getConnection(); + $db->getWrappedConnection()->setAttribute(PDO::MYSQL_ATTR_USE_BUFFERED_QUERY, false); $query_sql = 'SELECT * FROM invoice_item WHERE invoice_id = :id'; @@ -238,23 +252,23 @@ class GetJobOrderArchiveDataCommand extends Command ]; } - $file = $this->createDataFileRelatedArchiveData($ii_data, 'invoice_item', 'a'); + $file = $this->createDataFileRelatedArchiveData($ii_data, 'invoice_item', $year, 'a'); return $file; } - protected function deleteDataFiles($tname) + protected function deleteDataFiles($tname, $year) { // cache directory $cache_dir = __DIR__ . '/../../var/cache'; - $file = $cache_dir . '/' . $tname . '_archive.tab'; + $file = $cache_dir . '/' . $tname . '_archive_' . $year .'.tab'; if (file_exists($file)) unlink($file); } - protected function createDataFileRelatedArchiveData($archive_data, $table_name, $option) + protected function createDataFileRelatedArchiveData($archive_data, $table_name, $year, $option) { if (isset($archive_data[$table_name])) { @@ -263,7 +277,7 @@ class GetJobOrderArchiveDataCommand extends Command // cache directory $cache_dir = __DIR__ . '/../../var/cache'; - $file = $cache_dir . '/' . $table_name . '_archive.tab'; + $file = $cache_dir . '/' . $table_name . '_archive_'. $year . '.tab'; // error_log('opening file for archive - ' . $file); $fp = fopen($file, $option); @@ -766,6 +780,12 @@ class GetJobOrderArchiveDataCommand extends Command $db = $this->em->getConnection(); // delete the invoice items first + $inv_ids = str_repeat('?,', count($invoice_id_list) - 1) . '?'; + + $ii_del_sql = 'DELETE FROM invoice_item WHERE invoice_id IN ($inv_ids)'; + $ii_stmt = $db->prepare($ii_del_sql); + + $ii_stmt->execute($invoice_id_list); } protected function getInvoiceIds($jo_id_list) From d78dc72c35154cfb98048ae598b9e4bb6aff47a8 Mon Sep 17 00:00:00 2001 From: Korina Cordero Date: Wed, 27 Sep 2023 14:54:36 +0800 Subject: [PATCH 17/27] Fix deletion issues for related tables. #762 --- src/Command/GetJobOrderArchiveDataCommand.php | 38 +++++++++++++++++-- 1 file changed, 35 insertions(+), 3 deletions(-) diff --git a/src/Command/GetJobOrderArchiveDataCommand.php b/src/Command/GetJobOrderArchiveDataCommand.php index cc0fab18..3f9d3938 100644 --- a/src/Command/GetJobOrderArchiveDataCommand.php +++ b/src/Command/GetJobOrderArchiveDataCommand.php @@ -133,6 +133,7 @@ class GetJobOrderArchiveDataCommand extends Command case 'jo_rejection': case 'rider_rating': $query_sql = 'SELECT * FROM ' . $table_name . ' WHERE jo_id = :id'; + break; default: break; } @@ -171,7 +172,7 @@ class GetJobOrderArchiveDataCommand extends Command // error_log(print_r($invoice_id_list, true)); // at this point, all the job order and related data have been archived into the database - $this->deleteData($jo_id_list, $invoice_id_list); + $this->deleteData($jo_id_list, $invoice_id_list, $related_tables); } protected function getRelatedArchiveData($row, $query_stmt, $table_name, $year) @@ -775,17 +776,48 @@ class GetJobOrderArchiveDataCommand extends Command } } - protected function deleteData($jo_id_list, $invoice_id_list) + protected function deleteData($jo_id_list, $invoice_id_list, $related_tables) { $db = $this->em->getConnection(); // delete the invoice items first $inv_ids = str_repeat('?,', count($invoice_id_list) - 1) . '?'; - $ii_del_sql = 'DELETE FROM invoice_item WHERE invoice_id IN ($inv_ids)'; + $ii_del_sql = 'DELETE FROM invoice_item WHERE invoice_id IN (' . $inv_ids . ')'; $ii_stmt = $db->prepare($ii_del_sql); $ii_stmt->execute($invoice_id_list); + + // delete from invoice, jo_rejection, rider_rating, ticket, and jo_event + $jo_ids = str_repeat('?,', count($jo_id_list) - 1) . '?'; + + foreach ($related_tables as $table_name) + { + switch ($table_name) { + case 'jo_event': + case 'invoice': + case 'ticket': + $related_del_sql = 'DELETE FROM ' . $table_name . ' WHERE job_order_id IN ('. $jo_ids . ')'; + break; + case 'jo_rejection': + case 'rider_rating': + $related_del_sql = 'DELETE FROM ' . $table_name . ' WHERE jo_id IN (' . $jo_ids. ')'; + break; + default: + break; + } + + $related_stmt = $db->prepare($related_del_sql); + + $related_stmt->execute($jo_id_list); + } + + // TODO: hitting a snag here if JO to be deleted is a reference JO for another JO + // delete from job order last + $jo_del_sql = 'DELETE FROM job_order WHERE id IN (' . $jo_ids . ')'; + $jo_stmt = $db->prepare($jo_del_sql); + + $jo_stmt->execute($jo_id_list); } protected function getInvoiceIds($jo_id_list) From 96bd783ae98e5762a1e509bd59a0da025879fccf Mon Sep 17 00:00:00 2001 From: Korina Cordero Date: Thu, 28 Sep 2023 14:15:41 +0800 Subject: [PATCH 18/27] Add reference_jo_id field to hold the job order id for reference orders. Add a command to decouple reference job order from job order. #762 --- .../SetJobOrderReferenceJOIdCommand.php | 53 +++++++++++++++++++ src/Entity/JobOrder.php | 19 ++++++- 2 files changed, 71 insertions(+), 1 deletion(-) create mode 100644 src/Command/SetJobOrderReferenceJOIdCommand.php diff --git a/src/Command/SetJobOrderReferenceJOIdCommand.php b/src/Command/SetJobOrderReferenceJOIdCommand.php new file mode 100644 index 00000000..20ee236a --- /dev/null +++ b/src/Command/SetJobOrderReferenceJOIdCommand.php @@ -0,0 +1,53 @@ +em = $em; + + parent::__construct(); + } + + protected function configure() + { + $this->setName('joborder:setreferencejoid') + ->setDescription('Set job order reference jo id for existing job orders.') + ->setHelp('Set job order reference jo id for existing job orders. Decoupling job order from reference job order.'); + } + + protected function execute(InputInterface $input, OutputInterface $output) + { + // get the job orders where ref_jo is not null + $query = $this->em->createQuery('SELECT jo FROM App\Entity\JobOrder jo WHERE jo.ref_jo IS NOT NULL'); + + $jos = $query->getResult(); + + foreach ($jos as $jo) + { + $ref_jo_id = $jo->getReferenceJO()->getID(); + + error_log('Setting reference jo id ' . $ref_jo_id . ' for job order ' . $jo->getID()); + + $jo->setReferenceJOId($ref_jo_id); + + // set the ref_jo to null to decouple ref jo and jo + $jo->setReferenceJO(null); + } + + $this->em->flush(); + + return 0; + } +} diff --git a/src/Entity/JobOrder.php b/src/Entity/JobOrder.php index c71f1846..a410f9e0 100644 --- a/src/Entity/JobOrder.php +++ b/src/Entity/JobOrder.php @@ -435,6 +435,12 @@ class JobOrder */ protected $inventory_count; + // reference JO id. We are now decoupling job order from itself + /** + * @ORM\Column(type="integer", nullable=true) + */ + protected $reference_jo_id; + public function __construct() { $this->date_create = new DateTime(); @@ -784,7 +790,7 @@ class JobOrder return $this->tickets; } - public function setReferenceJO(JobOrder $ref_jo) + public function setReferenceJO(JobOrder $ref_jo = null) { $this->ref_jo = $ref_jo; return $this; @@ -1237,4 +1243,15 @@ class JobOrder return $this->inventory_count; } + public function setReferenceJOId($reference_jo_id) + { + $this->reference_jo_id = $reference_jo_id; + return $this; + } + + public function getReferenceJOId() + { + return $this->reference_jo_id; + } + } From 4fa5ea156f4c90d326b138c4ed50274e08a9e90a Mon Sep 17 00:00:00 2001 From: Korina Cordero Date: Thu, 28 Sep 2023 14:41:49 +0800 Subject: [PATCH 19/27] Modify saving of reference job order for job order. #762 --- src/Service/JobOrderHandler/ResqJobOrderHandler.php | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Service/JobOrderHandler/ResqJobOrderHandler.php b/src/Service/JobOrderHandler/ResqJobOrderHandler.php index 3e86f434..28eecea6 100644 --- a/src/Service/JobOrderHandler/ResqJobOrderHandler.php +++ b/src/Service/JobOrderHandler/ResqJobOrderHandler.php @@ -527,7 +527,7 @@ class ResqJobOrderHandler implements JobOrderHandlerInterface if (empty($ref_jo)) { $error_array['ref_jo'] = 'Invalid reference job order specified.'; } else { - $jo->setReferenceJO($ref_jo); + $jo->setReferenceJOId($ref_jo->getID()); } } @@ -2129,7 +2129,7 @@ class ResqJobOrderHandler implements JobOrderHandlerInterface if (empty($ref_jo)) { $error_array['ref_jo'] = 'Invalid reference job order specified.'; } else { - $jo->setReferenceJO($ref_jo); + $jo->setReferenceJOId($ref_jo->getID()); } } From 12d9c14a03501520bab21573e0b4217cef6d7c57 Mon Sep 17 00:00:00 2001 From: Korina Cordero Date: Thu, 28 Sep 2023 16:00:28 +0800 Subject: [PATCH 20/27] Add updating of rider's active and current job orders. #762 --- src/Command/GetJobOrderArchiveDataCommand.php | 50 +++++++++++++++++-- 1 file changed, 47 insertions(+), 3 deletions(-) diff --git a/src/Command/GetJobOrderArchiveDataCommand.php b/src/Command/GetJobOrderArchiveDataCommand.php index 3f9d3938..563f77d0 100644 --- a/src/Command/GetJobOrderArchiveDataCommand.php +++ b/src/Command/GetJobOrderArchiveDataCommand.php @@ -169,7 +169,13 @@ class GetJobOrderArchiveDataCommand extends Command // need to get the list of invoice ids for deletion for invoice items $invoice_id_list = $this->getInvoiceIds($jo_id_list); - // error_log(print_r($invoice_id_list, true)); + // need to get the list of riders whose active_jo_id or current_jo_id is + // set to very old JOs + $rider_id_list = $this->findRiderIDs($jo_id_list); + + // update rider's active_jo_id and current_jo_id to null + // so we can delete the old job orders (foreign key constraint) + $this->updateRiderJobOrders($rider_id_list); // at this point, all the job order and related data have been archived into the database $this->deleteData($jo_id_list, $invoice_id_list, $related_tables); @@ -812,7 +818,6 @@ class GetJobOrderArchiveDataCommand extends Command $related_stmt->execute($jo_id_list); } - // TODO: hitting a snag here if JO to be deleted is a reference JO for another JO // delete from job order last $jo_del_sql = 'DELETE FROM job_order WHERE id IN (' . $jo_ids . ')'; $jo_stmt = $db->prepare($jo_del_sql); @@ -846,12 +851,51 @@ class GetJobOrderArchiveDataCommand extends Command return $invoice_id_list; } + protected function findRiderIDs($jo_id_list) + { + $rider_id_list = []; + + $db = $this->em->getConnection(); + + $rider_sql = 'SELECT id FROM rider WHERE ((current_jo_id = :jo_id) OR (active_jo_id = :jo_id))'; + $rider_stmt = $db->prepare($rider_sql); + + // find the riders with current_jo_id or active_jo_id are still set to the old JOs + foreach ($jo_id_list as $jo_id) + { + $rider_stmt->bindValue('jo_id', $jo_id); + + $results = $rider_stmt->executeQuery(); + + while ($row = $results->fetchAssociative()) + { + $rider_id_list[] = $row['id']; + } + } + + return $rider_id_list; + } + + protected function updateRiderJobOrders($rider_id_list) + { + $db = $this->em->getConnection(); + + $update_sql = 'UPDATE rider SET current_jo_id = NULL, SET active_jo_id = NULL WHERE id = :id'; + $update_stmt = $db->prepare($update_sql); + + foreach ($rider_id_list as $rider_id) + { + $update_stmt->execute([ + 'id' => $rider_id, + ]); + } + } + protected function loadDataFileForArchiveData($load_stmt) { $conn = $this->em->getConnection(); // this statement is for job order - // TODO: make for other tables $stmt = $conn->prepare($load_stmt); $result = $stmt->execute(); From 9bf5d5851a17b2df58f0de9996661811902bb937 Mon Sep 17 00:00:00 2001 From: Korina Cordero Date: Thu, 28 Sep 2023 18:22:56 +0800 Subject: [PATCH 21/27] Add updating of rider's current and active jo ids to null. #762 --- src/Command/GetJobOrderArchiveDataCommand.php | 41 ++++--------------- 1 file changed, 7 insertions(+), 34 deletions(-) diff --git a/src/Command/GetJobOrderArchiveDataCommand.php b/src/Command/GetJobOrderArchiveDataCommand.php index 563f77d0..4fe27fb7 100644 --- a/src/Command/GetJobOrderArchiveDataCommand.php +++ b/src/Command/GetJobOrderArchiveDataCommand.php @@ -169,13 +169,9 @@ class GetJobOrderArchiveDataCommand extends Command // need to get the list of invoice ids for deletion for invoice items $invoice_id_list = $this->getInvoiceIds($jo_id_list); - // need to get the list of riders whose active_jo_id or current_jo_id is - // set to very old JOs - $rider_id_list = $this->findRiderIDs($jo_id_list); - // update rider's active_jo_id and current_jo_id to null // so we can delete the old job orders (foreign key constraint) - $this->updateRiderJobOrders($rider_id_list); + $this->updateRiderJobOrders($jo_id_list); // at this point, all the job order and related data have been archived into the database $this->deleteData($jo_id_list, $invoice_id_list, $related_tables); @@ -830,6 +826,7 @@ class GetJobOrderArchiveDataCommand extends Command $invoice_id_list = []; $db = $this->em->getConnection(); + $db->getWrappedConnection()->setAttribute(PDO::MYSQL_ATTR_USE_BUFFERED_QUERY, false); $query_sql = 'SELECT id FROM invoice WHERE job_order_id = :id'; @@ -851,42 +848,18 @@ class GetJobOrderArchiveDataCommand extends Command return $invoice_id_list; } - protected function findRiderIDs($jo_id_list) - { - $rider_id_list = []; - - $db = $this->em->getConnection(); - - $rider_sql = 'SELECT id FROM rider WHERE ((current_jo_id = :jo_id) OR (active_jo_id = :jo_id))'; - $rider_stmt = $db->prepare($rider_sql); - - // find the riders with current_jo_id or active_jo_id are still set to the old JOs - foreach ($jo_id_list as $jo_id) - { - $rider_stmt->bindValue('jo_id', $jo_id); - - $results = $rider_stmt->executeQuery(); - - while ($row = $results->fetchAssociative()) - { - $rider_id_list[] = $row['id']; - } - } - - return $rider_id_list; - } - - protected function updateRiderJobOrders($rider_id_list) + protected function updateRiderJobOrders($jo_id_list) { $db = $this->em->getConnection(); + $db->getWrappedConnection()->setAttribute(PDO::MYSQL_ATTR_USE_BUFFERED_QUERY, false); - $update_sql = 'UPDATE rider SET current_jo_id = NULL, SET active_jo_id = NULL WHERE id = :id'; + $update_sql = 'UPDATE rider SET current_jo_id = NULL, active_jo_id = NULL WHERE (current_jo_id = :id OR active_jo_id = :id)'; $update_stmt = $db->prepare($update_sql); - foreach ($rider_id_list as $rider_id) + foreach ($jo_id_list as $jo_id) { $update_stmt->execute([ - 'id' => $rider_id, + 'id' => $jo_id, ]); } } From 3e0084630f57c80e8afa3715fb59050bc0af7598 Mon Sep 17 00:00:00 2001 From: Korina Cordero Date: Thu, 28 Sep 2023 19:07:11 +0800 Subject: [PATCH 22/27] Add TODO for updating rider. #762 --- src/Command/GetJobOrderArchiveDataCommand.php | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/Command/GetJobOrderArchiveDataCommand.php b/src/Command/GetJobOrderArchiveDataCommand.php index 4fe27fb7..e266f6dd 100644 --- a/src/Command/GetJobOrderArchiveDataCommand.php +++ b/src/Command/GetJobOrderArchiveDataCommand.php @@ -850,6 +850,11 @@ class GetJobOrderArchiveDataCommand extends Command protected function updateRiderJobOrders($jo_id_list) { + // TODO: memory exception while updating rider table. + // need to figure out how to make this not run out of memory. + // first draft had us selecting and finding the rider ids. maybe we should bring + // that back with the set attribute? I can't remember if I ran it with that one :( + // so that updating will have a smaller result set. $db = $this->em->getConnection(); $db->getWrappedConnection()->setAttribute(PDO::MYSQL_ATTR_USE_BUFFERED_QUERY, false); From 67635b07d593c57e5076ff95b5c3d002b9691ce2 Mon Sep 17 00:00:00 2001 From: Korina Cordero Date: Thu, 28 Sep 2023 19:35:10 +0800 Subject: [PATCH 23/27] Modify updating of rider table. #762 --- src/Command/GetJobOrderArchiveDataCommand.php | 24 ++++++++----------- 1 file changed, 10 insertions(+), 14 deletions(-) diff --git a/src/Command/GetJobOrderArchiveDataCommand.php b/src/Command/GetJobOrderArchiveDataCommand.php index e266f6dd..e5ac289d 100644 --- a/src/Command/GetJobOrderArchiveDataCommand.php +++ b/src/Command/GetJobOrderArchiveDataCommand.php @@ -850,23 +850,19 @@ class GetJobOrderArchiveDataCommand extends Command protected function updateRiderJobOrders($jo_id_list) { - // TODO: memory exception while updating rider table. - // need to figure out how to make this not run out of memory. - // first draft had us selecting and finding the rider ids. maybe we should bring - // that back with the set attribute? I can't remember if I ran it with that one :( - // so that updating will have a smaller result set. + // TODO: test this $db = $this->em->getConnection(); - $db->getWrappedConnection()->setAttribute(PDO::MYSQL_ATTR_USE_BUFFERED_QUERY, false); - $update_sql = 'UPDATE rider SET current_jo_id = NULL, active_jo_id = NULL WHERE (current_jo_id = :id OR active_jo_id = :id)'; - $update_stmt = $db->prepare($update_sql); + $jo_ids = str_repeat('?,', count($jo_id_list) - 1) . '?'; - foreach ($jo_id_list as $jo_id) - { - $update_stmt->execute([ - 'id' => $jo_id, - ]); - } + $update_curr_sql = 'UPDATE rider SET current_jo_id = NULL WHERE current_jo_id IN (' . $jo_ids . ')'; + $update_curr_stmt = $db->prepare($update_curr_sql); + + $update_active_sql = 'UPDATE rider SET active_jo_id = NULL WHERE active_jo_id IN (' . $jo_ids . ')'; + $udpate_active_stmt = $db->prepare($update_active_sql); + + $update_curr_stmt->execute($jo_id_list); + $update_active_stmt->execute($jo_id_list); } protected function loadDataFileForArchiveData($load_stmt) From f78e37657666a61c426cb8d0ffa2af6dbc91fc77 Mon Sep 17 00:00:00 2001 From: Korina Cordero Date: Fri, 29 Sep 2023 20:47:13 +0800 Subject: [PATCH 24/27] Fix issues found during testing. #762 --- src/Command/GetJobOrderArchiveDataCommand.php | 96 ++++++++++--------- 1 file changed, 49 insertions(+), 47 deletions(-) diff --git a/src/Command/GetJobOrderArchiveDataCommand.php b/src/Command/GetJobOrderArchiveDataCommand.php index e5ac289d..1ff9ac6d 100644 --- a/src/Command/GetJobOrderArchiveDataCommand.php +++ b/src/Command/GetJobOrderArchiveDataCommand.php @@ -106,6 +106,7 @@ class GetJobOrderArchiveDataCommand extends Command $archive_files = []; $jo_id_list = []; + $ii_id_list = []; while ($row = $results->fetchAssociative()) { @@ -115,6 +116,24 @@ class GetJobOrderArchiveDataCommand extends Command $jo_id_list[] = $row['id']; } + // write the array into the file + $file = $this->createDataFileRelatedArchiveData($jo_data, $jo_tname, $year, 'w'); + + if ($file != null) + { + $archive_files[$jo_tname] = $file; + } + + // error_log('jo_data total ' . count($jo_data['job_order'])); + // error_log('jo id list total ' . count($jo_id_list)); + + unset($jo_data); + + // load the job order archive file for job order into the database + $this->loadArchiveFiles($archive_files, $year); + + unset($archive_files[$jo_tname]); + // get all related data for job order foreach ($jo_id_list as $jo_id) { @@ -155,26 +174,16 @@ class GetJobOrderArchiveDataCommand extends Command } } - // write the array into the file - $file = $this->createDataFileRelatedArchiveData($jo_data, $jo_tname, $year, 'w'); - - if ($file != null) - { - $archive_files[$jo_tname] = $file; - } - - // error_log(print_r($archive_files, true)); + // TODO: can we maybe find a way to not load the archive files successively? + // we get the memory exception after loading the data to the last table, jo_rejection. $this->loadArchiveFiles($archive_files, $year); - // need to get the list of invoice ids for deletion for invoice items - $invoice_id_list = $this->getInvoiceIds($jo_id_list); - // update rider's active_jo_id and current_jo_id to null // so we can delete the old job orders (foreign key constraint) $this->updateRiderJobOrders($jo_id_list); // at this point, all the job order and related data have been archived into the database - $this->deleteData($jo_id_list, $invoice_id_list, $related_tables); + $this->deleteData($jo_id_list, $related_tables); } protected function getRelatedArchiveData($row, $query_stmt, $table_name, $year) @@ -207,6 +216,7 @@ class GetJobOrderArchiveDataCommand extends Command } // get the invoice items for archiving + $ii_id_list = []; foreach ($invoice_id_list as $i_id) { $ii_file = $this->getInvoiceItemArchiveData($i_id, $year); @@ -217,6 +227,8 @@ class GetJobOrderArchiveDataCommand extends Command // write the array into the file $file = $this->createDataFileRelatedArchiveData($data, $table_name, $year, 'a'); + unset($data); + if ($file != null) $files[$table_name] = $file; @@ -236,6 +248,7 @@ class GetJobOrderArchiveDataCommand extends Command $results = $query_stmt->executeQuery(); $ii_data = []; + $ii_id_list = []; while ($ii_row = $results->fetchAssociative()) { $id = $ii_row['id']; @@ -253,10 +266,19 @@ class GetJobOrderArchiveDataCommand extends Command $price, $battery_id ]; + + $ii_id_list[] = $id; } $file = $this->createDataFileRelatedArchiveData($ii_data, 'invoice_item', $year, 'a'); + unset($ii_data); + + // special case, delete the invoice items already + // so that we don't have to query for the invoice items to delete + if (count($ii_id_list) > 0) + $this->deleteInvoiceItems($ii_id_list); + return $file; } @@ -778,17 +800,22 @@ class GetJobOrderArchiveDataCommand extends Command } } - protected function deleteData($jo_id_list, $invoice_id_list, $related_tables) + protected function deleteInvoiceItems($ii_id_list) { $db = $this->em->getConnection(); // delete the invoice items first - $inv_ids = str_repeat('?,', count($invoice_id_list) - 1) . '?'; + $ii_ids = str_repeat('?,', count($ii_id_list) - 1) . '?'; - $ii_del_sql = 'DELETE FROM invoice_item WHERE invoice_id IN (' . $inv_ids . ')'; + $ii_del_sql = 'DELETE FROM invoice_item WHERE id IN (' . $ii_ids . ')'; $ii_stmt = $db->prepare($ii_del_sql); - $ii_stmt->execute($invoice_id_list); + $ii_stmt->execute($ii_id_list); + } + + protected function deleteData($jo_id_list, $related_tables) + { + $db = $this->em->getConnection(); // delete from invoice, jo_rejection, rider_rating, ticket, and jo_event $jo_ids = str_repeat('?,', count($jo_id_list) - 1) . '?'; @@ -821,36 +848,8 @@ class GetJobOrderArchiveDataCommand extends Command $jo_stmt->execute($jo_id_list); } - protected function getInvoiceIds($jo_id_list) - { - $invoice_id_list = []; - - $db = $this->em->getConnection(); - $db->getWrappedConnection()->setAttribute(PDO::MYSQL_ATTR_USE_BUFFERED_QUERY, false); - - $query_sql = 'SELECT id FROM invoice WHERE job_order_id = :id'; - - $query_stmt = $db->prepare($query_sql); - - foreach ($jo_id_list as $jo_id) - { - // need to get the invoice ids for the invoice items to delete - $query_stmt->bindValue('id', $jo_id); - - $results = $query_stmt->executeQuery(); - - while ($row = $results->fetchAssociative()) - { - $invoice_id_list[] = $row['id']; - } - } - - return $invoice_id_list; - } - protected function updateRiderJobOrders($jo_id_list) { - // TODO: test this $db = $this->em->getConnection(); $jo_ids = str_repeat('?,', count($jo_id_list) - 1) . '?'; @@ -859,17 +858,20 @@ class GetJobOrderArchiveDataCommand extends Command $update_curr_stmt = $db->prepare($update_curr_sql); $update_active_sql = 'UPDATE rider SET active_jo_id = NULL WHERE active_jo_id IN (' . $jo_ids . ')'; - $udpate_active_stmt = $db->prepare($update_active_sql); + $update_active_stmt = $db->prepare($update_active_sql); $update_curr_stmt->execute($jo_id_list); $update_active_stmt->execute($jo_id_list); + + unset($update_curr_stmt); + unset($update_active_stmt); } protected function loadDataFileForArchiveData($load_stmt) { $conn = $this->em->getConnection(); + $conn->getWrappedConnection()->setAttribute(PDO::MYSQL_ATTR_USE_BUFFERED_QUERY, false); - // this statement is for job order $stmt = $conn->prepare($load_stmt); $result = $stmt->execute(); From 85483fb744d963d485de3bd079fc7911e22054c1 Mon Sep 17 00:00:00 2001 From: Korina Cordero Date: Tue, 3 Oct 2023 22:08:46 +0800 Subject: [PATCH 25/27] Added batch update for rider table. #762 --- src/Command/GetJobOrderArchiveDataCommand.php | 48 ++++++++++++++----- 1 file changed, 36 insertions(+), 12 deletions(-) diff --git a/src/Command/GetJobOrderArchiveDataCommand.php b/src/Command/GetJobOrderArchiveDataCommand.php index 1ff9ac6d..a6540080 100644 --- a/src/Command/GetJobOrderArchiveDataCommand.php +++ b/src/Command/GetJobOrderArchiveDataCommand.php @@ -174,16 +174,23 @@ class GetJobOrderArchiveDataCommand extends Command } } - // TODO: can we maybe find a way to not load the archive files successively? - // we get the memory exception after loading the data to the last table, jo_rejection. $this->loadArchiveFiles($archive_files, $year); - // update rider's active_jo_id and current_jo_id to null - // so we can delete the old job orders (foreign key constraint) - $this->updateRiderJobOrders($jo_id_list); + error_log('Done loading files into database...'); - // at this point, all the job order and related data have been archived into the database - $this->deleteData($jo_id_list, $related_tables); + $jo_id_array = array_chunk($jo_id_list, 10000); + + unset($jo_id_list); + + foreach($jo_id_array as $key => $jo_ids) + { + // update rider's active_jo_id and current_jo_id to null + // so we can delete the old job orders (foreign key constraint) + $this->updateRiderActiveJobOrders($jo_ids); + $this->updateRiderCurrentJobOrders($jo_ids); + + $this->deleteData($jo_ids, $related_tables); + } } protected function getRelatedArchiveData($row, $query_stmt, $table_name, $year) @@ -279,6 +286,8 @@ class GetJobOrderArchiveDataCommand extends Command if (count($ii_id_list) > 0) $this->deleteInvoiceItems($ii_id_list); + unset($ii_id_list); + return $file; } @@ -848,23 +857,38 @@ class GetJobOrderArchiveDataCommand extends Command $jo_stmt->execute($jo_id_list); } - protected function updateRiderJobOrders($jo_id_list) + protected function updateRiderActiveJobOrders($jo_id_list) { $db = $this->em->getConnection(); + $db->getWrappedConnection()->setAttribute(PDO::MYSQL_ATTR_USE_BUFFERED_QUERY, false); + + $jo_ids = str_repeat('?,', count($jo_id_list) - 1) . '?'; + + $update_active_sql = 'UPDATE rider SET active_jo_id = NULL WHERE active_jo_id IN (' . $jo_ids . ')'; + $update_active_stmt = $db->prepare($update_active_sql); + + error_log('Updating active rider job orders...'); + + $update_active_stmt->execute($jo_id_list); + + unset($update_active_stmt); + } + + protected function updateRiderCurrentJobOrders($jo_id_list) + { + $db = $this->em->getConnection(); + $db->getWrappedConnection()->setAttribute(PDO::MYSQL_ATTR_USE_BUFFERED_QUERY, false); $jo_ids = str_repeat('?,', count($jo_id_list) - 1) . '?'; $update_curr_sql = 'UPDATE rider SET current_jo_id = NULL WHERE current_jo_id IN (' . $jo_ids . ')'; $update_curr_stmt = $db->prepare($update_curr_sql); - $update_active_sql = 'UPDATE rider SET active_jo_id = NULL WHERE active_jo_id IN (' . $jo_ids . ')'; - $update_active_stmt = $db->prepare($update_active_sql); + error_log('Updating current rider job orders...'); $update_curr_stmt->execute($jo_id_list); - $update_active_stmt->execute($jo_id_list); unset($update_curr_stmt); - unset($update_active_stmt); } protected function loadDataFileForArchiveData($load_stmt) From 4b59380e723ecc497c9d5c9f25b9cf5b2c683857 Mon Sep 17 00:00:00 2001 From: Korina Cordero Date: Wed, 4 Oct 2023 10:08:42 +0800 Subject: [PATCH 26/27] Add fix for missing entries when loading file to database. #762 --- src/Command/GetJobOrderArchiveDataCommand.php | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/Command/GetJobOrderArchiveDataCommand.php b/src/Command/GetJobOrderArchiveDataCommand.php index a6540080..cd764e56 100644 --- a/src/Command/GetJobOrderArchiveDataCommand.php +++ b/src/Command/GetJobOrderArchiveDataCommand.php @@ -743,6 +743,7 @@ class GetJobOrderArchiveDataCommand extends Command // load statement for job order $load_stmt = 'LOAD DATA LOCAL INFILE \'' . $file . '\' INTO TABLE ' . $archive_tname . ' FIELDS TERMINATED BY \'|\' + ESCAPED BY \'\b\' LINES TERMINATED BY \'\\r\\n\' (id, customer_id, cvehicle_id, rider_id, date_create, date_schedule, date_fulfill, @coordinates, flag_advance, service_type, @@ -762,6 +763,7 @@ class GetJobOrderArchiveDataCommand extends Command { $load_stmt = 'LOAD DATA LOCAL INFILE \'' . $file . '\' INTO TABLE ' . $archive_tname . ' FIELDS TERMINATED BY \'|\' + ESCAPED BY \'\b\' LINES TERMINATED BY \'\\r\\n\' (id, create_user_id, job_order_id, date_create, date_happen, type_id, rider_id)'; } @@ -769,6 +771,7 @@ class GetJobOrderArchiveDataCommand extends Command { $load_stmt = 'LOAD DATA LOCAL INFILE \'' . $file . '\' INTO TABLE ' . $archive_tname . ' FIELDS TERMINATED BY \'|\' + ESCAPED BY \'\b\' LINES TERMINATED BY \'\\r\\n\' (id, user_id, hub_id, jo_id, date_create, reason, remarks, contact_person)'; } @@ -776,6 +779,7 @@ class GetJobOrderArchiveDataCommand extends Command { $load_stmt = 'LOAD DATA LOCAL INFILE \'' . $file . '\' INTO TABLE ' . $archive_tname . ' FIELDS TERMINATED BY \'|\' + ESCAPED BY \'\b\' LINES TERMINATED BY \'\\r\\n\' (id, user_id, job_order_id, date_create, date_paid, discount, trade_in, vat, vat_exclusive_price, total_price, status, promo_id, used_customer_tag_id)'; @@ -784,6 +788,7 @@ class GetJobOrderArchiveDataCommand extends Command { $load_stmt = 'LOAD DATA LOCAL INFILE \'' . $file . '\' INTO TABLE ' . $archive_tname . ' FIELDS TERMINATED BY \'|\' + ESCAPED BY \'\b\' LINES TERMINATED BY \'\\r\\n\' (id, rider_id, customer_id, jo_id, date_create, rating, comment)'; } @@ -791,6 +796,7 @@ class GetJobOrderArchiveDataCommand extends Command { $load_stmt = 'LOAD DATA LOCAL INFILE \'' . $file . '\' INTO TABLE ' . $archive_tname . ' FIELDS TERMINATED BY \'|\' + ESCAPED BY \'\b\' LINES TERMINATED BY \'\\r\\n\' (id, user_id, customer_id, date_create, status, ticket_type, other_ticket_type, first_name, last_name, contact_num, details, job_order_id, plate_number, ticket_type_id, subticket_type_id, @@ -800,6 +806,7 @@ class GetJobOrderArchiveDataCommand extends Command { $load_stmt = 'LOAD DATA LOCAL INFILE \'' . $file . '\' INTO TABLE ' . $archive_tname . ' FIELDS TERMINATED BY \'|\' + ESCAPED BY \'\b\' LINES TERMINATED BY \'\\r\\n\' (id, invoice_id, title, qty, price, battery_id)'; } From 1a0127072348f37776bc1c5dda2a3f4c15871293 Mon Sep 17 00:00:00 2001 From: Korina Cordero Date: Wed, 4 Oct 2023 13:58:53 +0800 Subject: [PATCH 27/27] Fix saving of the enye character. #762 --- src/Command/GetJobOrderArchiveDataCommand.php | 34 +++++++++++++------ 1 file changed, 24 insertions(+), 10 deletions(-) diff --git a/src/Command/GetJobOrderArchiveDataCommand.php b/src/Command/GetJobOrderArchiveDataCommand.php index cd764e56..b289cff6 100644 --- a/src/Command/GetJobOrderArchiveDataCommand.php +++ b/src/Command/GetJobOrderArchiveDataCommand.php @@ -403,7 +403,8 @@ class GetJobOrderArchiveDataCommand extends Command `initial_concern_notes` longtext COLLATE utf8_unicode_ci DEFAULT NULL, `gender` varchar(80) COLLATE utf8_unicode_ci DEFAULT NULL, `caller_classification` varchar(80) COLLATE utf8_unicode_ci DEFAULT NULL, - `inventory_count` smallint(6) NOT NULL, PRIMARY KEY (`id`))'; + `inventory_count` smallint(6) NOT NULL, PRIMARY KEY (`id`)) + DEFAULT CHARSET=utf8 COLLATE=utf8_unicode_ci'; $create_stmt = $db->prepare($create_sql); @@ -435,7 +436,8 @@ class GetJobOrderArchiveDataCommand extends Command `total_price` decimal(9,2) NOT NULL, `status` varchar(40) COLLATE utf8_unicode_ci NOT NULL, `promo_id` int(11) DEFAULT NULL, - `used_customer_tag_id` varchar(80) COLLATE utf8_unicode_ci DEFAULT NULL, PRIMARY KEY (`id`))'; + `used_customer_tag_id` varchar(80) COLLATE utf8_unicode_ci DEFAULT NULL, PRIMARY KEY (`id`)) + DEFAULT CHARSET=utf8 COLLATE=utf8_unicode_ci'; $create_stmt = $db->prepare($create_sql); @@ -457,7 +459,8 @@ class GetJobOrderArchiveDataCommand extends Command `title` varchar(80) COLLATE utf8_unicode_ci NOT NULL, `qty` smallint(6) NOT NULL, `price` decimal(9,2) NOT NULL, - `battery_id` int(11) DEFAULT NULL, PRIMARY KEY (`id`))'; + `battery_id` int(11) DEFAULT NULL, PRIMARY KEY (`id`)) + DEFAULT CHARSET=utf8 COLLATE=utf8_unicode_ci'; $create_stmt = $db->prepare($create_sql); @@ -491,7 +494,8 @@ class GetJobOrderArchiveDataCommand extends Command `subticket_type_id` int(11) DEFAULT NULL, `source_of_awareness` varchar(80) COLLATE utf8_unicode_ci DEFAULT NULL, `remarks` longtext COLLATE utf8_unicode_ci DEFAULT NULL, - `other_description` longtext COLLATE utf8_unicode_ci DEFAULT NULL, PRIMARY KEY (`id`))'; + `other_description` longtext COLLATE utf8_unicode_ci DEFAULT NULL, PRIMARY KEY (`id`)) + DEFAULT CHARSET=utf8 COLLATE=utf8_unicode_ci'; $create_stmt = $db->prepare($create_sql); @@ -515,7 +519,8 @@ class GetJobOrderArchiveDataCommand extends Command `date_create` datetime NOT NULL, `reason` varchar(255) COLLATE utf8_unicode_ci NOT NULL, `remarks` longtext COLLATE utf8_unicode_ci DEFAULT NULL, - `contact_person` varchar(255) COLLATE utf8_unicode_ci DEFAULT NULL, PRIMARY KEY (`id`))'; + `contact_person` varchar(255) COLLATE utf8_unicode_ci DEFAULT NULL, PRIMARY KEY (`id`)) + DEFAULT CHARSET=utf8 COLLATE=utf8_unicode_ci'; $create_stmt = $db->prepare($create_sql); @@ -538,7 +543,8 @@ class GetJobOrderArchiveDataCommand extends Command `jo_id` int(11) DEFAULT NULL, `date_create` datetime NOT NULL, `rating` int(11) NOT NULL, - `comment` longtext COLLATE utf8_unicode_ci NOT NULL, PRIMARY KEY (`id`))'; + `comment` longtext COLLATE utf8_unicode_ci NOT NULL, PRIMARY KEY (`id`)) + DEFAULT CHARSET=utf8 COLLATE=utf8_unicode_ci'; $create_stmt = $db->prepare($create_sql); @@ -561,7 +567,8 @@ class GetJobOrderArchiveDataCommand extends Command `date_create` datetime NOT NULL, `date_happen` datetime NOT NULL, `type_id` varchar(30) COLLATE utf8_unicode_ci NOT NULL, - `rider_id` int(11) DEFAULT NULL, PRIMARY KEY (`id`))'; + `rider_id` int(11) DEFAULT NULL, PRIMARY KEY (`id`)) + DEFAULT CHARSET=utf8 COLLATE=utf8_unicode_ci'; $create_stmt = $db->prepare($create_sql); @@ -741,7 +748,8 @@ class GetJobOrderArchiveDataCommand extends Command if ($tname == 'job_order') { // load statement for job order - $load_stmt = 'LOAD DATA LOCAL INFILE \'' . $file . '\' INTO TABLE ' . $archive_tname . ' + $load_stmt = 'LOAD DATA LOCAL INFILE \'' . $file . '\' INTO TABLE ' . $archive_tname . ' + CHARACTER SET UTF8 FIELDS TERMINATED BY \'|\' ESCAPED BY \'\b\' LINES TERMINATED BY \'\\r\\n\' @@ -762,6 +770,7 @@ class GetJobOrderArchiveDataCommand extends Command if ($tname == 'jo_event') { $load_stmt = 'LOAD DATA LOCAL INFILE \'' . $file . '\' INTO TABLE ' . $archive_tname . ' + CHARACTER SET UTF8 FIELDS TERMINATED BY \'|\' ESCAPED BY \'\b\' LINES TERMINATED BY \'\\r\\n\' @@ -770,6 +779,7 @@ class GetJobOrderArchiveDataCommand extends Command if ($tname == 'jo_rejection') { $load_stmt = 'LOAD DATA LOCAL INFILE \'' . $file . '\' INTO TABLE ' . $archive_tname . ' + CHARACTER SET UTF8 FIELDS TERMINATED BY \'|\' ESCAPED BY \'\b\' LINES TERMINATED BY \'\\r\\n\' @@ -778,6 +788,7 @@ class GetJobOrderArchiveDataCommand extends Command if ($tname == 'invoice') { $load_stmt = 'LOAD DATA LOCAL INFILE \'' . $file . '\' INTO TABLE ' . $archive_tname . ' + CHARACTER SET UTF8 FIELDS TERMINATED BY \'|\' ESCAPED BY \'\b\' LINES TERMINATED BY \'\\r\\n\' @@ -787,6 +798,7 @@ class GetJobOrderArchiveDataCommand extends Command if ($tname == 'rider_rating') { $load_stmt = 'LOAD DATA LOCAL INFILE \'' . $file . '\' INTO TABLE ' . $archive_tname . ' + CHARACTER SET UTF8 FIELDS TERMINATED BY \'|\' ESCAPED BY \'\b\' LINES TERMINATED BY \'\\r\\n\' @@ -795,6 +807,7 @@ class GetJobOrderArchiveDataCommand extends Command if ($tname == 'ticket') { $load_stmt = 'LOAD DATA LOCAL INFILE \'' . $file . '\' INTO TABLE ' . $archive_tname . ' + CHARACTER SET UTF8 FIELDS TERMINATED BY \'|\' ESCAPED BY \'\b\' LINES TERMINATED BY \'\\r\\n\' @@ -805,6 +818,7 @@ class GetJobOrderArchiveDataCommand extends Command if ($tname == 'invoice_item') { $load_stmt = 'LOAD DATA LOCAL INFILE \'' . $file . '\' INTO TABLE ' . $archive_tname . ' + CHARACTER SET UTF8 FIELDS TERMINATED BY \'|\' ESCAPED BY \'\b\' LINES TERMINATED BY \'\\r\\n\' @@ -874,7 +888,7 @@ class GetJobOrderArchiveDataCommand extends Command $update_active_sql = 'UPDATE rider SET active_jo_id = NULL WHERE active_jo_id IN (' . $jo_ids . ')'; $update_active_stmt = $db->prepare($update_active_sql); - error_log('Updating active rider job orders...'); + // error_log('Updating active rider job orders...'); $update_active_stmt->execute($jo_id_list); @@ -891,7 +905,7 @@ class GetJobOrderArchiveDataCommand extends Command $update_curr_sql = 'UPDATE rider SET current_jo_id = NULL WHERE current_jo_id IN (' . $jo_ids . ')'; $update_curr_stmt = $db->prepare($update_curr_sql); - error_log('Updating current rider job orders...'); + // error_log('Updating current rider job orders...'); $update_curr_stmt->execute($jo_id_list);