Modify array size for job order. #762

src/Command/GetJobOrderArchiveDataCommand.php

@@ -0,0 +1,941 @@
+<?php
+
+namespace App\Command;
+
+use Symfony\Component\Console\Command\Command;
+use Symfony\Component\Console\Input\InputArgument;
+use Symfony\Component\Console\Input\InputInterface;
+use Symfony\Component\Console\Output\OutputInterface;
+use Symfony\Component\HttpKernel\KernelInterface;
+use Symfony\Component\Filesystem\Filesystem;
+
+use Doctrine\ORM\EntityManagerInterface;
+
+use PDO;
+use DateTime;
+
+class GetJobOrderArchiveDataCommand extends Command
+{
+    protected $em;
+    protected $project_dir;
+    protected $filesystem;
+
+    public function __construct(EntityManagerInterface $em, KernelInterface $kernel, FileSystem $filesystem)
+    {
+        $this->em = $em;
+        $this->project_dir = $kernel->getProjectDir();
+        $this->filesystem = $filesystem;
+
+        parent::__construct();
+    }
+
+    protected function configure()
+    {
+        $this->setName('joborder:archive')
+            ->setDescription('Get job order data to archive.')
+            ->setHelp('Get job order data to archive.')
+            ->addArgument('year', InputArgument::REQUIRED, 'year');
+    }
+
+    protected function execute(InputInterface $input, OutputInterface $output)
+    {
+        $current_datetime = new DateTime('now');
+
+        error_log('Archive start time ' . $current_datetime->format('Y-m-d H:i'));
+
+        // get year to archive
+        $year = $input->getArgument('year');
+
+        // get the data to archive for the following tables:
+        // (1) job_order
+        // (2) invoice (has jo foreign key)
+        // (3) invoice_item (has invoice foreign key)
+        // (4) ticket (has jo foreign key)
+        // (5) jo_rejection (has jo foreign key)
+        // (6) rider_rating (has jo foreign key)
+
+        // create the archive tables
+        $this->createJobOrderArchiveTables($year);
+        $this->createInvoiceArchiveTable($year);
+        $this->createInvoiceItemArchiveTable($year);
+        $this->createTicketArchiveTable($year);
+        $this->createJORejectionArchiveTable($year);
+        $this->createRiderRatingArchiveTable($year);
+        $this->createJOEventArchiveTable($year);
+
+        $db = $this->em->getConnection();
+
+        // set the pdo connection to use unbuffered query so we don't run out of memory
+        // when processing the job order related tables
+        $db->getWrappedConnection()->setAttribute(PDO::MYSQL_ATTR_USE_BUFFERED_QUERY, false);
+
+        $query_sql = 'SELECT * 
+            FROM job_order 
+            WHERE YEAR(date_create) = :year  
+            ORDER BY date_create'; 
+
+        $query_stmt = $db->prepare($query_sql);
+        $query_stmt->bindValue('year', $year, PDO::PARAM_STR);
+
+        $callback = [$this, 'getRelatedArchiveData'];
+
+        $this->getArchiveData($query_stmt, $callback, 'job_order', $year);
+
+        $current_datetime = new DateTime('now');
+
+        error_log('Archive end time ' . $current_datetime->format('Y-m-d H:i'));
+
+        return 0;
+    }
+
+    protected function getArchiveData($stmt, $callbackJO, $jo_tname, $year)
+    {
+        $results = $stmt->executeQuery();
+
+        $related_tables = ['jo_event', 'invoice', 'ticket', 'jo_rejection', 'rider_rating'];
+
+        // delete the related data files
+        foreach ($related_tables as $tname)
+        {
+            $this->deleteDataFiles($tname, $year);
+        }
+
+        // special since this is not directly related to JO but to invoice
+        $this->deleteDataFiles('invoice_item', $year);
+
+        $archive_files = [];
+
+        $jo_id_list = [];
+        $ii_id_list = [];
+
+        while ($row = $results->fetchAssociative())
+        {
+            $jo_data['job_order'][$row['id']] = $this->createJobOrderArchiveData($row);
+
+            // add jo id to jo_id_list
+            $jo_id_list[] = $row['id'];
+        }
+
+        // write the array into the file
+        $file = $this->createDataFileRelatedArchiveData($jo_data, $jo_tname, $year, 'w');
+
+        if ($file != null)
+        {
+            $archive_files[$jo_tname] = $file;
+        }
+
+        // error_log('jo_data total ' . count($jo_data['job_order']));
+        // error_log('jo id list total ' . count($jo_id_list));
+
+        unset($jo_data);
+
+        // load the job order archive file for job order into the database
+        $this->loadArchiveFiles($archive_files, $year);
+
+        unset($archive_files[$jo_tname]);
+
+        // get all related data for job order
+        foreach ($jo_id_list as $jo_id)
+        {
+            // foreach job order id we got from the first query, we get the JO related
+            // data for that id from jo_event, invoice, ticket, jo_rejection, rider_rating
+            if (is_callable($callbackJO))
+            {  
+                foreach ($related_tables as $table_name)
+                {
+                    switch ($table_name) {
+                        case 'jo_event':
+                        case 'invoice':
+                        case 'ticket':
+                            $query_sql = 'SELECT *  FROM ' . $table_name . ' WHERE job_order_id = :id';
+                            break;
+                        case 'jo_rejection':
+                        case 'rider_rating':
+                            $query_sql = 'SELECT * FROM ' . $table_name . ' WHERE jo_id = :id';
+                            break;
+                        default:
+                            break;
+                    }
+
+                    $db = $this->em->getConnection();
+                    $db->getWrappedConnection()->setAttribute(PDO::MYSQL_ATTR_USE_BUFFERED_QUERY, false);
+
+                    $query_stmt = $db->prepare($query_sql);
+                    $query_stmt->bindValue('id', $jo_id);
+
+                    $files = call_user_func($callbackJO, $row, $query_stmt, $table_name, $year);
+
+                    foreach ($files as $key =>$file)
+                    {
+                        if ($file != null)
+                            $archive_files[$key] = $file;
+                    }
+                }
+            }
+        }
+
+        $this->loadArchiveFiles($archive_files, $year);
+
+        error_log('Done loading files into database...');
+
+        $jo_id_array = array_chunk($jo_id_list, 15000);
+
+        unset($jo_id_list);
+
+        foreach($jo_id_array as $key => $jo_ids)
+        {
+            // update rider's active_jo_id and current_jo_id to null
+            // so we can delete the old job orders (foreign key constraint)
+            $this->updateRiderActiveJobOrders($jo_ids);
+            $this->updateRiderCurrentJobOrders($jo_ids);
+
+            $this->deleteData($jo_ids, $related_tables);
+        }
+    }
+
+    protected function getRelatedArchiveData($row, $query_stmt, $table_name, $year)
+    {
+        $results = $query_stmt->executeQuery();
+
+        $data = [];
+        $files = [];
+        $invoice_id_list = [];
+
+        while ($q_row = $results->fetchAssociative())
+        {
+            // check if table name is invoice because we need to get 
+            // all invoice items for a specific invoice too
+            if ($table_name == 'invoice')
+            {
+                // add invoice id to list
+                $invoice_id_list[] = $q_row['id'];
+            }
+
+            $fields = [];
+ 
+            foreach ($q_row as $key => $value)
+            {
+                $cleaned_value = $this->cleanData(($value) ?? '\N');
+                $fields[] = $cleaned_value;
+            }
+
+            $data[$table_name][$q_row['id']] = $fields;
+        }
+
+        // get the invoice items for archiving
+        $ii_id_list = [];
+        foreach ($invoice_id_list as $i_id)
+        {
+            $ii_file = $this->getInvoiceItemArchiveData($i_id, $year);
+
+            $files['invoice_item'] = $ii_file;
+        }
+
+        // write the array into the file
+        $file = $this->createDataFileRelatedArchiveData($data, $table_name, $year, 'a');
+
+        unset($data);
+
+        if ($file != null)
+            $files[$table_name] = $file;
+
+        return $files;
+    }
+
+    protected function getInvoiceItemArchiveData($id, $year)
+    {
+        $db = $this->em->getConnection();
+        $db->getWrappedConnection()->setAttribute(PDO::MYSQL_ATTR_USE_BUFFERED_QUERY, false);
+ 
+        $query_sql = 'SELECT * FROM invoice_item WHERE invoice_id = :id';
+
+        $query_stmt = $db->prepare($query_sql);
+        $query_stmt->bindValue('id', $id);
+
+        $results = $query_stmt->executeQuery();
+
+        $ii_data = [];
+        $ii_id_list = [];
+        while ($ii_row = $results->fetchAssociative())
+        {
+            $id = $ii_row['id'];
+            $invoice_id = $ii_row['invoice_id'];
+            $title = $ii_row['title'];
+            $qty = $ii_row['qty'];
+            $price = $ii_row['price'];
+            $battery_id = $ii_row['battery_id'] ?? '\N';
+
+            $ii_data['invoice_item'][$id] = [
+                $id,
+                $invoice_id,
+                $title,
+                $qty,
+                $price,
+                $battery_id
+            ];
+
+            $ii_id_list[] = $id;
+        }
+
+        $file = $this->createDataFileRelatedArchiveData($ii_data, 'invoice_item', $year, 'a');
+
+        unset($ii_data);
+
+        // special case, delete the invoice items already
+        // so that we don't have to query for the invoice items to delete
+        if (count($ii_id_list) > 0)
+            $this->deleteInvoiceItems($ii_id_list);
+
+        unset($ii_id_list);
+
+        return $file;
+    }
+
+    protected function deleteDataFiles($tname, $year)
+    {
+        // cache directory
+        $cache_dir = __DIR__ . '/../../var/cache';
+
+        $file = $cache_dir . '/' . $tname . '_archive_' . $year .'.tab';
+
+        if (file_exists($file))
+            unlink($file);
+    }
+
+    protected function createDataFileRelatedArchiveData($archive_data, $table_name, $year, $option)
+    {
+        if (isset($archive_data[$table_name]))
+        {
+            $adata = $archive_data[$table_name];
+
+            // cache directory
+            $cache_dir = __DIR__ . '/../../var/cache';
+
+            $file = $cache_dir . '/' . $table_name . '_archive_'. $year . '.tab';
+            // error_log('opening file for archive - ' . $file);
+
+            $fp = fopen($file, $option);
+            if ($fp === false)
+            {
+                 error_log('could not open file for load data infile - ' . $file);
+            }
+            else
+            {
+                foreach ($adata as $key => $data)
+                {
+                    $line = implode('|', $data) . "\r\n";
+                    fwrite($fp, $line);
+                }
+            }
+
+            fclose($fp);
+
+            return $file;
+        }
+
+        return null;
+    }
+
+    protected function createJobOrderArchiveTables($year) {
+        // form the archive table name <original table name>_archive_<year of data being archived>
+        $archive_table_name = 'job_order_archive_' . $year;
+
+        // create the table if it doesn't exist
+        $db = $this->em->getConnection();
+
+        // TODO: What if table already exists?
+        $create_sql = 'CREATE TABLE IF NOT EXISTS `' . $archive_table_name . '` (
+            `id` int(11) NOT NULL,
+            `customer_id` int(11) DEFAULT NULL,
+            `cvehicle_id` int(11) DEFAULT NULL,
+            `rider_id` int(11) DEFAULT NULL,
+            `date_create` datetime NOT NULL,
+            `date_schedule` datetime NOT NULL,
+            `date_fulfill` datetime DEFAULT NULL,
+            `coordinates` point NOT NULL COMMENT \'(DC2Type:point)\',
+            `flag_advance` tinyint(1) NOT NULL,
+            `service_type` varchar(25) COLLATE utf8_unicode_ci NOT NULL,
+            `source` varchar(30) COLLATE utf8_unicode_ci NOT NULL,
+            `date_cancel` datetime DEFAULT NULL,
+            `status` varchar(15) COLLATE utf8_unicode_ci NOT NULL,
+            `delivery_instructions` longtext COLLATE utf8_unicode_ci DEFAULT NULL,
+            `delivery_address` longtext COLLATE utf8_unicode_ci NOT NULL,
+            `create_user_id` int(11) DEFAULT NULL,
+            `assign_user_id` int(11) DEFAULT NULL,
+            `date_assign` datetime DEFAULT NULL,
+            `warranty_class` varchar(25) COLLATE utf8_unicode_ci NOT NULL,
+            `process_user_id` int(11) DEFAULT NULL,
+            `hub_id` int(11) DEFAULT NULL,
+            `cancel_reason` varchar(200) COLLATE utf8_unicode_ci DEFAULT NULL,
+            `ref_jo_id` int(11) DEFAULT NULL,
+            `tier1_notes` longtext COLLATE utf8_unicode_ci DEFAULT NULL,
+            `tier2_notes` longtext COLLATE utf8_unicode_ci DEFAULT NULL,
+            `mode_of_payment` varchar(50) COLLATE utf8_unicode_ci NOT NULL,
+            `or_name` varchar(80) COLLATE utf8_unicode_ci NOT NULL,
+            `landmark` longtext COLLATE utf8_unicode_ci NOT NULL,
+            `promo_detail` varchar(80) COLLATE utf8_unicode_ci NOT NULL,
+            `or_num` varchar(80) COLLATE utf8_unicode_ci DEFAULT NULL,
+            `trade_in_type` varchar(25) COLLATE utf8_unicode_ci DEFAULT NULL,
+            `flag_rider_rating` tinyint(1) DEFAULT NULL,
+            `flag_coolant` tinyint(1) NOT NULL,
+            `facilitated_hub_id` int(11) DEFAULT NULL,
+            `facilitated_type` varchar(8) COLLATE utf8_unicode_ci DEFAULT NULL,
+            `coord_long` decimal(11,8) NOT NULL,
+            `coord_lat` decimal(11,8) NOT NULL,
+            `priority` int(11) NOT NULL DEFAULT 0,
+            `meta` longtext CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL,
+            `status_autoassign` varchar(30) COLLATE utf8_unicode_ci DEFAULT NULL,
+            `first_name` varchar(80) COLLATE utf8_unicode_ci NOT NULL,
+            `last_name` varchar(80) COLLATE utf8_unicode_ci NOT NULL,
+            `plate_number` varchar(100) COLLATE utf8_unicode_ci NOT NULL,
+            `phone_mobile` varchar(30) COLLATE utf8_unicode_ci NOT NULL,
+            `no_trade_in_reason` varchar(80) COLLATE utf8_unicode_ci DEFAULT NULL,
+            `will_wait` varchar(30) COLLATE utf8_unicode_ci NOT NULL,
+            `reason_not_waiting` varchar(80) COLLATE utf8_unicode_ci DEFAULT NULL,
+            `not_waiting_notes` longtext COLLATE utf8_unicode_ci DEFAULT NULL,
+            `delivery_status` varchar(30) COLLATE utf8_unicode_ci DEFAULT NULL,
+            `emergency_type_id` int(11) DEFAULT NULL,
+            `ownership_type_id` int(11) DEFAULT NULL,
+            `cust_location_id` int(11) DEFAULT NULL,
+            `source_of_awareness` varchar(80) COLLATE utf8_unicode_ci DEFAULT NULL,
+            `remarks` longtext COLLATE utf8_unicode_ci DEFAULT NULL,
+            `initial_concern` varchar(80) COLLATE utf8_unicode_ci DEFAULT NULL,
+            `initial_concern_notes` longtext COLLATE utf8_unicode_ci DEFAULT NULL,
+            `gender` varchar(80) COLLATE utf8_unicode_ci DEFAULT NULL,
+            `caller_classification` varchar(80) COLLATE utf8_unicode_ci DEFAULT NULL,
+            `inventory_count` smallint(6) NOT NULL, PRIMARY KEY (`id`))
+            DEFAULT CHARSET=utf8 COLLATE=utf8_unicode_ci';
+
+        $create_stmt = $db->prepare($create_sql);
+
+        $result = $create_stmt->execute();
+
+        return $archive_table_name;
+    }
+
+    protected function createInvoiceArchiveTable($year)
+    {
+        // form the archive table name <original table name>_archive_<year of data being archived>
+        $archive_table_name = 'invoice_archive_' . $year;
+
+        // create the table if it doesn't exist
+        $db = $this->em->getConnection();
+
+        // TODO: What if table already exists?
+        $create_sql = 'CREATE TABLE IF NOT EXISTS `' . $archive_table_name . '` (
+            `id` int(11) NOT NULL,
+            `user_id` int(11) DEFAULT NULL,
+            `job_order_id` int(11) DEFAULT NULL,
+            `date_create` datetime NOT NULL,
+            `date_paid` datetime DEFAULT NULL,
+            `date_cancel` datetime DEFAULT NULL,
+            `discount` decimal(9,2) NOT NULL,
+            `trade_in` decimal(9,2) NOT NULL,
+            `vat` decimal(9,2) NOT NULL,
+            `vat_exclusive_price` decimal(9,2) NOT NULL,
+            `total_price` decimal(9,2) NOT NULL,
+            `status` varchar(40) COLLATE utf8_unicode_ci NOT NULL,
+            `promo_id` int(11) DEFAULT NULL,
+            `used_customer_tag_id` varchar(80) COLLATE utf8_unicode_ci DEFAULT NULL, PRIMARY KEY (`id`))
+            DEFAULT CHARSET=utf8 COLLATE=utf8_unicode_ci';
+
+        $create_stmt = $db->prepare($create_sql);
+
+        $result = $create_stmt->execute();
+    }
+
+    protected function createInvoiceItemArchiveTable($year)
+    {
+        // form the archive table name <original table name>_archive_<year of data being archived>
+        $archive_table_name = 'invoice_item_archive_' . $year;
+
+        // create the table if it doesn't exist
+        $db = $this->em->getConnection();
+
+        // TODO: What if table already exists?
+        $create_sql = 'CREATE TABLE IF NOT EXISTS `' . $archive_table_name . '` (
+            `id` int(11) NOT NULL,
+            `invoice_id` int(11) DEFAULT NULL,
+            `title` varchar(80) COLLATE utf8_unicode_ci NOT NULL,
+            `qty` smallint(6) NOT NULL,
+            `price` decimal(9,2) NOT NULL,
+            `battery_id` int(11) DEFAULT NULL, PRIMARY KEY (`id`))
+            DEFAULT CHARSET=utf8 COLLATE=utf8_unicode_ci';
+
+        $create_stmt = $db->prepare($create_sql);
+
+        $result = $create_stmt->execute();
+    }
+
+    protected function createTicketArchiveTable($year)
+    {
+        // form the archive table name <original table name>_archive_<year of data being archived>
+        $archive_table_name = 'ticket_archive_' . $year;
+
+        // create the table if it doesn't exist
+        $db = $this->em->getConnection();
+
+        // TODO: What if table already exists?
+        $create_sql = 'CREATE TABLE IF NOT EXISTS `' . $archive_table_name . '` (
+            `id` int(11) NOT NULL,
+            `user_id` int(11) DEFAULT NULL,
+            `customer_id` int(11) DEFAULT NULL,
+            `date_create` datetime NOT NULL,
+            `status` varchar(15) COLLATE utf8_unicode_ci NOT NULL,
+            `ticket_type` varchar(15) COLLATE utf8_unicode_ci DEFAULT NULL,
+            `other_ticket_type` varchar(80) COLLATE utf8_unicode_ci DEFAULT NULL,
+            `first_name` varchar(80) COLLATE utf8_unicode_ci NOT NULL,
+            `last_name` varchar(80) COLLATE utf8_unicode_ci NOT NULL,
+            `contact_num` varchar(20) COLLATE utf8_unicode_ci DEFAULT NULL,
+            `details` longtext COLLATE utf8_unicode_ci DEFAULT NULL,
+            `job_order_id` int(11) DEFAULT NULL,
+            `plate_number` varchar(10) COLLATE utf8_unicode_ci DEFAULT NULL,
+            `ticket_type_id` int(11) DEFAULT NULL,
+            `subticket_type_id` int(11) DEFAULT NULL,
+            `source_of_awareness` varchar(80) COLLATE utf8_unicode_ci DEFAULT NULL,
+            `remarks` longtext COLLATE utf8_unicode_ci DEFAULT NULL,
+            `other_description` longtext COLLATE utf8_unicode_ci DEFAULT NULL, PRIMARY KEY (`id`))
+            DEFAULT CHARSET=utf8 COLLATE=utf8_unicode_ci';
+
+        $create_stmt = $db->prepare($create_sql);
+
+        $result = $create_stmt->execute();
+    }
+
+    protected function createJORejectionArchiveTable($year)
+    {
+        // form the archive table name <original table name>_archive_<year of data being archived>
+        $archive_table_name = 'jo_rejection_archive_' . $year;
+
+        // create the table if it doesn't exist
+        $db = $this->em->getConnection();
+
+        // TODO: What if table already exists?
+        $create_sql = 'CREATE TABLE IF NOT EXISTS `' . $archive_table_name . '` (
+            `id` int(11) NOT NULL,
+            `user_id` int(11) DEFAULT NULL,
+            `hub_id` int(11) DEFAULT NULL,
+            `jo_id` int(11) DEFAULT NULL,
+            `date_create` datetime NOT NULL,
+            `reason` varchar(255) COLLATE utf8_unicode_ci NOT NULL,
+            `remarks` longtext COLLATE utf8_unicode_ci DEFAULT NULL,
+            `contact_person` varchar(255) COLLATE utf8_unicode_ci DEFAULT NULL, PRIMARY KEY (`id`))
+            DEFAULT CHARSET=utf8 COLLATE=utf8_unicode_ci';
+
+        $create_stmt = $db->prepare($create_sql);
+
+        $result = $create_stmt->execute();
+    }
+
+    protected function createRiderRatingArchiveTable($year)
+    {
+        // form the archive table name <original table name>_archive_<year of data being archived>
+        $archive_table_name = 'rider_rating_archive_' . $year;
+
+        // create the table if it doesn't exist
+        $db = $this->em->getConnection();
+
+        // TODO: What if table already exists?
+        $create_sql = 'CREATE TABLE IF NOT EXISTS `' . $archive_table_name . '` (
+            `id` int(11) NOT NULL,
+            `rider_id` int(11) DEFAULT NULL,
+            `customer_id` int(11) DEFAULT NULL,
+            `jo_id` int(11) DEFAULT NULL,
+            `date_create` datetime NOT NULL,
+            `rating` int(11) NOT NULL,
+            `comment` longtext COLLATE utf8_unicode_ci NOT NULL, PRIMARY KEY (`id`))
+            DEFAULT CHARSET=utf8 COLLATE=utf8_unicode_ci';
+
+        $create_stmt = $db->prepare($create_sql);
+
+        $result = $create_stmt->execute();
+    }
+
+    protected function createJOEventArchiveTable($year)
+    {
+        // form the archive table name <original table name>_archive_<year of data being archived>
+        $archive_table_name = 'jo_event_archive_' . $year;
+
+        // create the table if it doesn't exist
+        $db = $this->em->getConnection();
+
+        // TODO: What if table already exists?
+        $create_sql = 'CREATE TABLE IF NOT EXISTS `' . $archive_table_name . '` (
+            `id` int(11) NOT NULL AUTO_INCREMENT,
+            `create_user_id` int(11) DEFAULT NULL,
+            `job_order_id` int(11) DEFAULT NULL,
+            `date_create` datetime NOT NULL,
+            `date_happen` datetime NOT NULL,
+            `type_id` varchar(30) COLLATE utf8_unicode_ci NOT NULL,
+            `rider_id` int(11) DEFAULT NULL, PRIMARY KEY (`id`))
+            DEFAULT CHARSET=utf8 COLLATE=utf8_unicode_ci';
+
+        $create_stmt = $db->prepare($create_sql);
+
+        $result = $create_stmt->execute();
+    }
+
+
+    protected function createJobOrderArchiveData($row)
+    {
+        // TODO: this could be shrunk further
+
+        // get job order data
+        // check for nulls. check the ff fields since these can be null: date_fulfill, date_cancel, date_assign, create_user_id, 
+        // assign_user_id, process_user_id, hub_id, rider_id, cancel_reason, ref_jo_id, or_num, trade_in_type, 
+        // flag_rider_rating, facilitated_type, facilitated_hub_id, status_autoassign, reason_not_waiting,
+        // not_waiting_notes, no_trade_in_reason, delivery_status, source_of_awareness, remarks, initial_concern,
+        // initial_concern_notes, gender, caller_classifications, emergency_type_id, ownership_type_id, cust_location_id
+
+        $id = $row['id'];
+        $cust_id = $row['customer_id'];
+        $cv_id = $row['cvehicle_id'];
+
+        $rider_id = $row['rider_id'] ?? '\N';
+
+        $date_create = $row['date_create'];
+        $date_schedule = $row['date_schedule'];
+
+        $date_fulfill = $row['date_fulfill'] ?? '\N';
+
+        $flag_advance = $row['flag_advance'];
+        $service_type = $row['service_type'];
+        $source = $row['source'];
+
+        $date_cancel = $row['date_cancel'] ?? '\N';
+
+        $status = $row['status'];
+
+        $del_instructions = $this->cleanData($row['delivery_instructions']);
+        $del_address = $this->cleanData($row['delivery_address']);
+
+        $create_user_id = $row['create_user_id'] ?? '\N';
+        $assign_user_id = $row['assign_user_id'] ?? '\N';
+        $date_assign = $row['date_assign'] ?? '\N';
+
+        $warr_class = $row['warranty_class']; 
+
+        $process_user_id = $row['process_user_id'] ?? '\N';
+        $hub_id =  $row['hub_id'] ?? '\N';
+        $cancel_reason = $row['cancel_reason'] ?? '\N';
+        $ref_jo_id = $row['ref_jo_id'] ?? '\N';
+
+        $tier1_notes = $this->cleanData($row['tier1_notes']);
+        $tier2_notes = $this->cleanData($row['tier2_notes']);
+
+        $mode_of_payment = $row['mode_of_payment'];
+        $or_name = $row['or_name'];
+
+        $landmark = $this->cleanData($row['landmark']);
+        $promo_details = $row['promo_detail'];
+
+        $or_num = $row['or_num'] ?? '\N';
+        $trade_in_type = $row['trade_in_type'] ?? '\N';
+        $flag_rider_rating = $row['flag_rider_rating'] ?? '\N';
+
+        $flag_coolant = $row['flag_coolant'];
+
+        $fac_hub_id = $row['facilitated_hub_id'] ?? '\N';
+        $fac_type = $row['facilitated_type'] ?? '\N';
+
+        $coord_long = $row['coord_long'];
+        $coord_lat = $row['coord_lat'];
+
+        // coordinates needs special handling since it's a spatial column
+        $geo_coordinates = 'POINT(' . $coord_lat . ' ' . $coord_long .')';
+
+        $priority = $row['priority'];
+        $meta = $row['meta'];
+
+        $status_autoassign = $row['status_autoassign'] ?? '\N';
+
+        $first_name = $row['first_name'];
+        $last_name = $row['last_name'];
+        $plate_number = $row['plate_number'];
+        $phone_mobile = $row['phone_mobile'];
+
+        $no_trade_in_reason = $row['no_trade_in_reason'] ?? '\N';
+
+        $will_wait = $row['will_wait'];
+
+        $reason_not_waiting = $row['reason_not_waiting'] ?? '\N';
+        $not_waiting_notes = $this->cleanData($row['not_waiting_notes']) ?? '\N';
+        $del_status = $row['delivery_status'] ?? '\N';
+        $emergency_type_id = $row['emergency_type_id'] ?? '\N';
+        $owner_type_id = $row['ownership_type_id'] ?? '\N';
+        $cust_location_id = $row['cust_location_id'] ?? '\N';
+        $source_of_awareness = $row['source_of_awareness'] ?? '\N';
+        $remarks = $this->cleanData($row['remarks']) ?? '\N';
+        $initial_concern = $row['initial_concern'] ?? '\N';
+        $initial_concern_notes = $this->cleanData($row['initial_concern_notes']) ?? '\N';
+        $gender = $row['gender'] ?? '\N';
+        $caller_class = $row['caller_classification'] ?? '\N';
+
+        $inv_count = $row['inventory_count'];
+
+        // create the array for the file
+        $data = [
+            $id,
+            $cust_id,
+            $cv_id,
+            $rider_id,
+            $date_create,
+            $date_schedule,
+            $date_fulfill,
+            $geo_coordinates,
+            $flag_advance,
+            $service_type,
+            $source,
+            $date_cancel,
+            $status,
+            $del_instructions,
+            $del_address,
+            $create_user_id,
+            $assign_user_id,
+            $date_assign,
+            $warr_class,
+            $process_user_id,
+            $hub_id,
+            $cancel_reason,
+            $ref_jo_id,
+            $tier1_notes,
+            $tier2_notes,
+            $mode_of_payment,
+            $or_name,
+            $landmark,
+            $promo_details,
+            $or_num,
+            $trade_in_type,
+            $flag_rider_rating,
+            $flag_coolant,
+            $fac_hub_id,
+            $fac_type,
+            $coord_long,
+            $coord_lat,
+            $priority,
+            $meta,
+            $status_autoassign,
+            $first_name,
+            $last_name,
+            $plate_number,
+            $phone_mobile,
+            $no_trade_in_reason,
+            $will_wait,
+            $reason_not_waiting,
+            $not_waiting_notes,
+            $del_status,
+            $emergency_type_id,
+            $owner_type_id,
+            $cust_location_id,
+            $source_of_awareness,
+            $remarks,
+            $initial_concern,
+            $initial_concern_notes,
+            $gender,
+            $caller_class,
+            $inv_count
+        ];
+
+        return $data;
+    }
+
+    protected function loadArchiveFiles($archive_files, $year)
+    {
+        foreach ($archive_files as $tname => $file)
+        {
+            $archive_tname = $tname . '_archive_' . $year;
+
+            if ($tname == 'job_order')
+            {
+                // load statement for job order
+                $load_stmt = 'LOAD DATA LOCAL INFILE \'' . $file . '\' INTO TABLE ' . $archive_tname . ' 
+                    CHARACTER SET UTF8 
+                    FIELDS TERMINATED BY \'|\' 
+                    ESCAPED BY \'\b\'
+                    LINES TERMINATED BY \'\\r\\n\'
+                    (id, customer_id, cvehicle_id, rider_id, date_create, 
+                     date_schedule, date_fulfill, @coordinates, flag_advance, service_type,
+                     source, date_cancel, status, delivery_instructions, delivery_address, 
+                     create_user_id, assign_user_id, date_assign, warranty_class, process_user_id,
+                     hub_id, cancel_reason, ref_jo_id, tier1_notes, tier2_notes,
+                     mode_of_payment, or_name, landmark, promo_detail, or_num,
+                     trade_in_type, flag_rider_rating, flag_coolant, facilitated_hub_id, facilitated_type,
+                     coord_long, coord_lat, priority, meta, status_autoassign,
+                     first_name, last_name, plate_number, phone_mobile, no_trade_in_reason,
+                     will_wait, reason_not_waiting, not_waiting_notes, delivery_status, emergency_type_id,
+                     ownership_type_id, cust_location_id, source_of_awareness, remarks, initial_concern,
+                     initial_concern_notes, gender, caller_classification, inventory_count)
+                     SET coordinates=ST_GeomFromText(@geo_coordinates)';
+            }
+            if ($tname == 'jo_event')
+            {
+                $load_stmt = 'LOAD DATA LOCAL INFILE \'' . $file . '\' INTO TABLE ' . $archive_tname . '
+                    CHARACTER SET UTF8
+                    FIELDS TERMINATED BY \'|\' 
+                    ESCAPED BY \'\b\'
+                    LINES TERMINATED BY \'\\r\\n\'
+                    (id, create_user_id, job_order_id, date_create, date_happen, type_id, rider_id)';
+            }
+            if ($tname == 'jo_rejection')
+            {
+                $load_stmt = 'LOAD DATA LOCAL INFILE \'' . $file . '\' INTO TABLE ' . $archive_tname . '
+                    CHARACTER SET UTF8
+                    FIELDS TERMINATED BY \'|\' 
+                    ESCAPED BY \'\b\'
+                    LINES TERMINATED BY \'\\r\\n\'
+                    (id, user_id, hub_id, jo_id, date_create, reason, remarks, contact_person)';
+            }
+            if ($tname == 'invoice')
+            {
+                $load_stmt = 'LOAD DATA LOCAL INFILE \'' . $file . '\' INTO TABLE ' . $archive_tname . '
+                    CHARACTER SET UTF8
+                    FIELDS TERMINATED BY \'|\' 
+                    ESCAPED BY \'\b\'
+                    LINES TERMINATED BY \'\\r\\n\'
+                    (id, user_id, job_order_id, date_create, date_paid, discount, trade_in, vat, vat_exclusive_price,
+                     total_price, status, promo_id, used_customer_tag_id)';
+            }
+            if ($tname == 'rider_rating')
+            {
+                $load_stmt = 'LOAD DATA LOCAL INFILE \'' . $file . '\' INTO TABLE ' . $archive_tname . '
+                    CHARACTER SET UTF8
+                    FIELDS TERMINATED BY \'|\' 
+                    ESCAPED BY \'\b\'
+                    LINES TERMINATED BY \'\\r\\n\'
+                    (id, rider_id, customer_id, jo_id, date_create, rating, comment)';
+            }
+            if ($tname == 'ticket')
+            {
+                $load_stmt = 'LOAD DATA LOCAL INFILE \'' . $file . '\' INTO TABLE ' . $archive_tname . '
+                    CHARACTER SET UTF8 
+                    FIELDS TERMINATED BY \'|\' 
+                    ESCAPED BY \'\b\'
+                    LINES TERMINATED BY \'\\r\\n\'
+                    (id, user_id, customer_id, date_create, status, ticket_type, other_ticket_type, first_name, last_name,
+                     contact_num, details, job_order_id, plate_number, ticket_type_id, subticket_type_id, 
+                     source_of_awareness, remarks, other_description)';
+            }
+            if ($tname == 'invoice_item')
+            {
+                $load_stmt = 'LOAD DATA LOCAL INFILE \'' . $file . '\' INTO TABLE ' . $archive_tname . '
+                    CHARACTER SET UTF8
+                    FIELDS TERMINATED BY \'|\' 
+                    ESCAPED BY \'\b\'
+                    LINES TERMINATED BY \'\\r\\n\'
+                    (id, invoice_id, title, qty, price, battery_id)';
+            }
+
+            // call load data infile
+            $this->loadDataFileForArchiveData($load_stmt);
+        }
+    }
+
+    protected function deleteInvoiceItems($ii_id_list)
+    {
+        $db = $this->em->getConnection();
+
+        // delete the invoice items first
+        $ii_ids = str_repeat('?,', count($ii_id_list) - 1) . '?';
+
+        $ii_del_sql = 'DELETE FROM invoice_item WHERE id IN (' . $ii_ids . ')';
+        $ii_stmt = $db->prepare($ii_del_sql);
+
+        $ii_stmt->execute($ii_id_list);
+    }
+
+    protected function deleteData($jo_id_list, $related_tables)
+    {
+        $db = $this->em->getConnection();
+
+        // delete from invoice, jo_rejection, rider_rating, ticket, and jo_event
+        $jo_ids = str_repeat('?,', count($jo_id_list) - 1) . '?';
+
+        foreach ($related_tables as $table_name)
+        {
+            switch ($table_name) {
+                case 'jo_event':
+                case 'invoice':
+                case 'ticket': 
+                    $related_del_sql = 'DELETE  FROM ' . $table_name . ' WHERE job_order_id IN ('. $jo_ids . ')';
+                    break;
+                case 'jo_rejection':
+                case 'rider_rating':
+                    $related_del_sql = 'DELETE FROM ' . $table_name . ' WHERE jo_id IN (' . $jo_ids. ')';
+                    break;
+                default:
+                    break;
+            }
+
+            $related_stmt = $db->prepare($related_del_sql);
+
+            $related_stmt->execute($jo_id_list);
+        }
+
+        // delete from job order last
+        $jo_del_sql = 'DELETE FROM job_order WHERE id IN (' . $jo_ids . ')';
+        $jo_stmt = $db->prepare($jo_del_sql);
+
+        $jo_stmt->execute($jo_id_list); 
+    }
+
+    protected function updateRiderActiveJobOrders($jo_id_list)
+    {
+        $db = $this->em->getConnection();
+        $db->getWrappedConnection()->setAttribute(PDO::MYSQL_ATTR_USE_BUFFERED_QUERY, false);
+
+        $jo_ids = str_repeat('?,', count($jo_id_list) - 1) . '?';
+
+        $update_active_sql = 'UPDATE rider SET active_jo_id = NULL WHERE active_jo_id IN (' . $jo_ids . ')';
+        $update_active_stmt = $db->prepare($update_active_sql);
+
+        // error_log('Updating active rider job orders...');
+
+        $update_active_stmt->execute($jo_id_list);
+
+        unset($update_active_stmt);
+    }
+
+    protected function updateRiderCurrentJobOrders($jo_id_list)
+    {
+        $db = $this->em->getConnection();
+        $db->getWrappedConnection()->setAttribute(PDO::MYSQL_ATTR_USE_BUFFERED_QUERY, false);
+
+        $jo_ids = str_repeat('?,', count($jo_id_list) - 1) . '?';
+
+        $update_curr_sql = 'UPDATE rider SET current_jo_id = NULL WHERE current_jo_id IN (' . $jo_ids . ')';
+        $update_curr_stmt = $db->prepare($update_curr_sql);
+
+        // error_log('Updating current rider job orders...');
+
+        $update_curr_stmt->execute($jo_id_list);
+
+        unset($update_curr_stmt);
+    }
+
+    protected function loadDataFileForArchiveData($load_stmt)
+    {
+        $conn = $this->em->getConnection();
+        $conn->getWrappedConnection()->setAttribute(PDO::MYSQL_ATTR_USE_BUFFERED_QUERY, false);
+
+        $stmt = $conn->prepare($load_stmt);  
+
+        $result = $stmt->execute();
+
+        if (!$result)
+            error_log('Failed loading data.');
+
+        // TODO: delete file?
+    }
+
+    protected function cleanData($text)
+    {
+        $clean_text = '';
+
+        // replace the new lines with whitespace
+        $clean_text = preg_replace("/[\n\r]/", ' ', $text);
+
+        return $clean_text;
+
+    }
+
+} 

src/Command/SetJobOrderReferenceJOIdCommand.php

@@ -0,0 +1,53 @@
+<?php
+
+namespace App\Command;
+
+use Symfony\Component\Console\Command\Command;
+use Symfony\Component\Console\Input\InputArgument;
+use Symfony\Component\Console\Input\InputInterface;
+use Symfony\Component\Console\Output\OutputInterface;
+
+use Doctrine\ORM\EntityManagerInterface;
+
+class SetJobOrderReferenceJOIdCommand extends Command
+{
+    protected $em;
+
+    public function __construct(EntityManagerInterface $em)
+    {
+        $this->em = $em;
+
+        parent::__construct();
+    }
+
+    protected function configure()
+    {
+        $this->setName('joborder:setreferencejoid')
+            ->setDescription('Set job order reference jo id for existing job orders.')
+            ->setHelp('Set job order reference jo id for existing job orders. Decoupling job order from reference job order.');
+    }
+
+    protected function execute(InputInterface $input, OutputInterface $output)
+    {
+        // get the job orders where ref_jo is not null
+        $query = $this->em->createQuery('SELECT jo FROM App\Entity\JobOrder jo WHERE jo.ref_jo IS NOT NULL');
+
+        $jos = $query->getResult();
+
+        foreach ($jos as $jo)
+        {
+            $ref_jo_id = $jo->getReferenceJO()->getID();
+
+            error_log('Setting reference jo id ' . $ref_jo_id . ' for job order ' . $jo->getID());
+
+            $jo->setReferenceJOId($ref_jo_id);
+
+            // set the ref_jo to null to decouple ref jo and jo
+            $jo->setReferenceJO(null);
+        }
+
+        $this->em->flush(); 
+
+        return 0;
+    }
+}

src/Entity/JobOrder.php

@@ -441,6 +441,12 @@ class JobOrder
      */
     protected $flag_cust_new;
 
+    // reference JO id. We are now decoupling job order from itself
+    /**
+     * @ORM\Column(type="integer", nullable=true)
+     */
+    protected $reference_jo_id;
+
     public function __construct()
     {
         $this->date_create = new DateTime();
@@ -792,7 +798,7 @@ class JobOrder
         return $this->tickets;
     }
 
-    public function setReferenceJO(JobOrder $ref_jo)
+    public function setReferenceJO(JobOrder $ref_jo = null)
     {
         $this->ref_jo = $ref_jo;
         return $this;
@@ -1256,4 +1262,15 @@ class JobOrder
         return $this->flag_cust_new;
     }
 
+    public function setReferenceJOId($reference_jo_id)
+    {
+        $this->reference_jo_id = $reference_jo_id;
+        return $this;
+    }
+
+    public function getReferenceJOId()
+    {
+        return $this->reference_jo_id;
+    }
+
 }

src/Service/JobOrderHandler/ResqJobOrderHandler.php

@@ -551,7 +551,7 @@ class ResqJobOrderHandler implements JobOrderHandlerInterface
                 if (empty($ref_jo)) {
                     $error_array['ref_jo'] = 'Invalid reference job order specified.';
                 } else {
-                    $jo->setReferenceJO($ref_jo);
+                    $jo->setReferenceJOId($ref_jo->getID());
                 }
             }
 
@@ -2158,7 +2158,7 @@ class ResqJobOrderHandler implements JobOrderHandlerInterface
                 if (empty($ref_jo)) {
                     $error_array['ref_jo'] = 'Invalid reference job order specified.';
                 } else {
-                    $jo->setReferenceJO($ref_jo);
+                    $jo->setReferenceJOId($ref_jo->getID());
                 }
             }