From 7c49bd9f0ab21a45ce49e3ac73193b220d28291c Mon Sep 17 00:00:00 2001 From: Korina Cordero Date: Tue, 22 Sep 2020 11:17:25 +0000 Subject: [PATCH 1/3] Add ImportSAPDeltaCommand. #504 --- src/Command/ImportSAPDeltaCommand.php | 325 ++++++++++++++++++++++++++ 1 file changed, 325 insertions(+) create mode 100644 src/Command/ImportSAPDeltaCommand.php diff --git a/src/Command/ImportSAPDeltaCommand.php b/src/Command/ImportSAPDeltaCommand.php new file mode 100644 index 00000000..151e033c --- /dev/null +++ b/src/Command/ImportSAPDeltaCommand.php @@ -0,0 +1,325 @@ +em = $em; + $this->initSAPBatteryHash(); + $this->initSAPBatterySizeHash(); + $this->initSAPBatteryBrandHash(); + + parent::__construct(); + } + + protected function configure() + { + $this->setName('sap_battery:delta') + ->setDescription('Update SAP battery table with new SAP battery data.') + ->setHelp('Creates SAP battery data entries from CSV file.') + ->addArgument('file', InputArgument::REQUIRED, 'Path to the CSV file.') + ->addArgument('output_file', InputArgument::REQUIRED, 'Path to the output CSV file.'); + } + + protected function execute(InputInterface $input, OutputInterface $output) + { + $csv_file = $input->getArgument('file'); + $report_file = $input->getArgument('output_file'); + + // CSV column order: + // 0 - brand + // 1 - sku + // 2 - size + + // attempt to open file + try + { + $fh = fopen($csv_file, "r"); + } + catch (Exception $e) + { + throw new Exception('The file "' . $csv_file . '" could not be read.'); + } + + // get entity manager + $em = $this->em; + + // loop through the rows + $row_num = 0; + error_log('Processing sap battery csv file...'); + $existing_brands = []; + $existing_sizes = []; + $existing_batteries = []; + $added_brands = []; + $added_sizes = []; + $added_batteries = []; + while (($fields = fgetcsv($fh)) !== false) + { + // data starts at row 2 + if ($row_num < 2) + { + $row_num++; + continue; + } + + // check if blank row + if (strlen(trim($fields[0])) == 0) + continue; + + // clean up fields + $clean_brand = $this->normalizeName(trim($fields[0])); + $clean_sku = $this->normalizeName(trim($fields[1])); + $clean_size = $this->normalizeName(trim($fields[2])); + + // check if sap battery code already exists + if (!isset($this->sap_battery_hash[$clean_sku])) + { + // check if brand in system + $sap_brand = null; + if (!isset($this->sap_battery_brand_hash[$clean_brand])) + { + // add brand + $sap_brand = new SAPBatteryBrand(); + $sap_brand->setName($clean_brand); + + $em->persist($sap_brand); + + // add to list of added brands + $added_brands[$clean_brand] = $clean_brand; + + // add to hash + $this->sap_battery_brand_hash[$clean_brand] = $sap_brand; + } + else + { + // get the brand + $sap_brand = $this->sap_battery_brand_hash[$clean_brand]; + + // need to check if this is in added_brands because the hash contains + // existing + added. What we need is the list of existing only + if (!isset($added_brands[$clean_brand])) + { + // add to list of existing brands + $existing_brands[$clean_brand] = $sap_brand; + } + } + + // check if size in system + $sap_size = null; + if (!isset($this->sap_battery_size_hash[$clean_size])) + { + // add size + $sap_size = new SAPBatterySize(); + $sap_size->setName($clean_size); + + $em->persist($sap_size); + + // add to list of added sizes + $added_sizes[$clean_size] = $clean_size; + + // add to hash + $this->sap_battery_size_hash[$clean_size] = $sap_size; + } + else + { + // get the size + $sap_size = $this->sap_battery_size_hash[$clean_size]; + + // need to check if this is in added_sizes because the hash contains + // existing + added. What we need is the list of existing only + if (!isset($added_sizes[$clean_size])) + { + // add to list of existing sizes + $existing_sizes[$clean_size] = $sap_size; + } + } + + // add the sap_battery + // create sap battery entry + $sap_battery = new SAPBattery(); + $sap_battery->setID($clean_sku) + ->setSize($sap_size) + ->setBrand($sap_brand); + + $em->persist($sap_battery); + + // add to list of added batteries + $added_batteries[$clean_sku] = $clean_sku; + + // add to hash + $this->sap_battery_hash[$clean_sku] = $sap_battery; + } + else + { + // TODO: for now, assume that the brand and size in system == brand and size in csv file + // need to check if this is in added_batteries because the hash contains + // existing + added. What we need is the list of existing only + if (!isset($added_batteries[$clean_sku])) + { + // add to list of existing batteries + $existing_batteries[$clean_sku] = $this->sap_battery_hash[$clean_sku]; + } + } + + $row_num++; + + } + + $em->flush(); + + // need to output the list of added and not added data + $this->writeOutputFile($report_file, $added_brands, $added_sizes, $added_batteries, + $existing_brands, $existing_sizes, $existing_batteries); + + return 0; + } + + protected function initSAPBatteryHash() + { + $this->sap_battery_hash = []; + + $batts = $this->em->getRepository(SAPBattery::class)->findAll(); + foreach ($batts as $batt) + { + $id = $this->normalizeName($batt->getID()); + $this->sap_battery_hash[$id] = $batt; + } + } + + protected function initSAPBatterySizeHash() + { + $this->sap_battery_size_hash = []; + + $sizes = $this->em->getRepository(SAPBatterySize::class)->findAll(); + foreach ($sizes as $size) + { + $name = $this->normalizeName($size->getName()); + $this->sap_battery_size_hash[$name] = $size; + } + } + + protected function initSAPBatteryBrandHash() + { + $this->sap_battery_brand_hash = []; + + $brands = $this->em->getRepository(SAPBatteryBrand::class)->findAll(); + foreach ($brands as $brand) + { + $name = $this->normalizeName($brand->getName()); + $this->sap_battery_brand_hash[$name] = $brand; + } + } + + protected function writeOutputFile($report_file, $added_brands, $added_sizes, $added_batteries, + $existing_brands, $existing_sizes, $existing_batteries) + { + try + { + $fh = fopen($report_file, "w"); + } + catch (Exception $e) + { + throw new Exception('The file "' . $report_file . '" could be opened.'); + } + + fputs($fh, 'Total brands added: ' . count($added_brands) . "\n"); + fputs($fh, 'Total sizes added: ' . count($added_sizes) . "\n"); + fputs($fh, 'Total batteries added: ' . count($added_batteries) . "\n"); + + // write the added batteries + // check the existing brands array for the added ones + $not_added_batteries = []; + fputs($fh, 'Added Batteries: ' . "\n"); + foreach($added_batteries as $added_battery) + { + fputs($fh, $added_battery, strlen($added_battery)); + fputs($fh, "\n"); + + if (isset($existing_batteries[$added_battery])) + { + $not_added_batteries[] = $added_battery; + } + } + + // write the added brands + // check the existing arrays for the added ones + $not_added_brands = []; + fputs($fh, 'Added Brands: ' . "\n"); + foreach ($added_brands as $added_brand) + { + fputs($fh, $added_brand, strlen($added_brand)); + fputs($fh, "\n"); + + if (isset($existing_brands[$added_brand])) + { + $not_added_brands[] = $added_brand; + } + } + + // write the added sizes + // check the existing array for the added ones + $not_added_sizes = []; + fputs($fh, 'Added Sizes: ' . "\n"); + foreach ($added_sizes as $added_size) + { + fputs($fh, $added_size, strlen($added_size)); + fputs($fh, "\n"); + + if (isset($existing_sizes[$added_size])) + { + $not_added_sizes[] = $added_size; + } + } + + // write the not added batteries + fputs($fh, 'Batteries already in system: ' . "\n"); + foreach($not_added_batteries as $not_added_battery) + { + fputs($fh, $not_added_battery, strlen($not_added_battery)); + fputs($fh, "\n"); + } + + // write the not added brands + fputs($fh, 'Brands already in system: ' . "\n"); + foreach($not_added_brands as $not_added_brand) + { + fputs($fh, $not_added_brand, strlen($not_added_brand)); + fputs($fh, "\n"); + } + + // write the not added sizes + fputs($fh, 'Sizes already in system: ' . "\n"); + foreach($not_added_sizes as $not_added_size) + { + fputs($fh, $not_added_size, strlen($not_added_size)); + fputs($fh, "\n"); + } + + fclose($fh); + } + + protected function normalizeName($name) + { + $normalized_key = trim(strtoupper($name)); + + return $normalized_key; + } +} From beee87c3b46bd4ce890eb30ab2ab5e61c34e1ede Mon Sep 17 00:00:00 2001 From: Korina Cordero Date: Wed, 23 Sep 2020 04:47:15 +0000 Subject: [PATCH 2/3] Modify the ImportSAPDeltaCommand to make import checking easier. #504 --- src/Command/ImportSAPDeltaCommand.php | 52 +++++++++++++++------------ 1 file changed, 29 insertions(+), 23 deletions(-) diff --git a/src/Command/ImportSAPDeltaCommand.php b/src/Command/ImportSAPDeltaCommand.php index 151e033c..aeea6366 100644 --- a/src/Command/ImportSAPDeltaCommand.php +++ b/src/Command/ImportSAPDeltaCommand.php @@ -71,6 +71,7 @@ class ImportSAPDeltaCommand extends Command $added_brands = []; $added_sizes = []; $added_batteries = []; + $total_processed = 0; while (($fields = fgetcsv($fh)) !== false) { // data starts at row 2 @@ -105,6 +106,8 @@ class ImportSAPDeltaCommand extends Command // add to list of added brands $added_brands[$clean_brand] = $clean_brand; + //$output->writeln('Adding brand: ' . $clean_brand); + // add to hash $this->sap_battery_brand_hash[$clean_brand] = $sap_brand; } @@ -118,7 +121,8 @@ class ImportSAPDeltaCommand extends Command if (!isset($added_brands[$clean_brand])) { // add to list of existing brands - $existing_brands[$clean_brand] = $sap_brand; + //$output->writeln('Brand already in db ' . $clean_brand); + $existing_brands[$clean_brand] = $clean_brand; } } @@ -135,6 +139,8 @@ class ImportSAPDeltaCommand extends Command // add to list of added sizes $added_sizes[$clean_size] = $clean_size; + //$output->writeln('Adding size: ' . $clean_size); + // add to hash $this->sap_battery_size_hash[$clean_size] = $sap_size; } @@ -148,7 +154,8 @@ class ImportSAPDeltaCommand extends Command if (!isset($added_sizes[$clean_size])) { // add to list of existing sizes - $existing_sizes[$clean_size] = $sap_size; + //$output->writeln('Size already in db ' . $clean_size); + $existing_sizes[$clean_size] = $clean_size; } } @@ -164,6 +171,8 @@ class ImportSAPDeltaCommand extends Command // add to list of added batteries $added_batteries[$clean_sku] = $clean_sku; + //$output->writeln('Adding battery: ' . $clean_sku); + // add to hash $this->sap_battery_hash[$clean_sku] = $sap_battery; } @@ -175,10 +184,16 @@ class ImportSAPDeltaCommand extends Command if (!isset($added_batteries[$clean_sku])) { // add to list of existing batteries + //$output->writeln('Battery already in db ' . $clean_sku); $existing_batteries[$clean_sku] = $this->sap_battery_hash[$clean_sku]; + if (!isset($existing_brands[$clean_brand])) + $existing_brands[$clean_brand] = $clean_brand; + if (!isset($existing_sizes[$clean_size])) + $existing_sizes[$clean_size] = $clean_size; } } + $total_processed++; $row_num++; } @@ -187,7 +202,7 @@ class ImportSAPDeltaCommand extends Command // need to output the list of added and not added data $this->writeOutputFile($report_file, $added_brands, $added_sizes, $added_batteries, - $existing_brands, $existing_sizes, $existing_batteries); + $existing_brands, $existing_sizes, $existing_batteries, $total_processed); return 0; } @@ -229,7 +244,7 @@ class ImportSAPDeltaCommand extends Command } protected function writeOutputFile($report_file, $added_brands, $added_sizes, $added_batteries, - $existing_brands, $existing_sizes, $existing_batteries) + $existing_brands, $existing_sizes, $existing_batteries, $total_processed) { try { @@ -240,10 +255,16 @@ class ImportSAPDeltaCommand extends Command throw new Exception('The file "' . $report_file . '" could be opened.'); } + fputs($fh, 'Total entries processed: ' . $total_processed . "\n"); + fputs($fh, 'Total brands added: ' . count($added_brands) . "\n"); fputs($fh, 'Total sizes added: ' . count($added_sizes) . "\n"); fputs($fh, 'Total batteries added: ' . count($added_batteries) . "\n"); + fputs($fh, 'Total number of brands in csv file that are in the system: ' . count($existing_brands) . "\n"); + fputs($fh, 'Total number of sizes in csv file that are in the system: ' . count($existing_sizes) . "\n"); + fputs($fh, 'Total number of batteries in csv file that are in the system: ' . count($existing_batteries) . "\n"); + // write the added batteries // check the existing brands array for the added ones $not_added_batteries = []; @@ -252,11 +273,6 @@ class ImportSAPDeltaCommand extends Command { fputs($fh, $added_battery, strlen($added_battery)); fputs($fh, "\n"); - - if (isset($existing_batteries[$added_battery])) - { - $not_added_batteries[] = $added_battery; - } } // write the added brands @@ -267,11 +283,6 @@ class ImportSAPDeltaCommand extends Command { fputs($fh, $added_brand, strlen($added_brand)); fputs($fh, "\n"); - - if (isset($existing_brands[$added_brand])) - { - $not_added_brands[] = $added_brand; - } } // write the added sizes @@ -282,24 +293,19 @@ class ImportSAPDeltaCommand extends Command { fputs($fh, $added_size, strlen($added_size)); fputs($fh, "\n"); - - if (isset($existing_sizes[$added_size])) - { - $not_added_sizes[] = $added_size; - } } // write the not added batteries fputs($fh, 'Batteries already in system: ' . "\n"); - foreach($not_added_batteries as $not_added_battery) + foreach($existing_batteries as $not_added_battery) { - fputs($fh, $not_added_battery, strlen($not_added_battery)); + fputs($fh, $not_added_battery->getID(), strlen($not_added_battery->getID())); fputs($fh, "\n"); } // write the not added brands fputs($fh, 'Brands already in system: ' . "\n"); - foreach($not_added_brands as $not_added_brand) + foreach($existing_brands as $not_added_brand) { fputs($fh, $not_added_brand, strlen($not_added_brand)); fputs($fh, "\n"); @@ -307,7 +313,7 @@ class ImportSAPDeltaCommand extends Command // write the not added sizes fputs($fh, 'Sizes already in system: ' . "\n"); - foreach($not_added_sizes as $not_added_size) + foreach($existing_sizes as $not_added_size) { fputs($fh, $not_added_size, strlen($not_added_size)); fputs($fh, "\n"); From 7107c57eb60ac12ffc9a0fc743d5e2b1b6bbd4f3 Mon Sep 17 00:00:00 2001 From: Korina Cordero Date: Fri, 25 Sep 2020 09:58:07 +0000 Subject: [PATCH 3/3] Create command to import SAP information. #504 --- src/Command/ImportSAPMasterCommand.php | 256 +++++++++++++++++++++++++ 1 file changed, 256 insertions(+) create mode 100644 src/Command/ImportSAPMasterCommand.php diff --git a/src/Command/ImportSAPMasterCommand.php b/src/Command/ImportSAPMasterCommand.php new file mode 100644 index 00000000..9e266c2b --- /dev/null +++ b/src/Command/ImportSAPMasterCommand.php @@ -0,0 +1,256 @@ +em = $em; + $this->initSAPBatteryHash(); + $this->initSAPBatterySizeHash(); + $this->initSAPBatteryBrandHash(); + + parent::__construct(); + } + + protected function configure() + { + $this->setName('sap_battery:master') + ->setDescription('Import SAP battery master list.') + ->setHelp('Creates SAP battery, SAP battery brands, and SAP battery sizes from CSV file.') + ->addArgument('file', InputArgument::REQUIRED, 'Path to the CSV file.') + ->addArgument('output_file', InputArgument::REQUIRED, 'Path to the output CSV file.'); + } + + protected function execute(InputInterface $input, OutputInterface $output) + { + $csv_file = $input->getArgument('file'); + $report_file = $input->getArgument('output_file'); + + // CSV column order: + // 0 - brand + // 1 - sku + // 2 - size + + // attempt to open file + try + { + $fh = fopen($csv_file, "r"); + } + catch (Exception $e) + { + throw new Exception('The file "' . $csv_file . '" could not be read.'); + } + + // get entity manager + $em = $this->em; + // loop through the rows + $row_num = 0; + error_log('Processing sap battery csv file...'); + $added_brands = []; + $added_sizes = []; + $added_batteries = []; + $repeat_brands = []; + $repeat_sizes = []; + $repeat_batteries = []; + $total_processed = 0; + while (($fields = fgetcsv($fh)) !== false) + { + // data starts at row 2 + if ($row_num < 2) + { + $row_num++; + continue; + } + + // check if blank row + if (strlen(trim($fields[0])) == 0) + continue; + + // clean up fields + $clean_brand = $this->normalizeName(trim($fields[0])); + $clean_sku = $this->normalizeName(trim($fields[1])); + $clean_size = $this->normalizeName(trim($fields[2])); + + // check if brand in system + $sap_brand = null; + if (!isset($this->sap_battery_brand_hash[$clean_brand])) + { + // add brand + $sap_brand = new SAPBatteryBrand(); + $sap_brand->setName($clean_brand); + + $em->persist($sap_brand); + + // add to list of added brands + $added_brands[$clean_brand] = $clean_brand; + + //$output->writeln('Adding brand: ' . $clean_brand); + + // add to hash + $this->sap_battery_brand_hash[$clean_brand] = $sap_brand; + } + else + { + // get the brand + $sap_brand = $this->sap_battery_brand_hash[$clean_brand]; + + // repeat brand so no need to add but still need to count + $repeat_brands[] = $clean_brand; + } + + // check if size in system + $sap_size = null; + if (!isset($this->sap_battery_size_hash[$clean_size])) + { + // add size + $sap_size = new SAPBatterySize(); + $sap_size->setName($clean_size); + + $em->persist($sap_size); + + // add to list of added sizes + $added_sizes[$clean_size] = $clean_size; + + //$output->writeln('Adding size: ' . $clean_size); + + // add to hash + $this->sap_battery_size_hash[$clean_size] = $sap_size; + } + else + { + // get the size + $sap_size = $this->sap_battery_size_hash[$clean_size]; + + // repeat size so no need to add to system but still need to count + $repeat_sizes[] = $clean_size; + } + + // check if sap battery code already exists + if (!isset($this->sap_battery_hash[$clean_sku])) + { + // add the sap_battery + // create sap battery entry + $sap_battery = new SAPBattery(); + $sap_battery->setID($clean_sku) + ->setSize($sap_size) + ->setBrand($sap_brand); + + $em->persist($sap_battery); + + // add to list of added batteries + $added_batteries[$clean_sku] = $clean_sku; + + //$output->writeln('Adding battery: ' . $clean_sku); + + // add to hash + $this->sap_battery_hash[$clean_sku] = $sap_battery; + } + else + { + $repeat_batteries[] = $clean_sku; + } + + $total_processed++; + $row_num++; + } + + $em->flush(); + + // need to output the list of added and not added data + $this->writeOutputFile($report_file, $added_brands, $added_sizes, $added_batteries, $total_processed, + $repeat_brands, $repeat_sizes, $repeat_batteries); + + return 0; + } + + protected function initSAPBatteryHash() + { + $this->sap_battery_hash = []; + + $batts = $this->em->getRepository(SAPBattery::class)->findAll(); + foreach ($batts as $batt) + { + $id = $this->normalizeName($batt->getID()); + $this->sap_battery_hash[$id] = $batt; + } + } + + protected function initSAPBatterySizeHash() + { + $this->sap_battery_size_hash = []; + + $sizes = $this->em->getRepository(SAPBatterySize::class)->findAll(); + foreach ($sizes as $size) + { + $name = $this->normalizeName($size->getName()); + $this->sap_battery_size_hash[$name] = $size; + } + } + + protected function initSAPBatteryBrandHash() + { + $this->sap_battery_brand_hash = []; + + $brands = $this->em->getRepository(SAPBatteryBrand::class)->findAll(); + foreach ($brands as $brand) + { + $name = $this->normalizeName($brand->getName()); + $this->sap_battery_brand_hash[$name] = $brand; + } + } + + protected function writeOutputFile($report_file, $added_brands, $added_sizes, $added_batteries, $total_processed, + $repeat_brands, $repeat_sizes, $repeat_batteries) + { + try + { + $fh = fopen($report_file, "w"); + } + catch (Exception $e) + { + throw new Exception('The file "' . $report_file . '" could be opened.'); + } + + fputs($fh, 'Total entries processed: ' . $total_processed . "\n"); + + fputs($fh, 'Total brands added: ' . count($added_brands) . "\n"); + fputs($fh, 'Total sizes added: ' . count($added_sizes) . "\n"); + fputs($fh, 'Total batteries added: ' . count($added_batteries) . "\n"); + + fputs($fh, 'Repeat brands: ' . count($repeat_brands) . "\n"); + fputs($fh, 'Repeat sizes: ' . count($repeat_sizes) . "\n"); + fputs($fh, 'Repeat batteries: ' . count($repeat_batteries) . "\n"); + + foreach ($repeat_batteries as $repeat_battery) + { + fputs($fh, 'Duplicate SKU: ' . $repeat_battery . "\n"); + } + + } + + protected function normalizeName($name) + { + $normalized_key = trim(strtoupper($name)); + + return $normalized_key; + } + +}