diff --git a/composer.json b/composer.json
index 68b7bb4..b565655 100644
--- a/composer.json
+++ b/composer.json
@@ -2,14 +2,14 @@
"name": "honl/magento2-import",
"description": "",
"require": {
- "php": "~7.1.0 || ~7.2.0 || ~7.3.0 || ~7.4.0 || ~8.2",
+ "php": "~7.1.0 || ~7.2.0 || ~7.3.0 || ~7.4.0 || ~8.1 || ~8.2",
"guzzlehttp/guzzle": "^7",
"prewk/xml-string-streamer": "^1.2",
"prewk/xml-string-streamer-guzzle": "^1.2",
"kevinrob/guzzle-cache-middleware": "^4",
- "ecomdev/magento-psr6-bridge": "^0.2.1",
+ "ho-nl-fork/magento-psr6-bridge": "dev-master",
"symfony/stopwatch": "^v6.2",
- "bakame/psr7-csv-factory": "^1.0@dev",
+ "ho-nl-fork/psr7-csv-factory": "dev-master",
"league/csv": "^9.1",
"monolog/monolog": "^2.8",
"ext-mbstring": "*"
diff --git a/src/Api/ImportProfileInterface.php b/src/Api/ImportProfileInterface.php
index 4cf5bbe..ede89cf 100644
--- a/src/Api/ImportProfileInterface.php
+++ b/src/Api/ImportProfileInterface.php
@@ -15,6 +15,8 @@ interface ImportProfileInterface
*/
public function getItems();
+ public function getProcessedItems(): array;
+
/**
* Array of the config values
*
diff --git a/src/Model/ImportProfile.php b/src/Model/ImportProfile.php
index 67e5159..5cab0c1 100644
--- a/src/Model/ImportProfile.php
+++ b/src/Model/ImportProfile.php
@@ -52,6 +52,8 @@ abstract class ImportProfile implements ImportProfileInterface
*/
private $errors = null;
+ private ?array $processedItems = null;
+
/**
* @param ObjectManagerFactory $objectManagerFactory
* @param Stopwatch $stopwatch
@@ -89,7 +91,10 @@ public function run()
$errors = $importer->processImport($items);
$stopwatchEvent = $this->stopwatch->stop('importinstance');
- $output = (string) new Phrase('%1 items imported in %2 sec, %3 items / sec (%4mb used)', [
+ $message = $errors
+ ? 'Tried to import %1 items in %2 sec, %3 items / sec (%4mb used)'
+ : '%1 items imported in %2 sec, %3 items / sec (%4mb used)';
+ $output = (string) new Phrase($message, [
count($items),
round($stopwatchEvent->getDuration() / 1000, 1),
round(count($items) / ($stopwatchEvent->getDuration() / 1000), 1),
@@ -102,6 +107,7 @@ public function run()
$this->consoleOutput->writeln("$errors");
$this->log->error($errors);
+ $this->processedItems = $items;
$this->errors = $errors;
return \Magento\Framework\Console\Cli::RETURN_SUCCESS;
@@ -135,6 +141,11 @@ public function getErrors()
return $this->errors;
}
+ public function getProcessedItems(): array
+ {
+ return $this->processedItems;
+ }
+
/**
* Get all items that need to be imported
*
diff --git a/src/Rewrite/ImportExport/Import/Product.php b/src/Rewrite/ImportExport/Import/Product.php
index 91487d9..75fd9c8 100644
--- a/src/Rewrite/ImportExport/Import/Product.php
+++ b/src/Rewrite/ImportExport/Import/Product.php
@@ -12,10 +12,16 @@
use Magento\Catalog\Model\Config as CatalogConfig;
use Magento\Catalog\Model\Product\Visibility;
use Magento\CatalogImportExport\Model\Import\Product\ImageTypeProcessor;
+use Magento\CatalogImportExport\Model\Import\Product\LinkProcessor;
use Magento\CatalogImportExport\Model\Import\Product\MediaGalleryProcessor;
use Magento\CatalogImportExport\Model\Import\Product\RowValidatorInterface as ValidatorInterface;
+use Magento\CatalogImportExport\Model\Import\Product\SkuStorage;
+use Magento\CatalogImportExport\Model\Import\Product\StatusProcessor;
+use Magento\CatalogImportExport\Model\Import\Product\StockProcessor;
use Magento\CatalogImportExport\Model\StockItemImporterInterface;
+use Magento\CatalogImportExport\Model\StockItemProcessorInterface;
use Magento\Framework\App\ObjectManager;
+use Magento\Framework\Filesystem\Driver\File;
use Magento\Framework\Intl\DateTimeFactory;
use Magento\Framework\Model\ResourceModel\Db\ObjectRelationProcessor;
use Magento\Framework\Model\ResourceModel\Db\TransactionManagerInterface;
@@ -25,68 +31,19 @@
use Magento\ImportExport\Model\Import\ErrorProcessing\ProcessingErrorAggregatorInterface;
use Magento\Store\Model\Store;
+/**
+ * @prettier-ignore
+ */
class Product extends \Magento\CatalogImportExport\Model\Import\Product
{
const SKIP_ATTRIBUTES_WHEN_UPDATING = '_import_skip_attributes_when_updating';
- /** @var LineFormatterMulti $lineFormatterMulti */
- private $lineFormatterMulti;
-
- /** @var CatalogConfig $catalogConfig */
- private $catalogConfig;
-
+ private LineFormatterMulti $lineFormatterMulti;
+ private ?CatalogConfig $catalogConfig;
/** @var string $productEntityLinkField */
private $productEntityLinkField;
+ private ?SkuStorage $skuStorage;
- /**
- * @param \Magento\Framework\Json\Helper\Data $jsonHelper
- * @param \Magento\ImportExport\Helper\Data $importExportData
- * @param \Magento\ImportExport\Model\ResourceModel\Import\Data $importData
- * @param \Magento\Eav\Model\Config $config
- * @param \Magento\Framework\App\ResourceConnection $resource
- * @param \Magento\ImportExport\Model\ResourceModel\Helper $resourceHelper
- * @param \Magento\Framework\Stdlib\StringUtils $string
- * @param ProcessingErrorAggregatorInterface $errorAggregator
- * @param \Magento\Framework\Event\ManagerInterface $eventManager
- * @param \Magento\CatalogInventory\Api\StockRegistryInterface $stockRegistry
- * @param \Magento\CatalogInventory\Api\StockConfigurationInterface $stockConfiguration
- * @param \Magento\CatalogInventory\Model\Spi\StockStateProviderInterface $stockStateProvider
- * @param \Magento\Catalog\Helper\Data $catalogData
- * @param Import\Config $importConfig
- * @param \Magento\CatalogImportExport\Model\Import\Proxy\Product\ResourceModelFactory $resourceFactory
- * @param \Magento\CatalogImportExport\Model\Import\Product $optionFactory
- * @param \Magento\Eav\Model\ResourceModel\Entity\Attribute\Set\CollectionFactory $setColFactory
- * @param \Magento\CatalogImportExport\Model\Import\Product $productTypeFactory
- * @param \Magento\Catalog\Model\ResourceModel\Product\LinkFactory $linkFactory
- * @param \Magento\CatalogImportExport\Model\Import\Proxy\ProductFactory $proxyProdFactory
- * @param \Magento\CatalogImportExport\Model\Import\UploaderFactory $uploaderFactory
- * @param \Magento\Framework\Filesystem $filesystem
- * @param \Magento\CatalogInventory\Model\ResourceModel\Stock\ItemFactory $stockResItemFac
- * @param DateTime\TimezoneInterface $localeDate
- * @param DateTime $dateTime
- * @param \Psr\Log\LoggerInterface $logger
- * @param \Magento\Framework\Indexer\IndexerRegistry $indexerRegistry
- * @param \Magento\CatalogImportExport\Model\Import\Product $storeResolver
- * @param \Magento\CatalogImportExport\Model\Import\Product $skuProcessor
- * @param \Magento\CatalogImportExport\Model\Import\Product $categoryProcessor
- * @param \Magento\CatalogImportExport\Model\Import\Product $validator
- * @param ObjectRelationProcessor $objectRelationProcessor
- * @param TransactionManagerInterface $transactionManager
- * @param \Magento\CatalogImportExport\Model\Import\Product $taxClassProcessor
- * @param \Magento\Framework\App\Config\ScopeConfigInterface $scopeConfig
- * @param \Magento\Catalog\Model\Product\Url $productUrl
- * @param array $data
- * @param array $dateAttrCodes
- * @param CatalogConfig|null $catalogConfig
- * @param ImageTypeProcessor|null $imageTypeProcessor
- * @param MediaGalleryProcessor|null $mediaProcessor
- * @param StockItemImporterInterface|null $stockItemImporter
- * @param DateTimeFactory|null $dateTimeFactory
- * @param ProductRepositoryInterface|null $productRepository
- *
- * @throws \Magento\Framework\Exception\FileSystemException
- * @throws \Magento\Framework\Exception\LocalizedException
- */
public function __construct(
\Magento\Framework\Json\Helper\Data $jsonHelper,
\Magento\ImportExport\Helper\Data $importExportData,
@@ -133,109 +90,70 @@ public function __construct(
StockItemImporterInterface $stockItemImporter = null,
DateTimeFactory $dateTimeFactory = null,
ProductRepositoryInterface $productRepository = null,
- $statusProcessor = null,
- $stockProcessor = null
+ StatusProcessor $statusProcessor = null,
+ StockProcessor $stockProcessor = null,
+ LinkProcessor $linkProcessor = null,
+ ?File $fileDriver = null,
+ ?StockItemProcessorInterface $stockItemProcessor = null,
+ ?SkuStorage $skuStorage = null
) {
- if ($statusProcessor && \is_a($statusProcessor, 'Magento\CatalogImportExport\Model\Import\Product\StatusProcessor')) {
- parent::__construct(
- $jsonHelper,
- $importExportData,
- $importData,
- $config,
- $resource,
- $resourceHelper,
- $string,
- $errorAggregator,
- $eventManager,
- $stockRegistry,
- $stockConfiguration,
- $stockStateProvider,
- $catalogData,
- $importConfig,
- $resourceFactory,
- $optionFactory,
- $setColFactory,
- $productTypeFactory,
- $linkFactory,
- $proxyProdFactory,
- $uploaderFactory,
- $filesystem,
- $stockResItemFac,
- $localeDate,
- $dateTime,
- $logger,
- $indexerRegistry,
- $storeResolver,
- $skuProcessor,
- $categoryProcessor,
- $validator,
- $objectRelationProcessor,
- $transactionManager,
- $taxClassProcessor,
- $scopeConfig,
- $productUrl,
- $data,
- $dateAttrCodes,
- $catalogConfig,
- $imageTypeProcessor,
- $mediaProcessor,
- $stockItemImporter,
- $dateTimeFactory,
- $productRepository,
- $statusProcessor,
- $stockProcessor
- );
- } else {
- parent::__construct(
- $jsonHelper,
- $importExportData,
- $importData,
- $config,
- $resource,
- $resourceHelper,
- $string,
- $errorAggregator,
- $eventManager,
- $stockRegistry,
- $stockConfiguration,
- $stockStateProvider,
- $catalogData,
- $importConfig,
- $resourceFactory,
- $optionFactory,
- $setColFactory,
- $productTypeFactory,
- $linkFactory,
- $proxyProdFactory,
- $uploaderFactory,
- $filesystem,
- $stockResItemFac,
- $localeDate,
- $dateTime,
- $logger,
- $indexerRegistry,
- $storeResolver,
- $skuProcessor,
- $categoryProcessor,
- $validator,
- $objectRelationProcessor,
- $transactionManager,
- $taxClassProcessor,
- $scopeConfig,
- $productUrl,
- $data,
- $dateAttrCodes,
- $catalogConfig,
- $imageTypeProcessor,
- $mediaProcessor,
- $stockItemImporter,
- $dateTimeFactory,
- $productRepository
- );
- }
+ parent::__construct(
+ $jsonHelper,
+ $importExportData,
+ $importData,
+ $config,
+ $resource,
+ $resourceHelper,
+ $string,
+ $errorAggregator,
+ $eventManager,
+ $stockRegistry,
+ $stockConfiguration,
+ $stockStateProvider,
+ $catalogData,
+ $importConfig,
+ $resourceFactory,
+ $optionFactory,
+ $setColFactory,
+ $productTypeFactory,
+ $linkFactory,
+ $proxyProdFactory,
+ $uploaderFactory,
+ $filesystem,
+ $stockResItemFac,
+ $localeDate,
+ $dateTime,
+ $logger,
+ $indexerRegistry,
+ $storeResolver,
+ $skuProcessor,
+ $categoryProcessor,
+ $validator,
+ $objectRelationProcessor,
+ $transactionManager,
+ $taxClassProcessor,
+ $scopeConfig,
+ $productUrl,
+ $data,
+ $dateAttrCodes,
+ $catalogConfig,
+ $imageTypeProcessor,
+ $mediaProcessor,
+ $stockItemImporter,
+ $dateTimeFactory,
+ $productRepository,
+ $statusProcessor,
+ $stockProcessor,
+ $linkProcessor,
+ $fileDriver,
+ $stockItemProcessor,
+ $skuStorage,
+ );
$this->lineFormatterMulti = $lineFormatterMulti;
$this->catalogConfig = $catalogConfig ?: ObjectManager::getInstance()->get(CatalogConfig::class);
+ $this->skuStorage = $skuStorage ?? ObjectManager::getInstance()
+ ->get(SkuStorage::class);
}
/**
@@ -316,7 +234,7 @@ protected function _saveProducts()
}
// 1. Entity phase
- if (isset($this->_oldSku[\strtolower($rowSku)])) {
+ if ($this->isSkuExist($rowSku)) {
// existing row
if (isset($rowData['attribute_set_code'])) {
$attributeSetId = $this->catalogConfig->getAttributeSetId(
@@ -337,10 +255,11 @@ protected function _saveProducts()
$attributeSetId = $this->skuProcessor->getNewSku($rowSku)['attr_set_id'];
}
// existing row
+ $entityLinkField = $this->getProductEntityLinkField();
$entityRowsUp[] = [
'updated_at' => (new \DateTime())->format(DateTime::DATETIME_PHP_FORMAT),
'attribute_set_id' => $attributeSetId,
- $this->getProductEntityLinkField() => $this->_oldSku[\strtolower($rowSku)][$this->getProductEntityLinkField()],
+ $entityLinkField => $this->getExistingSku($rowSku)[$entityLinkField]
];
} else {
if (!$productLimit || $productsQty < $productLimit) {
@@ -505,7 +424,7 @@ protected function _saveProducts()
$rowData = $productTypeModel->prepareAttributesWithDefaultValueForSave(
$rowData,
- !isset($this->_oldSku[strtolower($rowSku)])
+ !$this->isSkuExist($rowSku)
);
$product = $this->_proxyProdFactory->create(['data' => $rowData]);
@@ -551,7 +470,7 @@ protected function _saveProducts()
} elseif (self::SCOPE_STORE == $attribute->getIsGlobal()) {
$storeIds = [$rowStore];
}
- if (!isset($this->_oldSku[strtolower($rowSku)])) {
+ if (!$this->isSkuExist($rowSku)) {
$storeIds[] = 0;
}
}
@@ -595,9 +514,11 @@ protected function _saveProducts()
return $this;
}
- public function skipUpdatingAttribute($rowSku, $attrCode)
+ public function skipUpdatingAttribute($rowSku, $attrCode): bool
{
- return isset($this->_oldSku[strtolower($rowSku)]) && \array_key_exists(self::SKIP_ATTRIBUTES_WHEN_UPDATING, $this->_parameters) && \array_key_exists($attrCode, $this->_parameters[self::SKIP_ATTRIBUTES_WHEN_UPDATING]);
+ return $this->isSkuExist($rowSku)
+ && \array_key_exists(self::SKIP_ATTRIBUTES_WHEN_UPDATING, $this->_parameters)
+ && \array_key_exists($attrCode, $this->_parameters[self::SKIP_ATTRIBUTES_WHEN_UPDATING]);
}
/**
@@ -633,11 +554,15 @@ public function validateRow(array $rowData, $rowNum)
// check that row is already validated
return !$this->getErrorAggregator()->isRowInvalid($rowNum);
}
+
$this->_validatedRows[$rowNum] = true;
+
$rowScope = $this->getRowScope($rowData);
+ $sku = $rowData[self::COL_SKU];
+
// BEHAVIOR_DELETE use specific validation logic
if (Import::BEHAVIOR_DELETE == $this->getBehavior()) {
- if (self::SCOPE_DEFAULT == $rowScope && !isset($this->_oldSku[strtolower($rowData[self::COL_SKU])])) {
+ if (self::SCOPE_DEFAULT == $rowScope && !$this->isSkuExist($sku)) {
$this->addRowError(ValidatorInterface::ERROR_SKU_NOT_FOUND_FOR_DELETE, $rowNum);
return false;
}
@@ -648,7 +573,7 @@ public function validateRow(array $rowData, $rowNum)
$this->addRowError($message, $rowNum, $this->validator->getInvalidAttribute());
}
}
- $sku = $rowData[self::COL_SKU];
+
if (null === $sku) {
$this->addRowError(ValidatorInterface::ERROR_SKU_IS_EMPTY, $rowNum);
} elseif (false === $sku) {
@@ -660,11 +585,11 @@ public function validateRow(array $rowData, $rowNum)
}
// SKU is specified, row is SCOPE_DEFAULT, new product block begins
$this->_processedEntitiesCount++;
- $sku = $rowData[self::COL_SKU];
- if (isset($this->_oldSku[\strtolower($sku)])) {
+
+ if ($this->isSkuExist($sku)) {
// can we get all necessary data from existent DB product?
// check for supported type of existing product
- if (isset($this->_productTypeModels[$this->_oldSku[\strtolower($sku)]['type_id']])) {
+ if (isset($this->_productTypeModels[$this->getExistingSku($sku)['type_id']])) {
$this->skuProcessor->addNewSku(
$sku,
$this->prepareNewSkuData($sku)
@@ -709,7 +634,7 @@ public function validateRow(array $rowData, $rowNum)
$rowAttributesValid = $this->_productTypeModels[$newSku['type_id']]->isRowValid(
$rowData,
$rowNum,
- !isset($this->_oldSku[\strtolower($sku)])
+ !($this->isSkuExist($sku))
);
if (!$rowAttributesValid && self::SCOPE_DEFAULT == $rowScope) {
// mark SCOPE_DEFAULT row as invalid for future child rows if product not in DB already
@@ -746,7 +671,7 @@ public function validateRow(array $rowData, $rowNum)
*/
private function isNeedToValidateUrlKey($rowData)
{
- return (!empty($rowData[self::URL_KEY]) || !empty($rowData[self::COL_NAME]))
+ return !empty($rowData[self::URL_KEY])
&& (empty($rowData[self::COL_VISIBILITY])
|| $rowData[self::COL_VISIBILITY]
!== (string)Visibility::getOptionArray()[Visibility::VISIBILITY_NOT_VISIBLE]);
@@ -762,10 +687,37 @@ private function isNeedToValidateUrlKey($rowData)
private function prepareNewSkuData($sku)
{
$data = [];
- foreach ($this->_oldSku[\strtolower($sku)] as $key => $value) {
- $data[$key] = $value;
+ foreach ($this->getExistingSku($sku) as $key => $value) {
+ $data[$key] = $value;
}
- $data['attr_set_code'] = $this->_attrSetIdToName[$this->_oldSku[\strtolower($sku)]['attr_set_id']];
+
+ $data['attr_set_code'] = $this->_attrSetIdToName[$this->getExistingSku($sku)['attr_set_id']];
+
return $data;
}
+
+ /**
+ * Check if product exists for specified SKU
+ *
+ * @param string $sku
+ * @return bool
+ */
+ private function isSkuExist($sku): bool
+ {
+ if ($sku !== null) {
+ return $this->skuStorage->has($sku);
+ }
+ return false;
+ }
+
+ /**
+ * Get existing product data for specified SKU
+ *
+ * @param string $sku
+ * @return array
+ */
+ private function getExistingSku($sku)
+ {
+ return $this->skuStorage->get((string)$sku);
+ }
}
diff --git a/src/Streamer/HttpXml.php b/src/Streamer/HttpXml.php
index c3c7f26..42ce2f1 100644
--- a/src/Streamer/HttpXml.php
+++ b/src/Streamer/HttpXml.php
@@ -128,33 +128,7 @@ public function getIterator()
"Streamer\HttpXml: Getting data from URL {$this->requestUrl}"
);
- $stack = \GuzzleHttp\HandlerStack::create();
- $stack->push(new \Kevinrob\GuzzleCache\CacheMiddleware(
- new \Kevinrob\GuzzleCache\Strategy\GreedyCacheStrategy(
- new \Kevinrob\GuzzleCache\Storage\Psr6CacheStorage(
- $this->cacheItemPool
- ),
- $this->ttl
- )
- ), 'cache');
-
- $httpClient = new \GuzzleHttp\Client(['handler' => $stack]);
-
- $result = $httpClient->request(
- $this->requestMethod,
- $this->requestUrl,
- $this->requestOptions + ['stream' => true, 'auth' => $this->auth]
- );
- $stream = new \Prewk\XmlStringStreamer\Stream\Guzzle('');
-
- if ($result->getHeader('X-Kevinrob-Cache') && $result->getHeader('X-Kevinrob-Cache')[0] == 'HIT') {
- $this->consoleOutput->write(" [Cache {$result->getHeader('X-Kevinrob-Cache')[0]}]");
- } else {
- $this->consoleOutput->write(" [Cache {$result->getHeader('X-Kevinrob-Cache')[0]}]");
- }
- $this->consoleOutput->write("\n");
-
- $stream->setGuzzleStream($result->getBody());
+ $stream = new \Prewk\XmlStringStreamer\Stream\Guzzle($this->requestUrl);
$class = isset($this->xmlOptions['uniqueNode']) ? UniqueNode::class : StringWalker::class;
$parser = new $class($this->xmlOptions + [