_select) {
$this->_select = new Zend_Db_Select($this->_getDbAdapter());
}
return $this->_select->reset();
}
protected function _explodeFile($filePath, $filename)
{
if (filesize($filePath) > self::MAX_FILE_SIZE) {
Mage::helper('orange35_productImport/timer')->addTimer('Explode Files');
$this->log("Explode file");
/** @var Orange35_ProductImport_Helper_Config $conf */
$conf = Mage::app()->getHelper('orange35_productImport/config');
require_once 'Product/Import/Adapter/Xml.php';
$adapter = new Product_Import_Adapter_Xml();
$adapter->setFilePath($filePath);
$currentPart = 1;
$currentPartContent = '';
$hasContent = false;
while ($products = $adapter->getProductsData()) {
foreach ($products as $product) {
$hasContent = true;
$currentPartContent .= '';
foreach ($product as $key => $value) {
$value = htmlspecialchars($value);
$currentPartContent .= "<$key>$value";
}
$currentPartContent .= '';
}
if (strlen($currentPartContent) > 0.6 * self::MAX_FILE_SIZE) {
$currentPartContent .= '';
$newFile = $conf->explodedXmlDirectoryPath . $filename
. '_' . ($currentPart++) . '.xml';
file_put_contents($newFile, $currentPartContent);
$currentPartContent = '';
$hasContent = false;
}
}
if ($hasContent) {
$currentPartContent .= '';
$newFile = $conf->explodedXmlDirectoryPath . $filename
. '_' . ($currentPart) . '.xml';
file_put_contents($newFile, $currentPartContent);
}
return true;
}
return false;
}
public function startImport()
{
Mage::helper('orange35_productImport/timer')->addTimer('Delete Logs');
$this->_deleteOldLogsFromDb();
$this->_deleteOldXmlFiles();
$this->_deleteOldLogFiles();
/** @var Orange35_ProductImport_Helper_Config $conf */
$conf = Mage::app()->getHelper('orange35_productImport/config');
header('Content-Type: text/html; charset=utf-8');
$oldProcessedFiles = $this->getOldProcessedFiles($conf->importLogTableName);
$filename = '';
$fullFilePath = '';
//Find file to import (in exploded first)
foreach (array($conf->explodedXmlDirectoryPath, $conf->xmlDirectoryPath) as $rootDirectory) {
$files = $this->getFilesForImport(glob($rootDirectory . '*.xml'), $oldProcessedFiles);
if (!empty($files)) {
$modifiedTimes = array_map('filemtime', $files);
$filename = basename($files[array_search(min($modifiedTimes), $modifiedTimes)]);
$fullFilePath = $rootDirectory . $filename;
break;
}
}
if (!empty($filename)) {
file_put_contents(self::DAY_FLAG_FILE, 1);
$this->_reset();
$import = $this->_getImportObj();
$importAdapter = new Product_Import_Adapter_Xml();
$import->setImportAdapter($importAdapter);
$this->log(array('filename' => $filename));
if ($this->_explodeFile($fullFilePath, $filename)) {
$log = new Product_Import_Log();
$log->setDbAdapter($this->_getDbAdapter());
$now = time();
$log->saveLog($now, $now, $fullFilePath);
Mage::helper('orange35_productImport/timer')->addTimer('Restart Import by exploded');
$this->startImport();
return $this;
}
Mage::helper('orange35_productImport/timer')->addTimer('Insert/Update new simple products');
$importAdapter->setFilePath($fullFilePath);
$import->processDataImport($new, $update);
$this->importCompleteLogging($import, $filename);
Mage::helper('orange35_productImport/timer')->addTimer('Create/Update configured products');
$nonConfigurableSkuList = $import->getNonConfigurableSkuList();
$processedProducts = array_unique(array_merge($new, $update));
$processedProducts = array_diff($processedProducts, $nonConfigurableSkuList);
/** @var Orange35_ProductImport_Helper_Configurable $helper */
$helper = Mage::app()->getHelper('orange35_productImport/configurable');
$helper->processConfigurableProducts($processedProducts, $this->_getConfigurableImportObj());
$this->_printConfigurableProductsCreatingResult($this->_getConfigurableImportObj());
} else {
$this->log(array('message' => 'Not found non-processed xml files'));
$this->_reindexData();
}
return $this;
}
protected function _reindexData()
{
/** @var Orange35_ProductImport_Helper_Config $conf */
$conf = Mage::app()->getHelper('orange35_productImport/config');
//Select not indexed files
$lastIndexTimeSelect = new Zend_Db_Select($this->_getDbAdapter());
$lastIndexTimeSelect->from($conf->indexProcessTableName, array('MIN(`ended_at`)'));
$select = new Zend_Db_Select($this->_getDbAdapter());
$select->from($conf->importLogTableName, array('*'))
->where('end_import_time > (' . $lastIndexTimeSelect . ')');
$rows = $select->query()->fetchAll();
if (!empty($rows)) {
//Start reindex
Mage::helper('orange35_productImport/index')->magentoReindex();
}
return $this;
}
protected function _reset()
{
$this->_importObj = null;
$this->_importConfigurableObj = null;
}
/**
* @param $import Product_Import
* @return $this
*/
protected function _printConfigurableProductsCreatingResult($import)
{
$log = $import->getLogObj();
$resultMessage = "\nCreating configurable products! \n" . "Proceed products: " . $log->getProceedProductsCount()
. "( new: " . $log->getNewProductsCount() . " . update: " . $log->getUpdateProductsCount()
. ")\n" . "Error products: " . $log->getErrorProductsCount() . "\n";
$this->log(array('message' => $resultMessage));
if ($log->getErrorProductsCount()) {
$errorMessage = 'Errors:' . print_r($log->getErrorsArray(), true) . '
';
$this->log(array('message' => $errorMessage));
}
return $this;
}
protected function _getDbAdapter()
{
if (null == $this->_dbAdapter) {
$config = Mage::getConfig();
$dbConfig = $config->getResourceConnectionConfig('default_setup');
$dbAdapter = new Zend_Db_Adapter_Pdo_Mysql(array(
'dbname' => $dbConfig->dbname,
'host' => $dbConfig->host,
'username' => $dbConfig->username,
'password' => $dbConfig->password,
));
$dbAdapter->query('SET NAMES UTF8');
$this->_dbAdapter = $dbAdapter;
}
return $this->_dbAdapter;
}
protected function _getImportObj()
{
/** @var Orange35_ProductImport_Helper_Config $conf */
$conf = Mage::app()->getHelper('orange35_productImport/config');
if (null == $this->_importObj) {
$import = new Product_Import();
//Db Adapter
$import->setDbAdapter($this->_getDbAdapter());
//Attribute sets map
$import->getAttributesManager()->setAttributeSetsMap($conf->attributeSetsMap);
//Stock defaults
$import->setStockDefaults(array(
'use_config_manage_stock' => 0,
'manage_stock' => 1,
'is_in_stock' => 1,
));
//Opt customers price convert rule
$import->setGroupPriceConvertRule('price_opt', Orange35_ProductImport_Helper_Config::OPT_CUSTOMER_GROUP_ID);
$import->setStoresMap($conf->storesMap);
//Attributes defaults
$import->getAttributesManager()->setAttributesDefaults(array(
'is_recurring' => 'No',
'website_ids' => '1',
'status' => 'Disabled',
'weight' => 0,
'options_container' => 'Product Info Column',
'short_description' => ' ',
'description' => ' ',
'base_color' => 'Не вказано',
'base_size' => 'Не вказано',
'base_cloth' => 'Не вказано',
))
->setRequireAttributesList(array(
'sku',
'name',
))
->setAttributeFilter('base_size', array($this, 'filterSize'))
->setAttributeFilter('base_cloth', array($this, 'conCatSizeWithColor'))
->setAttributeFilter('furniture_cloth', array($this, 'conCatFurnitureSizeWithColor'))
->setAttributeFilter('sku', array($this, 'prepareManageStock'))
->setAttributeFilter('is_in_stock', array($this, 'filterIsInStock'));
$import->setAppendParentCategoryIds(false);
//Images
$import->setPrepareImages(true);
$import->setImagesPath($conf->imagesPath);
//import db log
$log = new Product_Import_Log();
$log->setDbAdapter($this->_getDbAdapter());
$import->setLogObj($log);
$this->_importObj = $import;
}
return $this->_importObj;
}
public function filterIsInStock()
{
return 1;
}
protected function _getManageStockAttributeSetsIds()
{
if (null == $this->_manageStockAttributeIds) {
/** @var Orange35_ProductImport_Helper_Config $config */
$config = Mage::helper('orange35_productImport/config');
$this->_manageStockAttributeIds = $this->_getSubCategoriesRecursive(
$config->manageStockParentCategories
);
}
return $this->_manageStockAttributeIds;
}
protected function _getSubCategoriesRecursive($categoriesIds)
{
$newCategories = $categoriesIds;
foreach ($categoriesIds as $categoryId) {
$childCategories = $this->_getChildrenCategories($categoryId);
$childCategories = array_diff($childCategories, $newCategories);
if (count($childCategories)) {
$newCategories = array_merge(
$newCategories, $this->_getSubCategoriesRecursive($childCategories)
);
}
}
return array_unique($newCategories);
}
protected function _getChildrenCategories($categoryId)
{
$categories = array();
$subCategories = Mage::getModel('catalog/category')->getCategories($categoryId, 0, false, true);
foreach ($subCategories as $category) {
$categories[] = $category->getId();
}
return $categories;
}
public function prepareManageStock($attribute, $storeId, $sku, &$context)
{
if ($attribute == 'sku') {
if (isset($context['category_ids'])) {
$categoryIds = array_map('intval', explode(',', str_replace(' ', '', $context['category_ids'])));
$context['manage_stock'] = (int) !count(array_intersect(
$categoryIds, $this->_getManageStockAttributeSetsIds())
);
$context['use_config_manage_stock'] = $context['manage_stock'];
}
if (isset($context['attribute_set_id'])) {
$id = (int) $context['attribute_set_id'];
if (in_array($id, array(
Orange35_ProductImport_Helper_Config::ATTRIBUTE_SET_BEADS_SET,
Orange35_ProductImport_Helper_Config::ATTRIBUTE_SET_THREADS_SET
))) {
$context['manage_stock'] = 0;
$context['use_config_manage_stock'] = 0;
}
}
}
return $sku;
}
public function conCatSizeWithColor($attribute, $storeId, $cloth, $context)
{
if (isset($context['base_color'])
&& 'base_cloth' == $attribute
) {
return $context['base_color'] . ' ' . $cloth;
}
return $cloth;
}
public function conCatFurnitureSizeWithColor($attribute, $storeId, $cloth, $context)
{
if (isset($context['furniture_color'])
&& 'furniture_cloth' == $attribute
) {
return $context['furniture_color'] . ' ' . $cloth;
}
return $cloth;
}
public function filterSize($attribute, $storeId, $size, $context)
{
if ('base_size' == $attribute) {
return $size != '0' || !$storeId ? $size : 'Не вказано';
} else {
return $size;
}
}
/**
* @return Product_Import
*/
protected function _getConfigurableImportObj()
{
if (null == $this->_importConfigurableObj) {
$this->_importConfigurableObj = clone $this->_getImportObj();
$adapter = new Product_Import_Adapter_Array();
$this->_importConfigurableObj->setImportAdapter($adapter);
$log = new Product_Import_Log();
$log->enableDbLog(false);
$this->_importConfigurableObj->setLogObj($log);
$this->_importConfigurableObj->setPrepareImages(false);
}
return $this->_importConfigurableObj;
}
protected function _deleteOldLogsFromDb()
{
$importTable = new Zend_Db_Table(array(
Zend_Db_Table::ADAPTER => $this->_getDbAdapter(),
Zend_Db_Table::NAME => Mage::getModel('core/resource')->getConnection('core_read')->getTableName(
'orange35_product_import'
),
));
$errorTable = new Zend_Db_Table(array(
Zend_Db_Table::ADAPTER => $this->_getDbAdapter(),
Zend_Db_Table::NAME => Mage::getModel('core/resource')->getConnection('core_read')->getTableName(
'orange35_product_import_errors'
),
));
$select= new Zend_Db_Select($this->_getDbAdapter());
$select->from('orange35_product_import', array('import_id', 'processed_filename'))
->where('TO_DAYS(NOW()) - TO_DAYS(begin_import_time) > 30');
$rows = $this->_getDbAdapter()->fetchAll($select);
$ids = array();
foreach ($rows as $row) {
$ids[] = $row['import_id'];
if (file_exists($row['processed_filename'])) {
unlink($row['processed_filename']);
}
}
if (!empty($ids)) {
$errorTable->delete('import_id IN (\'' . implode('\',\'', $ids) . '\')');
$importTable->delete('import_id IN (\'' . implode('\',\'', $ids) . '\')');
}
return $this;
}
protected function _deleteOldXmlFiles()
{
/** @var Orange35_ProductImport_Helper_Config $config */
$config = Mage::helper('orange35_productImport/config');
foreach (array($config->xmlDirectoryPath, $config->explodedXmlDirectoryPath) as $directory) {
$dir = opendir($directory);
$lastMonth = strtotime('-1month');
while ($file = readdir($dir)) {
if (preg_match('~.*[.]xml~', $file)
&& filemtime($directory . $file) < $lastMonth
) {
unlink($directory . $file);
}
}
}
return $this;
}
public static function getLogFile()
{
if (null == self::$_logFile) {
self::$_logFile = self::LOG_FILE . '_' . strtolower(date('M'));
}
return self::$_logFile;
}
protected function _deleteOldLogFiles()
{
$oldLog = self::LOG_FILE . '_' . strtolower(date('M', strtotime('-2month')));
if (file_exists($oldLog)) {
unlink($oldLog);
}
return $this;
}
/**
* @param $params String|Array ($params['message', 'filename'])
* @param bool $showTime
*/
public function log($params, $showTime = false)
{
if (is_array($params)) {
$message = array_key_exists('message', $params) ? $params['message'] : '';
if (array_key_exists('filename', $params) && !empty($params['filename'])){
$filename = $params['filename'];
$message .= "\nCurrent File: " . $filename;
}
} else {
$message = $params;
}
$existingContent = file_exists(Orange35_ProductImport_Helper_Data::getLogFile()) ? file_get_contents(
Orange35_ProductImport_Helper_Data::getLogFile()
) : '';
$timezone = date_default_timezone_get();
date_default_timezone_set('Europe/Kiev');
$time = $showTime ?
"\n(" . date('m/d/Y H:i:s') . ") " . round(memory_get_usage(true) / 1024 / 1024) . ' / ' . round(
memory_get_peak_usage(true) / 1024 / 1024) . ' Mb' . "\n" : "";
date_default_timezone_set($timezone);
file_put_contents(
Orange35_ProductImport_Helper_Data::getLogFile(),
$existingContent .
"\n" . $message .
$time
);
}
/**
* @param $newFiles array
* @param $oldFiles array
* @return array
*/
protected function getFilesForImport($newFiles, $oldFiles)
{
$oldFilesUnique = array();
$differentByDate = array();
foreach ($oldFiles as $value) {
$oldFilesUnique[$value['processed_filename']] = strtotime($value['begin_import_time']);
}
$oldFileNames = array_keys($oldFilesUnique);
$differentByName = array_diff($newFiles, $oldFileNames);
$modifiedTimes = array();
if(!empty($newFiles) && ($newFiles = array_combine($newFiles, $newFiles))) {
$modifiedTimes = array_map('filemtime', $newFiles);
}
foreach ($modifiedTimes as $fileName => $fileTime) {
if (array_key_exists($fileName, $oldFilesUnique) && !empty($oldFilesUnique[$fileName])
&& $oldFilesUnique[$fileName] <= $fileTime
) {
$differentByDate[] = $fileName;
}
}
return array_merge($differentByName, $differentByDate);
}
/**
* @param $import Product_Import
* @param $filename
*/
protected function importCompleteLogging($import, $filename)
{
$log = $import->getLogObj();
$completeMessage = "Import complete!\n" . "Processed filename: " . $filename . "\n" . "Proceed products: "
. $log->getProceedProductsCount() . "( new: " . $log->getNewProductsCount() . ", update: "
. $log->getUpdateProductsCount() . ")\n" . "Error products: " . $log->getErrorProductsCount(
) . "\n";
$this->log(array('message' => $completeMessage), true);
if ($log->getErrorProductsCount()) {
$errorMessage = 'Errors:' . print_r($log->getErrorsArray(), true) . '
';
$this->log(array('message' => $errorMessage));
}
}
/**
* @param $importLogTableName
* @return array
*/
protected function getOldProcessedFiles($importLogTableName)
{
$select = new Zend_Db_Select($this->_getDbAdapter());
$select->from($importLogTableName, array('processed_filename', 'begin_import_time'));
$oldProcessedFiles = $select->query()->fetchAll();
return $oldProcessedFiles;
}
}