Page MenuHomeIn-Portal Phabricator
Files F800523

in-portal
No OneTemporary
Actions

File Metadata

Created
Sat, Feb 22, 12:08 AM

in-portal
View Options

Index: branches/5.2.x/core/units/helpers/csv_helper.php
===================================================================
--- branches/5.2.x/core/units/helpers/csv_helper.php (revision 16566)
+++ branches/5.2.x/core/units/helpers/csv_helper.php (revision 16567)
@@ -1,379 +1,415 @@
<?php
/**
* @version $Id$
* @package In-Portal
* @copyright Copyright (C) 1997 - 2009 Intechnic. All rights reserved.
* @license GNU/GPL
* In-Portal is Open Source software.
* This means that this software may have been modified pursuant
* the GNU General Public License, and as distributed it includes
* or is derivative of works licensed under the GNU General Public License
* or other free or open source software licenses.
* See http://www.in-portal.org/license for copyright notices and details.
*/
defined('FULL_PATH') or die('restricted access!');
kUtil::safeDefine('EXPORT_STEP', 100); // export by 100 items
kUtil::safeDefine('IMPORT_STEP', 10);
class kCSVHelper extends kHelper {
var $PrefixSpecial;
var $grid;
var $delimiter_mapping = Array(0 => "\t", 1 => ',', 2 => ';', 3 => ' ', 4 => ':');
var $enclosure_mapping = Array(0 => '"', 1 => "'");
var $separator_mapping = Array(0 => "\n", 1 => "\r\n");
function ExportStep()
{
$export_data = $this->Application->RecallVar('export_data');
$export_rand = $this->Application->RecallVar('export_rand');
$get_rand = $this->Application->GetVar('export_rand');
/** @var FileHelper $file_helper */
$file_helper = $this->Application->recallObject('FileHelper');
if ( $export_data && $export_rand == $get_rand ) {
$export_data = unserialize($export_data);
$first_step = false;
}
else {
// first step
$export_data = Array ();
$export_data['prefix'] = $this->PrefixSpecial;
$export_data['grid'] = $this->grid;
$export_data['file_name'] = EXPORT_PATH . '/' . $file_helper->ensureUniqueFilename(EXPORT_PATH, 'export_' . $export_data['prefix'] . '.csv');
$export_data['step'] = EXPORT_STEP;
$export_data['delimiter'] = $this->delimiter_mapping[(int)$this->Application->ConfigValue('CSVExportDelimiter')];
$export_data['enclosure'] = $this->enclosure_mapping[(int)$this->Application->ConfigValue('CSVExportEnclosure')];
$export_data['record_separator'] = $this->separator_mapping[(int)$this->Application->ConfigValue('CSVExportSeparator')];
$export_data['page'] = 1;
$export_data['source_encoding'] = strtoupper(CHARSET);
$export_data['encoding'] = $this->Application->ConfigValue('CSVExportEncoding') ? false : 'UTF-16LE';
$this->Application->StoreVar('export_rand', $get_rand);
$first_step = true;
}
$file = fopen($export_data['file_name'], $first_step ? 'w' : 'a');
$prefix_elems = preg_split('/\.|_/', $export_data['prefix'], 2);
- $grids = $this->Application->getUnitOption($prefix_elems[0], 'Grids');
- $grid_config = $grids[$export_data['grid']]['Fields'];
+ $grid_config = $this->getGridColumns($export_data);
$list_params = Array ('per_page' => $export_data['step'], 'grid' => $export_data['grid']);
/** @var kDBList $list */
$list = $this->Application->recallObject(rtrim(implode('.', $prefix_elems), '.'), $prefix_elems[0] . '_List', $list_params);
$list->SetPage($export_data['page']);
$list->Query();
$list->GoFirst();
$picker_helper = new kColumnPickerHelper(rtrim(implode('.', $prefix_elems), '.'), $export_data['grid']);
$grid_config = $picker_helper->apply($grid_config);
if ( $first_step ) {
// if UTF-16, write Unicode marker
if ( $export_data['encoding'] == 'UTF-16LE' ) {
fwrite($file, chr(0xFF) . chr(0xFE));
}
// inserting header line
$headers = Array ();
foreach ($grid_config as $field_name => $field_data) {
$use_phrases = array_key_exists('use_phrases', $field_data) ? $field_data['use_phrases'] : true;
$field_title = isset($field_data['title']) ? $field_data['title'] : 'column:la_fld_' . $field_name;
$header = $use_phrases ? $this->Application->Phrase($field_title) : $field_title;
array_push($headers, $header);
}
$csv_line = kUtil::getcsvline($headers, $export_data['delimiter'], $export_data['enclosure'], $export_data['record_separator']);
if ( $export_data['encoding'] ) {
$csv_line = mb_convert_encoding($csv_line, $export_data['encoding'], $export_data['source_encoding']);
}
fwrite($file, $csv_line);
}
while (!$list->EOL()) {
$data = Array ();
foreach ($grid_config as $field_name => $field_data) {
if ( isset($field_data['export_field']) ) {
$field_name = $field_data['export_field'];
}
$value = $list->GetField($field_name, isset($field_data['format']) ? $field_data['format'] : null);
$value = str_replace("\r\n", "\n", $value);
$value = str_replace("\r", "\n", $value);
array_push($data, $value);
}
if ( $export_data['encoding'] == 'UTF-16LE' ) {
fwrite($file, chr(0xFF) . chr(0xFE));
}
$csv_line = kUtil::getcsvline($data, $export_data['delimiter'], $export_data['enclosure'], $export_data['record_separator']);
if ( $export_data['encoding'] ) {
$csv_line = mb_convert_encoding($csv_line, $export_data['encoding'], $export_data['source_encoding']);
}
fwrite($file, $csv_line);
$list->GoNext();
}
$records_processed = $export_data['page'] * $export_data['step'];
$percent_complete = min($records_processed / $list->GetRecordsCount() * 100, 100);
fclose($file);
if ( $records_processed >= $list->GetRecordsCount() ) {
$this->Application->StoreVar('export_data', serialize($export_data));
$this->Application->Redirect($this->Application->GetVar('finish_template'));
}
echo $percent_complete;
$export_data['page']++;
$this->Application->StoreVar('export_data', serialize($export_data));
}
function ExportData($name)
{
$export_data = unserialize($this->Application->RecallVar('export_data'));
return isset($export_data[$name]) ? $export_data[$name] : false;
}
/**
* Returns prefix from request or from stored import/export data
*
* @param bool $is_import
* @return string
*/
public function getPrefix($is_import = false)
{
$prefix = $this->Application->GetVar('PrefixSpecial');
if ( !$prefix ) {
return $is_import ? $this->ImportData('prefix') : $this->ExportData('prefix');
}
return $prefix;
}
function GetCSV()
{
kUtil::safeDefine('DBG_SKIP_REPORTING', 1);
$export_data = unserialize($this->Application->RecallVar('export_data'));
$filename = preg_replace('/(.*)\.csv$/', '\1', basename($export_data['file_name'])) . '.csv';
$this->Application->setContentType('text/csv');
header('Content-Disposition: attachment; filename="' . $filename . '"');
readfile($export_data['file_name']);
die();
}
function ImportStart($filename)
{
if(!file_exists($filename) || !is_file($filename)) return 'cant_open_file';
$import_data = Array();
$import_data['source_encoding'] = strtoupper(CHARSET);
$import_data['encoding'] = $this->Application->ConfigValue('CSVExportEncoding') ? false : 'UTF-16LE';
$import_data['errors'] = '';
// convert file in case of UTF-16LE
if($import_data['source_encoding'] != $import_data['encoding']) {
copy($filename, $filename.'.orginal');
$file_content = file_get_contents($filename);
$file = fopen($filename, 'w');
fwrite($file, mb_convert_encoding(str_replace(chr(0xFF).chr(0xFE), '', $file_content), $import_data['source_encoding'], $import_data['encoding']));
fclose($file);
}
$import_data['prefix'] = $this->PrefixSpecial;
$import_data['grid'] = $this->grid;
$import_data['file'] = $filename;
$import_data['total_lines'] = count(file($filename));
if(!$import_data['total_lines']) $import_data['total_lines'] = 1;
unset($file_content);
$import_data['lines_processed'] = 0;
$import_data['delimiter'] = $this->delimiter_mapping[(int)$this->Application->ConfigValue('CSVExportDelimiter')];
$import_data['enclosure'] = $this->enclosure_mapping[(int)$this->Application->ConfigValue('CSVExportEnclosure')];
$import_data['step'] = IMPORT_STEP;
$import_data['not_imported_lines'] = '';
$import_data['added'] = 0;
$import_data['updated'] = 0;
$file = fopen($filename, 'r');
// getting first line for headers
$headers = fgetcsv($file, 8192, $import_data['delimiter'], $import_data['enclosure']);
fclose($file);
$prefix_elems = preg_split('/\.|_/', $import_data['prefix'], 2);
- $grids = $this->Application->getUnitOption($prefix_elems[0], 'Grids');
- $grid_config = $grids[ $import_data['grid'] ]['Fields'];
+ $grid_config = $this->getGridColumns($import_data);
$field_list = Array();
foreach($grid_config as $field_name => $field_data) {
if(isset($field_data['export_field'])) {
$field_name = $field_data['export_field'];
}
$field_title = isset($field_data['title']) ? $field_data['title'] : 'column:la_fld_' . $field_name;
$field_label = $this->Application->Phrase($field_title);
$field_pos = array_search($field_label, $headers);
if($field_pos !== false) {
$field_list[$field_pos] = $field_name;
}
}
if(!count($field_list)) return 'no_matching_columns';
$import_data['field_list'] = $field_list;
// getting key list
$field_positions = Array();
$config_key_list = $this->Application->getUnitOption($prefix_elems[0], 'ImportKeys');
if(!$config_key_list) $config_key_list = Array();
array_unshift($config_key_list, Array($this->Application->getUnitOption($prefix_elems[0], 'IDField')));
$key_list = Array();
foreach($config_key_list as $arr_key => $import_key) {
$key_list[$arr_key] = is_array($import_key) ? $import_key : Array($import_key);
foreach($key_list[$arr_key] as $key_field) {
$field_positions[$key_field] = array_search($key_field, $import_data['field_list']);
if($field_positions[$key_field] === false) {
// no such key field combination in imported file
unset($key_list[$arr_key]);
break;
}
}
}
$import_data['key_list'] = $key_list;
$import_data['field_positions'] = $field_positions;
$this->Application->StoreVar('import_data', serialize($import_data));
return true;
}
function ImportStep()
{
$import_data = unserialize($this->Application->RecallVar('import_data'));
$prefix_elems = preg_split('/\.|_/', $import_data['prefix'], 2);
/** @var kDBItem $object */
$object = $this->Application->recallObject($prefix_elems[0].'.-csvimport', $prefix_elems[0], Array('skip_autoload' => true, 'populate_ml_fields' => true));
$file = fopen($import_data['file'], 'r');
$eof = false;
// skipping lines that has been already imported
for($i = 0; $i < $import_data['lines_processed'] + 1; $i++) {
if(feof($file)) break;
fgets($file, 8192);
}
$import_event = new kEvent($prefix_elems[0].'.-csvimport:OnBeforeCSVLineImport');
for($i = 0; $i < $import_data['step']; $i++) {
if(feof($file)) break;
$data = fgetcsv($file, 8192, $import_data['delimiter'], $import_data['enclosure']);
if(!$data) continue;
$object->Clear();
$action = 'Create';
// 1. trying to load object by keys
foreach($import_data['key_list'] as $key) {
$fail = false;
$key_array = Array();
foreach($key as $key_field) {
if(!isset($data[ $import_data['field_positions'][$key_field] ])) {
$fail = true;
break;
}
$key_array[$key_field] = $data[ $import_data['field_positions'][$key_field] ];
}
if($fail) continue;
if($object->Load($key_array)) {
$action = 'Update';
break;
}
}
- // 2. set object fields
- foreach($import_data['field_list'] as $position => $field_name) {
- if(isset($data[$position])) {
+ // 2. set object fields.
+ $grid_config = $this->getGridColumns($import_data);
+
+ foreach ( $import_data['field_list'] as $position => $field_name ) {
+ if ( isset($data[$position]) ) {
+ $formatter_class = $object->GetFieldOption($field_name, 'formatter');
+
+ if ( $formatter_class !== false ) {
+ $formatter = $this->Application->recallObject($formatter_class);
+
+ if ( $formatter instanceof kDateFormatter ) {
+ if ( isset($grid_config[$field_name]['format']) ) {
+ $format = $grid_config[$field_name]['format'];
+ }
+ else {
+ $format = $object->GetFieldOption($field_name, 'format');
+ }
+
+ // Use export format during import.
+ $object->SetFieldOption($field_name, 'input_format', $format);
+
+ // Read date/time from single column.
+ $object->SetDBField($field_name . '_combined', 1);
+ }
+ }
+
$object->SetField($field_name, $data[$position]);
}
}
// 3. validate item and run event
$status = $object->Validate();
$import_event->status = $status ? kEvent::erSUCCESS : kEvent::erFAIL;
$this->Application->HandleEvent($import_event);
if($import_event->status == kEvent::erSUCCESS && $object->$action()) {
$import_data[ ($action == 'Create') ? 'added' : 'updated' ]++;
}
else {
$msg = '';
$errors = $object->GetFieldErrors();
foreach ($errors as $field => $info) {
if (!$info['pseudo']) continue;
$msg .= "$field: {$info['pseudo']} ";
}
$import_data['errors'] .= ($i + $import_data['lines_processed'] + 1).": $msg\n";
$import_data['not_imported_lines'] .= ','.($i + $import_data['lines_processed'] + 1);
}
}
$import_data['lines_processed'] += $import_data['step'];
$import_data['not_imported_lines'] = ltrim($import_data['not_imported_lines'], ',');
$this->Application->StoreVar('import_data', serialize($import_data));
$feof = feof($file);
fclose($file);
if($feof) {
$this->Application->Redirect($this->Application->GetVar('finish_template'));
}
else {
$percent_complete = floor($import_data['lines_processed'] / $import_data['total_lines'] * 100);
if($percent_complete > 99) $percent_complete = 99;
echo $percent_complete;
}
}
+ /**
+ * Returns grid columns.
+ *
+ * @param array $import_data Import data.
+ *
+ * @return array
+ */
+ protected function getGridColumns(array $import_data)
+ {
+ $prefix_elements = preg_split('/\.|_/', $import_data['prefix'], 2);
+ $grids = $this->Application->getUnitOption($prefix_elements[0], 'Grids');
+
+ return $grids[$import_data['grid']]['Fields'];
+ }
+
function ImportData($name)
{
$import_data = unserialize($this->Application->RecallVar('import_data'));
return isset($import_data[$name]) ? $import_data[$name] : false;
}
function GetNotImportedLines()
{
$import_data = unserialize($this->Application->RecallVar('import_data'));
if(!$import_data['not_imported_lines']) return false;
$line_numbers = explode(',', $import_data['not_imported_lines']);
$line_numbers[] = 0; // include header row in output
$file = fopen($import_data['file'], 'r');
$eof = false;
$result = '';
for($i = 0; $i <= max($line_numbers); $i++) {
if(feof($file)) break;
$line = fgets($file, 8192);
if(in_array($i, $line_numbers)) {
$result .= $i.':'.$line;
}
}
return $result."\n\n".$import_data['errors'];
}
}

Event Timeline

In-Portal CMS · In-Portal Community Pages · In-Portal Issue Tracker · In-Portal Repository Browser · Quality Assurance · In-Portal CI Server