Index: core/install.php
===================================================================
--- core/install.php
+++ core/install.php
@@ -332,7 +332,10 @@
case 'db_reconfig':
$fields = Array (
'DBType', 'DBHost', 'DBName', 'DBUser',
- 'DBUserPassword', 'DBCollation', 'TablePrefix'
+ 'DBUserPassword', 'DBCollation', 'TablePrefix',
+ 'DBErrorBackoffMaxRetryAttempts',
+ 'DBErrorBackoffLogicBaseTime',
+ 'DBEnableLockRetryDebugging',
);
// set fields
@@ -531,9 +534,14 @@
case 'db_reconfig':
// 1. check if required fields are filled
$section_name = 'Database';
- $required_fields = Array ('DBType', 'DBHost', 'DBName', 'DBUser', 'DBCollation');
+ $required_fields = Array (
+ 'DBType', 'DBHost', 'DBName', 'DBUser', 'DBCollation',
+ 'DBErrorBackoffMaxRetryAttempts',
+ 'DBErrorBackoffLogicBaseTime',
+ 'DBEnableLockRetryDebugging',
+ );
foreach ($required_fields as $required_field) {
- if (!$this->toolkit->systemConfig->get($required_field, $section_name)) {
+ if ($this->toolkit->systemConfig->get($required_field, $section_name) === '') {
$status = false;
$this->errorMessage = 'Please fill all required fields';
break;
@@ -1745,15 +1753,25 @@
/**
* Installation error handler for sql errors
*
- * @param int $code
- * @param string $msg
- * @param string $sql
- * @return bool
- * @access private
+ * @param integer $code Error code.
+ * @param string $msg Error message.
+ * @param string $sql SQL query.
+ * @param boolean|null $throw_exception Throw an exception.
+ *
+ * @return boolean
+ * @throws RuntimeException When requested.
*/
- function DBErrorHandler($code, $msg, $sql)
+ function DBErrorHandler($code, $msg, $sql, $throw_exception = null)
{
- $this->errorMessage = 'Query: <br />'.htmlspecialchars($sql, ENT_QUOTES, 'UTF-8').'<br />execution result is error:<br />['.$code.'] '.$msg;
+ $error_msg = 'Query: <br />' . htmlspecialchars($sql, ENT_QUOTES, 'UTF-8') . '<br />';
+ $error_msg .= 'execution result is error:<br />[' . $code . '] ' . $msg;
+
+ if ( $throw_exception === true ) {
+ throw new RuntimeException($error_msg);
+ }
+
+ $this->errorMessage = $error_msg;
+
return true;
}
Index: core/install/step_templates/db_config.tpl
===================================================================
--- core/install/step_templates/db_config.tpl
+++ core/install/step_templates/db_config.tpl
@@ -66,6 +66,29 @@
</td>
</tr>
+<tr class="table-color2">
+ <td class="text"><b>Database Error Backoff Max Retry Attempts <span class="error">*</span>:</b></td>
+ <td align="left">
+ <input type="text" name="DBErrorBackoffMaxRetryAttempts" class="text" value="<?php echo $this->toolkit->systemConfig->get('DBErrorBackoffMaxRetryAttempts', 'Database'); ?>" />
+ </td>
+</tr>
+
+<tr class="table-color2">
+ <td class="text"><b>Database Error Backoff Logic Base Time (in milliseconds) <span class="error">*</span>:</b></td>
+ <td align="left">
+ <input type="text" name="DBErrorBackoffLogicBaseTime" class="text" value="<?php echo $this->toolkit->systemConfig->get('DBErrorBackoffLogicBaseTime', 'Database'); ?>" />
+ </td>
+</tr>
+
+<tr class="table-color2">
+ <td class="text"><b>Database Lock Retry Debugging Enabled <span class="error">*</span>:</b></td>
+ <td align="left">
+ <?php $lock_debugging_enabled = $this->toolkit->systemConfig->get('DBEnableLockRetryDebugging', 'Database'); ?>
+ <label><input type="radio" name="DBEnableLockRetryDebugging" class="text" value="0" <?php if ( !$lock_debugging_enabled ) { echo 'checked'; } ?>/> No</label>
+ <label><input type="radio" name="DBEnableLockRetryDebugging" class="text" value="1" <?php if ( $lock_debugging_enabled ) { echo 'checked'; } ?>/> Yes</label>
+ </td>
+</tr>
+
<?php if ($this->GetVar('preset') != 'already_installed') { ?>
<!--show this option ONLY when config.php is empty or cant connect to In-Portal installation using current DB settings -->
<tr class="table-color2">
@@ -74,4 +97,4 @@
<input type="radio" name="UseExistingSetup" id="UseExistingSetup_1" value="1"/> <label for="UseExistingSetup_1">Yes</label> <input type="radio" name="UseExistingSetup" id="UseExistingSetup_0" value="0" checked/> <label for="UseExistingSetup_0">No</label>
</td>
</tr>
-<?php } ?>
\ No newline at end of file
+<?php } ?>
Index: core/install/step_templates/db_reconfig.tpl
===================================================================
--- core/install/step_templates/db_reconfig.tpl
+++ core/install/step_templates/db_reconfig.tpl
@@ -64,4 +64,27 @@
<td align="left">
<input type="text" name="TablePrefix" class="text" maxlength="7" value="<?php echo $this->toolkit->systemConfig->get('TablePrefix', 'Database'); ?>" />
</td>
-</tr>
\ No newline at end of file
+</tr>
+
+<tr class="table-color2">
+ <td class="text"><b>Database Error Backoff Max Retry Attempts <span class="error">*</span>:</b></td>
+ <td align="left">
+ <input type="text" name="DBErrorBackoffMaxRetryAttempts" class="text" value="<?php echo $this->toolkit->systemConfig->get('DBErrorBackoffMaxRetryAttempts', 'Database'); ?>" />
+ </td>
+</tr>
+
+<tr class="table-color2">
+ <td class="text"><b>Database Error Backoff Logic Base Time (in milliseconds)<span class="error">*</span>:</b></td>
+ <td align="left">
+ <input type="text" name="DBErrorBackoffLogicBaseTime" class="text" value="<?php echo $this->toolkit->systemConfig->get('DBErrorBackoffLogicBaseTime', 'Database'); ?>" />
+ </td>
+</tr>
+
+<tr class="table-color2">
+ <td class="text"><b>Database Lock Retry Debugging Enabled <span class="error">*</span>:</b></td>
+ <td align="left">
+ <?php $lock_debugging_enabled = $this->toolkit->systemConfig->get('DBEnableLockRetryDebugging', 'Database'); ?>
+ <label><input type="radio" name="DBEnableLockRetryDebugging" class="text" value="0" <?php if ( !$lock_debugging_enabled ) { echo 'checked'; } ?>/> No</label>
+ <label><input type="radio" name="DBEnableLockRetryDebugging" class="text" value="1" <?php if ( $lock_debugging_enabled ) { echo 'checked'; } ?>/> Yes</label>
+ </td>
+</tr>
Index: core/kernel/application.php
===================================================================
--- core/kernel/application.php
+++ core/kernel/application.php
@@ -2458,17 +2458,17 @@
/**
* SQL Error Handler
*
- * @param int $code
- * @param string $msg
- * @param string $sql
- * @return bool
- * @access public
- * @throws Exception
+ * @param integer $code Error code.
+ * @param string $msg Error message.
+ * @param string $sql SQL query.
+ * @param boolean|null $throw_exception Throw an exception.
+ *
+ * @return boolean
* @deprecated
*/
- public function handleSQLError($code, $msg, $sql)
+ public function handleSQLError($code, $msg, $sql, $throw_exception = null)
{
- return $this->_logger->handleSQLError($code, $msg, $sql);
+ return $this->_logger->handleSQLError($code, $msg, $sql, $throw_exception);
}
/**
Index: core/kernel/db/db_connection.php
===================================================================
--- core/kernel/db/db_connection.php
+++ core/kernel/db/db_connection.php
@@ -84,6 +84,13 @@
protected $errorMessage = '';
/**
+ * Error retry count.
+ *
+ * @var integer
+ */
+ protected $errorRetryCount = 0;
+
+ /**
* Defines if database connection
* operations should generate debug
* information
@@ -102,6 +109,27 @@
protected $_captureStatistics = false;
/**
+ * Error Backoff Maximal Retry Attempts.
+ *
+ * @var integer
+ */
+ protected $errorBackoffMaxRetryAttempts;
+
+ /**
+ * Error backoff logic base time (in milliseconds).
+ *
+ * @var integer
+ */
+ protected $errorBackoffLogicBaseTime;
+
+ /**
+ * Enable lock retry debugging.
+ *
+ * @var integer
+ */
+ protected $enableLockRetryDebugging;
+
+ /**
* Last query to database
*
* @var string
@@ -278,11 +306,18 @@
$this->debugMode = $this->Application->isDebugMode();
}
+ $this->errorBackoffMaxRetryAttempts = $config['Database']['DBErrorBackoffMaxRetryAttempts'];
+ $this->errorBackoffLogicBaseTime = $config['Database']['DBErrorBackoffLogicBaseTime'];
+ $this->enableLockRetryDebugging = $config['Database']['DBEnableLockRetryDebugging'];
+
+ $retry = array_key_exists('DBRetry', $config['Database']) ? $config['Database']['DBRetry'] : false;
+
return $this->Connect(
$config['Database']['DBHost'],
$config['Database']['DBUser'],
$config['Database']['DBUserPassword'],
- $config['Database']['DBName']
+ $config['Database']['DBName'],
+ $retry
);
}
@@ -327,13 +362,12 @@
* @param string $sql
* @param string $key_field
* @param boolean|null $no_debug
+ * @param string $iterator_class
* @return bool
* @access protected
*/
- protected function showError($sql = '', $key_field = null, $no_debug = null)
+ protected function showError($sql = '', $key_field = null, $no_debug = null, $iterator_class = '')
{
- static $retry_count = 0;
-
if ( $no_debug === null ) {
$no_debug = $this->noDebuggingState;
}
@@ -361,54 +395,141 @@
if ( $this->hasError() ) {
$this->errorMessage = $this->connectionID->error;
- $ret = $this->callErrorHandler($sql);
+ $recoverable_errors = array(
+ 2006, // MySQL server has gone away.
+ 2013, // Lost connection to MySQL server during query.
+ 1205, // Lock wait timeout exceeded; try restarting transaction.
+ 1213, // Deadlock found when trying to get lock; try restarting transaction.
+ );
+
+ if ( in_array($this->errorCode, $recoverable_errors) ) {
+ try {
+ $ret = $this->callErrorHandler($sql, true);
+ }
+ catch ( RuntimeException $e ) {
+
+ }
+
+ // Handle case, when specified error handler isn't respecting "$throw_exception" argument.
+ if ( !isset($e) ) {
+ $e = new RuntimeException(kLogger::shortenMessage(sprintf(
+ '%s #%d - %s. SQL: %s',
+ kLogger::DB_ERROR_PREFIX,
+ $this->errorCode,
+ $this->errorMessage,
+ trim($sql)
+ )));
+ }
- if ( ($this->errorCode == 2006 || $this->errorCode == 2013) && ($retry_count < 3) ) {
- // #2006 - MySQL server has gone away
- // #2013 - Lost connection to MySQL server during query
- $retry_count++;
+ if ( $this->errorRetryCount < $this->errorBackoffMaxRetryAttempts ) {
+ $this->errorRetryCount++;
+ $wait_time = $this->getErrorBackoffWaitTime($this->errorRetryCount);
+ usleep($wait_time * 1000);
+
+ if ( ($this->errorCode == 1205 || $this->errorCode == 1213)
+ && $this->enableLockRetryDebugging
+ ) {
+ $log = $this->Application->log('');
+ $log->addException($e);
+
+ $cause_mapping = array(
+ 1205 => 'Lock detected.',
+ 1213 => 'Deadlock detected.',
+ );
+ $log->addUserData(PHP_EOL . $cause_mapping[$this->errorCode]);
+
+ $log->addUserData(sprintf(
+ 'Retry: %d of %d.',
+ $this->errorRetryCount,
+ $this->errorBackoffMaxRetryAttempts
+ ));
+ $log->addUserData('Wait time: ' . $wait_time . 'ms');
+ $log->write();
+ }
+
+ // Attempt to reconnect upon disconnect.
+ if ( ($this->errorCode == 2006 || $this->errorCode == 2013) && !$this->ReConnect() ) {
+ if ( !$ret ) {
+ exit;
+ }
+
+ return false;
+ }
+
+ if ( $iterator_class ) {
+ return $this->GetIterator($sql, $key_field, $no_debug, $iterator_class);
+ }
- if ( $this->ReConnect() ) {
return $this->Query($sql, $key_field, $no_debug);
}
+
+ // Show an error back to the user after all the possible retry attempts were exhausted.
+ throw $e;
}
+ $ret = $this->callErrorHandler($sql);
+
if (!$ret) {
exit;
}
}
else {
- $retry_count = 0;
+ $this->errorRetryCount = 0;
}
return false;
}
/**
+ * Returns the error backoff wait time in milliseconds (exponential strategy).
+ *
+ * @param integer $attempt Attempt.
+ *
+ * @return integer
+ */
+ protected function getErrorBackoffWaitTime($attempt)
+ {
+ if ( $attempt == 1 ) {
+ return $this->errorBackoffLogicBaseTime;
+ }
+
+ return (int)(pow(2, $attempt) * $this->errorBackoffLogicBaseTime);
+ }
+
+ /**
* Sends db error to a predefined error handler
*
- * @param $sql
- * @return bool
- * @access protected
+ * @param string $sql SQL query.
+ * @param boolean|null $throw_exception Throw an exception.
+ *
+ * @return boolean
*/
- protected function callErrorHandler($sql)
+ protected function callErrorHandler($sql, $throw_exception = null)
{
- return call_user_func($this->errorHandler, $this->errorCode, $this->errorMessage, $sql);
+ return call_user_func($this->errorHandler, $this->errorCode, $this->errorMessage, $sql, $throw_exception);
}
/**
* Default error handler for sql errors
*
- * @param int $code
- * @param string $msg
- * @param string $sql
- * @return bool
- * @access public
+ * @param integer $code Error code.
+ * @param string $msg Error message.
+ * @param string $sql SQL query.
+ * @param boolean|null $throw_exception Throw an exception.
+ *
+ * @return boolean
+ * @throws RuntimeException When requested.
*/
- public function handleError($code, $msg, $sql)
+ public function handleError($code, $msg, $sql, $throw_exception = null)
{
- echo '<strong>Processing SQL</strong>: ' . $sql . '<br/>';
- echo '<strong>Error (' . $code . '):</strong> ' . $msg . '<br/>';
+ $error_msg = '<strong>Processing SQL</strong>: ' . $sql . '<br/>';
+ $error_msg .= '<strong>Error (' . $code . '):</strong> ' . $msg . '<br/>';
+
+ if ( $throw_exception === true ) {
+ throw new RuntimeException(kLogger::shortenMessage($error_msg));
+ }
+
+ echo $error_msg;
return false;
}
@@ -632,7 +753,7 @@
// set 2nd checkpoint: end
}
- return $this->showError($sql, $key_field, $no_debug);
+ return $this->showError($sql, $key_field, $no_debug, $iterator_class);
}
/**
@@ -1159,7 +1280,7 @@
// set 2nd checkpoint: end
}
- return $this->showError($sql, $key_field);
+ return $this->showError($sql, $key_field, null, $iterator_class);
}
}
Index: core/kernel/db/db_load_balancer.php
===================================================================
--- core/kernel/db/db_load_balancer.php
+++ core/kernel/db/db_load_balancer.php
@@ -145,6 +145,10 @@
$this->servers = Array ();
$this->servers[0] = Array (
+ 'DBErrorBackoffMaxRetryAttempts' => $config['Database']['DBErrorBackoffMaxRetryAttempts'],
+ 'DBErrorBackoffLogicBaseTime' => $config['Database']['DBErrorBackoffLogicBaseTime'],
+ 'DBEnableLockRetryDebugging' => $config['Database']['DBEnableLockRetryDebugging'],
+
'DBHost' => $config['Database']['DBHost'],
'DBUser' => $config['Database']['DBUser'],
'DBUserPassword' => $config['Database']['DBUserPassword'],
@@ -502,8 +506,20 @@
/** @var kDBConnection $db */
$db = $this->Application->makeClass($db_class, Array ($this->dbType, $this->errorHandler, $server['serverIndex']));
- $db->debugMode = $debug_mode;
- $db->Connect($server['DBHost'], $server['DBUser'], $server['DBUserPassword'], $this->servers[0]['DBName'], !$is_master);
+ $config = array(
+ 'Database' => array(
+ 'DBErrorBackoffMaxRetryAttempts' => $this->servers[0]['DBErrorBackoffMaxRetryAttempts'],
+ 'DBErrorBackoffLogicBaseTime' => $this->servers[0]['DBErrorBackoffLogicBaseTime'],
+ 'DBEnableLockRetryDebugging' => $this->servers[0]['DBEnableLockRetryDebugging'],
+
+ 'DBHost' => $server['DBHost'],
+ 'DBUser' => $server['DBUser'],
+ 'DBUserPassword' => $server['DBUserPassword'],
+ 'DBName' => $this->servers[0]['DBName'],
+ 'DBRetry' => !$is_master,
+ ),
+ );
+ $db->setup($config);
return $db;
}
Index: core/kernel/utility/logger.php
===================================================================
--- core/kernel/utility/logger.php
+++ core/kernel/utility/logger.php
@@ -927,24 +927,29 @@
*
* When not debug mode, then fatal database query won't break anything.
*
- * @param int $code
- * @param string $msg
- * @param string $sql
- * @return bool
- * @access public
- * @throws RuntimeException
+ * @param integer $code Error code.
+ * @param string $msg Error message.
+ * @param string $sql SQL query.
+ * @param boolean|null $throw_exception Throw an exception.
+ *
+ * @return boolean
+ * @throws RuntimeException In CLI mode or while in the Debug Mode.
*/
- public function handleSQLError($code, $msg, $sql)
+ public function handleSQLError($code, $msg, $sql, $throw_exception = null)
{
$error_msg = self::shortenMessage(self::DB_ERROR_PREFIX . ' #' . $code . ' - ' . $msg . '. SQL: ' . trim($sql));
+ if ( $throw_exception === true ) {
+ throw new RuntimeException($error_msg);
+ }
+
if ( isset($this->Application->Debugger) ) {
if ( kUtil::constOn('DBG_SQL_FAILURE') && !defined('IS_INSTALL') ) {
throw new RuntimeException($error_msg);
}
- else {
- $this->Application->Debugger->appendTrace();
- }
+
+ // To show trace above the warning message.
+ $this->Application->Debugger->appendTrace();
}
if ( PHP_SAPI === 'cli' ) {
Index: core/kernel/utility/system_config.php
===================================================================
--- core/kernel/utility/system_config.php
+++ core/kernel/utility/system_config.php
@@ -63,32 +63,39 @@
protected function getDefaults()
{
$ret = array(
- 'AdminDirectory' => '/admin',
- 'AdminPresetsDirectory' => '/admin',
- 'ApplicationClass' => 'kApplication',
- 'ApplicationPath' => '/core/kernel/application.php',
- 'CacheHandler' => 'Fake',
- 'CmsMenuRebuildTime' => 10,
- 'DomainsParsedRebuildTime' => 2,
- 'EditorPath' => '/core/ckeditor/',
- 'EnableSystemLog' => '0',
- 'MemcacheServers' => 'localhost:11211',
- 'CompressionEngine' => '',
- 'RestrictedPath' => DIRECTORY_SEPARATOR . 'system' . DIRECTORY_SEPARATOR . '.restricted',
- 'SectionsParsedRebuildTime' => 5,
- 'StructureTreeRebuildTime' => 10,
- 'SystemLogMaxLevel' => 5,
- 'TemplateMappingRebuildTime' => 5,
- 'TrustProxy' => '0',
- 'UnitCacheRebuildTime' => 10,
- 'WebsiteCharset' => 'utf-8',
- 'WebsitePath' => rtrim(preg_replace('/'.preg_quote(rtrim(defined('REL_PATH') ? REL_PATH : '', '/'), '/').'$/', '', str_replace('\\', '/', dirname($_SERVER['PHP_SELF']))), '/'),
- 'WriteablePath' => DIRECTORY_SEPARATOR . 'system',
- 'SecurityHmacKey' => '',
- 'SecurityEncryptionKey' => '',
+ 'Misc' => array(
+ 'AdminDirectory' => '/admin',
+ 'AdminPresetsDirectory' => '/admin',
+ 'ApplicationClass' => 'kApplication',
+ 'ApplicationPath' => '/core/kernel/application.php',
+ 'CacheHandler' => 'Fake',
+ 'CmsMenuRebuildTime' => 10,
+ 'DomainsParsedRebuildTime' => 2,
+ 'EditorPath' => '/core/ckeditor/',
+ 'EnableSystemLog' => '0',
+ 'MemcacheServers' => 'localhost:11211',
+ 'CompressionEngine' => '',
+ 'RestrictedPath' => DIRECTORY_SEPARATOR . 'system' . DIRECTORY_SEPARATOR . '.restricted',
+ 'SectionsParsedRebuildTime' => 5,
+ 'StructureTreeRebuildTime' => 10,
+ 'SystemLogMaxLevel' => 5,
+ 'TemplateMappingRebuildTime' => 5,
+ 'TrustProxy' => '0',
+ 'UnitCacheRebuildTime' => 10,
+ 'WebsiteCharset' => 'utf-8',
+ 'WebsitePath' => rtrim(preg_replace('/'.preg_quote(rtrim(defined('REL_PATH') ? REL_PATH : '', '/'), '/').'$/', '', str_replace('\\', '/', dirname($_SERVER['PHP_SELF']))), '/'),
+ 'WriteablePath' => DIRECTORY_SEPARATOR . 'system',
+ 'SecurityHmacKey' => '',
+ 'SecurityEncryptionKey' => '',
+ ),
+ 'Database' => array(
+ 'DBErrorBackoffMaxRetryAttempts' => 3,
+ 'DBErrorBackoffLogicBaseTime' => 100,
+ 'DBEnableLockRetryDebugging' => 0,
+ ),
);
- return $this->parseSections ? array('Misc' => $ret) : $ret;
+ return $this->parseSections ? $ret : call_user_func_array('array_merge', $ret);
}
/**
Index: core/kernel/utility/temp_handler.php
===================================================================
--- core/kernel/utility/temp_handler.php
+++ core/kernel/utility/temp_handler.php
@@ -776,8 +776,20 @@
/** @var kDBConnection $connection */
$connection = $this->Application->makeClass( 'kDBConnection', Array (SQL_TYPE, Array ($this->Application, 'handleSQLError')) );
- $connection->debugMode = $this->Application->isDebugMode();
- $connection->Connect(SQL_SERVER, SQL_USER, SQL_PASS, SQL_DB);
+ $vars = kUtil::getSystemConfig()->getData();
+ $config = array(
+ 'Database' => array(
+ 'DBErrorBackoffMaxRetryAttempts' => $vars['DBErrorBackoffMaxRetryAttempts'],
+ 'DBErrorBackoffLogicBaseTime' => $vars['DBErrorBackoffLogicBaseTime'],
+ 'DBEnableLockRetryDebugging' => $vars['DBEnableLockRetryDebugging'],
+
+ 'DBHost' => SQL_SERVER,
+ 'DBUser' => SQL_USER,
+ 'DBUserPassword' => SQL_PASS,
+ 'DBName' => SQL_DB,
+ ),
+ );
+ $connection->setup($config);
}
return $connection;
Index: core/units/helpers/deployment_helper.php
===================================================================
--- core/units/helpers/deployment_helper.php
+++ core/units/helpers/deployment_helper.php
@@ -671,16 +671,23 @@
/**
* Error handler for sql errors.
*
- * @param integer $code Error code.
- * @param string $msg Error message.
- * @param string $sql SQL query.
+ * @param integer $code Error code.
+ * @param string $msg Error message.
+ * @param string $sql SQL query.
+ * @param boolean|null $throw_exception Throw an exception.
*
* @return void
* @throws Exception When SQL error happens.
+ * @throws RuntimeException When requested.
*/
- public function handleSqlError($code, $msg, $sql)
+ public function handleSqlError($code, $msg, $sql, $throw_exception = null)
{
$error_msg = 'FAILED' . PHP_EOL . 'SQL Error #' . $code . ': ' . $msg;
+
+ if ( $throw_exception === true ) {
+ throw new RuntimeException($error_msg);
+ }
+
$this->toLog($error_msg);
$this->displayStatus($error_msg);