Index: core/install.php =================================================================== --- core/install.php +++ core/install.php @@ -332,7 +332,10 @@ case 'db_reconfig': $fields = Array ( 'DBType', 'DBHost', 'DBName', 'DBUser', - 'DBUserPassword', 'DBCollation', 'TablePrefix' + 'DBUserPassword', 'DBCollation', 'TablePrefix', + 'DBErrorBackoffMaxRetryAttempts', + 'DBErrorBackoffLogicBaseTime', + 'DBEnableLockRetryDebugging', ); // set fields @@ -531,9 +534,14 @@ case 'db_reconfig': // 1. check if required fields are filled $section_name = 'Database'; - $required_fields = Array ('DBType', 'DBHost', 'DBName', 'DBUser', 'DBCollation'); + $required_fields = Array ( + 'DBType', 'DBHost', 'DBName', 'DBUser', 'DBCollation', + 'DBErrorBackoffMaxRetryAttempts', + 'DBErrorBackoffLogicBaseTime', + 'DBEnableLockRetryDebugging', + ); foreach ($required_fields as $required_field) { - if (!$this->toolkit->systemConfig->get($required_field, $section_name)) { + if ($this->toolkit->systemConfig->get($required_field, $section_name) === '') { $status = false; $this->errorMessage = 'Please fill all required fields'; break; @@ -1745,15 +1753,25 @@ /** * Installation error handler for sql errors * - * @param int $code - * @param string $msg - * @param string $sql - * @return bool - * @access private + * @param integer $code Error code. + * @param string $msg Error message. + * @param string $sql SQL query. + * @param boolean|null $throw_exception Throw an exception. + * + * @return boolean + * @throws RuntimeException When requested. */ - function DBErrorHandler($code, $msg, $sql) + function DBErrorHandler($code, $msg, $sql, $throw_exception = null) { - $this->errorMessage = 'Query:
'.htmlspecialchars($sql, ENT_QUOTES, 'UTF-8').'
execution result is error:
['.$code.'] '.$msg; + $error_msg = 'Query:
' . htmlspecialchars($sql, ENT_QUOTES, 'UTF-8') . '
'; + $error_msg .= 'execution result is error:
[' . $code . '] ' . $msg; + + if ( $throw_exception === true ) { + throw new RuntimeException($error_msg); + } + + $this->errorMessage = $error_msg; + return true; } Index: core/install/step_templates/db_config.tpl =================================================================== --- core/install/step_templates/db_config.tpl +++ core/install/step_templates/db_config.tpl @@ -66,6 +66,29 @@ + + Database Error Backoff Max Retry Attempts *: + + + + + + + Database Error Backoff Logic Base Time (in milliseconds) *: + + + + + + + Database Lock Retry Debugging Enabled *: + + toolkit->systemConfig->get('DBEnableLockRetryDebugging', 'Database'); ?> + + + + + GetVar('preset') != 'already_installed') { ?> @@ -74,4 +97,4 @@ - \ No newline at end of file + Index: core/install/step_templates/db_reconfig.tpl =================================================================== --- core/install/step_templates/db_reconfig.tpl +++ core/install/step_templates/db_reconfig.tpl @@ -64,4 +64,27 @@ - \ No newline at end of file + + + + Database Error Backoff Max Retry Attempts *: + + + + + + + Database Error Backoff Logic Base Time (in milliseconds)*: + + + + + + + Database Lock Retry Debugging Enabled *: + + toolkit->systemConfig->get('DBEnableLockRetryDebugging', 'Database'); ?> + + + + Index: core/kernel/application.php =================================================================== --- core/kernel/application.php +++ core/kernel/application.php @@ -2458,17 +2458,17 @@ /** * SQL Error Handler * - * @param int $code - * @param string $msg - * @param string $sql - * @return bool - * @access public - * @throws Exception + * @param integer $code Error code. + * @param string $msg Error message. + * @param string $sql SQL query. + * @param boolean|null $throw_exception Throw an exception. + * + * @return boolean * @deprecated */ - public function handleSQLError($code, $msg, $sql) + public function handleSQLError($code, $msg, $sql, $throw_exception = null) { - return $this->_logger->handleSQLError($code, $msg, $sql); + return $this->_logger->handleSQLError($code, $msg, $sql, $throw_exception); } /** Index: core/kernel/db/db_connection.php =================================================================== --- core/kernel/db/db_connection.php +++ core/kernel/db/db_connection.php @@ -84,6 +84,20 @@ protected $errorMessage = ''; /** + * Error retry count. + * + * @var integer + */ + protected $errorRetryCount = 0; + + /** + * Database query, that caused an error. + * + * @var string + */ + protected $erroredQuery = ''; + + /** * Defines if database connection * operations should generate debug * information @@ -102,6 +116,27 @@ protected $_captureStatistics = false; /** + * Error Backoff Maximal Retry Attempts. + * + * @var integer + */ + protected $errorBackoffMaxRetryAttempts; + + /** + * Error backoff logic base time (in milliseconds). + * + * @var integer + */ + protected $errorBackoffLogicBaseTime; + + /** + * Enable lock retry debugging. + * + * @var integer + */ + protected $enableLockRetryDebugging; + + /** * Last query to database * * @var string @@ -278,11 +313,18 @@ $this->debugMode = $this->Application->isDebugMode(); } + $this->errorBackoffMaxRetryAttempts = $config['Database']['DBErrorBackoffMaxRetryAttempts']; + $this->errorBackoffLogicBaseTime = $config['Database']['DBErrorBackoffLogicBaseTime']; + $this->enableLockRetryDebugging = $config['Database']['DBEnableLockRetryDebugging']; + + $retry = array_key_exists('DBRetry', $config['Database']) ? $config['Database']['DBRetry'] : false; + return $this->Connect( $config['Database']['DBHost'], $config['Database']['DBUser'], $config['Database']['DBUserPassword'], - $config['Database']['DBName'] + $config['Database']['DBName'], + $retry ); } @@ -327,13 +369,12 @@ * @param string $sql * @param string $key_field * @param boolean|null $no_debug + * @param string $iterator_class * @return bool * @access protected */ - protected function showError($sql = '', $key_field = null, $no_debug = null) + protected function showError($sql = '', $key_field = null, $no_debug = null, $iterator_class = '') { - static $retry_count = 0; - if ( $no_debug === null ) { $no_debug = $this->noDebuggingState; } @@ -359,56 +400,148 @@ $this->errorCode = $this->connectionID->errno; if ( $this->hasError() ) { + $this->erroredQuery = $sql; $this->errorMessage = $this->connectionID->error; - $ret = $this->callErrorHandler($sql); + $recoverable_errors = array( + 2006, // MySQL server has gone away. + 2013, // Lost connection to MySQL server during query. + 1205, // Lock wait timeout exceeded; try restarting transaction. + 1213, // Deadlock found when trying to get lock; try restarting transaction. + ); - if ( ($this->errorCode == 2006 || $this->errorCode == 2013) && ($retry_count < 3) ) { - // #2006 - MySQL server has gone away - // #2013 - Lost connection to MySQL server during query - $retry_count++; + if ( in_array($this->errorCode, $recoverable_errors) ) { + try { + $ret = $this->callErrorHandler($sql, true); + } + catch ( RuntimeException $e ) { + // Collect the exception for potential re-throwing later. + } + + // Handle case, when specified error handler isn't respecting "$throw_exception" argument. + if ( !isset($e) ) { + $e = new RuntimeException(kLogger::shortenMessage(sprintf( + '%s #%d - %s. SQL: %s', + kLogger::DB_ERROR_PREFIX, + $this->errorCode, + $this->errorMessage, + trim($sql) + ))); + } + + if ( $this->errorRetryCount < $this->errorBackoffMaxRetryAttempts ) { + $this->errorRetryCount++; + $wait_time = $this->getErrorBackoffWaitTime($this->errorRetryCount); + usleep($wait_time * 1000); + + // Write down additional info about retrying a query after a lock error. + if ( ($this->errorCode == 1205 || $this->errorCode == 1213) + && $this->enableLockRetryDebugging + ) { + $log = $this->Application->log(''); + $log->addException($e); + + $cause_mapping = array( + 1205 => 'Lock detected.', + 1213 => 'Deadlock detected.', + ); + $log->addUserData(PHP_EOL . $cause_mapping[$this->errorCode]); + + $log->addUserData(sprintf( + 'Retry: %d of %d.', + $this->errorRetryCount, + $this->errorBackoffMaxRetryAttempts + )); + $log->addUserData('Wait time: ' . $wait_time . 'ms'); + $log->notify(true); + $log->write(); + } + + // Attempt to reconnect upon disconnect. + if ( ($this->errorCode == 2006 || $this->errorCode == 2013) && !$this->ReConnect() ) { + if ( !$ret ) { + exit; + } + + return false; + } + + if ( $iterator_class ) { + return $this->GetIterator($sql, $key_field, $no_debug, $iterator_class); + } - if ( $this->ReConnect() ) { return $this->Query($sql, $key_field, $no_debug); } + + // Show an error back to the user after all the possible retry attempts were exhausted. + throw $e; } + $ret = $this->callErrorHandler($sql); + if (!$ret) { exit; } } - else { - $retry_count = 0; + elseif ( $this->erroredQuery === $sql ) { + // When previously failed query is working now. + $this->erroredQuery = ''; + $this->errorRetryCount = 0; } return false; } /** + * Returns the error backoff wait time in milliseconds (exponential strategy). + * + * @param integer $attempt Attempt. + * + * @return integer + */ + protected function getErrorBackoffWaitTime($attempt) + { + if ( $attempt == 1 ) { + return $this->errorBackoffLogicBaseTime; + } + + return (int)(pow(2, $attempt) * $this->errorBackoffLogicBaseTime); + } + + /** * Sends db error to a predefined error handler * - * @param $sql - * @return bool - * @access protected + * @param string $sql SQL query. + * @param boolean|null $throw_exception Throw an exception. + * + * @return boolean */ - protected function callErrorHandler($sql) + protected function callErrorHandler($sql, $throw_exception = null) { - return call_user_func($this->errorHandler, $this->errorCode, $this->errorMessage, $sql); + return call_user_func($this->errorHandler, $this->errorCode, $this->errorMessage, $sql, $throw_exception); } /** * Default error handler for sql errors * - * @param int $code - * @param string $msg - * @param string $sql - * @return bool - * @access public + * @param integer $code Error code. + * @param string $msg Error message. + * @param string $sql SQL query. + * @param boolean|null $throw_exception Throw an exception. + * + * @return boolean + * @throws RuntimeException When requested. */ - public function handleError($code, $msg, $sql) + public function handleError($code, $msg, $sql, $throw_exception = null) { - echo 'Processing SQL: ' . $sql . '
'; - echo 'Error (' . $code . '): ' . $msg . '
'; + $error_msg = 'Processing SQL: ' . $sql . '
'; + $error_msg .= 'Error (' . $code . '): ' . $msg . '
'; + + if ( $throw_exception === true ) { + throw new RuntimeException(kLogger::shortenMessage($error_msg)); + } + + echo $error_msg; return false; } @@ -632,7 +765,7 @@ // set 2nd checkpoint: end } - return $this->showError($sql, $key_field, $no_debug); + return $this->showError($sql, $key_field, $no_debug, $iterator_class); } /** @@ -1159,7 +1292,7 @@ // set 2nd checkpoint: end } - return $this->showError($sql, $key_field); + return $this->showError($sql, $key_field, null, $iterator_class); } } Index: core/kernel/db/db_load_balancer.php =================================================================== --- core/kernel/db/db_load_balancer.php +++ core/kernel/db/db_load_balancer.php @@ -145,6 +145,10 @@ $this->servers = Array (); $this->servers[0] = Array ( + 'DBErrorBackoffMaxRetryAttempts' => $config['Database']['DBErrorBackoffMaxRetryAttempts'], + 'DBErrorBackoffLogicBaseTime' => $config['Database']['DBErrorBackoffLogicBaseTime'], + 'DBEnableLockRetryDebugging' => $config['Database']['DBEnableLockRetryDebugging'], + 'DBHost' => $config['Database']['DBHost'], 'DBUser' => $config['Database']['DBUser'], 'DBUserPassword' => $config['Database']['DBUserPassword'], @@ -502,8 +506,20 @@ /** @var kDBConnection $db */ $db = $this->Application->makeClass($db_class, Array ($this->dbType, $this->errorHandler, $server['serverIndex'])); - $db->debugMode = $debug_mode; - $db->Connect($server['DBHost'], $server['DBUser'], $server['DBUserPassword'], $this->servers[0]['DBName'], !$is_master); + $config = array( + 'Database' => array( + 'DBErrorBackoffMaxRetryAttempts' => $this->servers[0]['DBErrorBackoffMaxRetryAttempts'], + 'DBErrorBackoffLogicBaseTime' => $this->servers[0]['DBErrorBackoffLogicBaseTime'], + 'DBEnableLockRetryDebugging' => $this->servers[0]['DBEnableLockRetryDebugging'], + + 'DBHost' => $server['DBHost'], + 'DBUser' => $server['DBUser'], + 'DBUserPassword' => $server['DBUserPassword'], + 'DBName' => $this->servers[0]['DBName'], + 'DBRetry' => !$is_master, + ), + ); + $db->setup($config); return $db; } Index: core/kernel/utility/logger.php =================================================================== --- core/kernel/utility/logger.php +++ core/kernel/utility/logger.php @@ -927,24 +927,29 @@ * * When not debug mode, then fatal database query won't break anything. * - * @param int $code - * @param string $msg - * @param string $sql - * @return bool - * @access public - * @throws RuntimeException + * @param integer $code Error code. + * @param string $msg Error message. + * @param string $sql SQL query. + * @param boolean|null $throw_exception Throw an exception. + * + * @return boolean + * @throws RuntimeException In CLI mode or while in the Debug Mode. */ - public function handleSQLError($code, $msg, $sql) + public function handleSQLError($code, $msg, $sql, $throw_exception = null) { $error_msg = self::shortenMessage(self::DB_ERROR_PREFIX . ' #' . $code . ' - ' . $msg . '. SQL: ' . trim($sql)); + if ( $throw_exception === true ) { + throw new RuntimeException($error_msg); + } + if ( isset($this->Application->Debugger) ) { if ( kUtil::constOn('DBG_SQL_FAILURE') && !defined('IS_INSTALL') ) { throw new RuntimeException($error_msg); } - else { - $this->Application->Debugger->appendTrace(); - } + + // To show trace above the warning message. + $this->Application->Debugger->appendTrace(); } if ( PHP_SAPI === 'cli' ) { Index: core/kernel/utility/system_config.php =================================================================== --- core/kernel/utility/system_config.php +++ core/kernel/utility/system_config.php @@ -63,32 +63,39 @@ protected function getDefaults() { $ret = array( - 'AdminDirectory' => '/admin', - 'AdminPresetsDirectory' => '/admin', - 'ApplicationClass' => 'kApplication', - 'ApplicationPath' => '/core/kernel/application.php', - 'CacheHandler' => 'Fake', - 'CmsMenuRebuildTime' => 10, - 'DomainsParsedRebuildTime' => 2, - 'EditorPath' => '/core/ckeditor/', - 'EnableSystemLog' => '0', - 'MemcacheServers' => 'localhost:11211', - 'CompressionEngine' => '', - 'RestrictedPath' => DIRECTORY_SEPARATOR . 'system' . DIRECTORY_SEPARATOR . '.restricted', - 'SectionsParsedRebuildTime' => 5, - 'StructureTreeRebuildTime' => 10, - 'SystemLogMaxLevel' => 5, - 'TemplateMappingRebuildTime' => 5, - 'TrustProxy' => '0', - 'UnitCacheRebuildTime' => 10, - 'WebsiteCharset' => 'utf-8', - 'WebsitePath' => rtrim(preg_replace('/'.preg_quote(rtrim(defined('REL_PATH') ? REL_PATH : '', '/'), '/').'$/', '', str_replace('\\', '/', dirname($_SERVER['PHP_SELF']))), '/'), - 'WriteablePath' => DIRECTORY_SEPARATOR . 'system', - 'SecurityHmacKey' => '', - 'SecurityEncryptionKey' => '', + 'Misc' => array( + 'AdminDirectory' => '/admin', + 'AdminPresetsDirectory' => '/admin', + 'ApplicationClass' => 'kApplication', + 'ApplicationPath' => '/core/kernel/application.php', + 'CacheHandler' => 'Fake', + 'CmsMenuRebuildTime' => 10, + 'DomainsParsedRebuildTime' => 2, + 'EditorPath' => '/core/ckeditor/', + 'EnableSystemLog' => '0', + 'MemcacheServers' => 'localhost:11211', + 'CompressionEngine' => '', + 'RestrictedPath' => DIRECTORY_SEPARATOR . 'system' . DIRECTORY_SEPARATOR . '.restricted', + 'SectionsParsedRebuildTime' => 5, + 'StructureTreeRebuildTime' => 10, + 'SystemLogMaxLevel' => 5, + 'TemplateMappingRebuildTime' => 5, + 'TrustProxy' => '0', + 'UnitCacheRebuildTime' => 10, + 'WebsiteCharset' => 'utf-8', + 'WebsitePath' => rtrim(preg_replace('/'.preg_quote(rtrim(defined('REL_PATH') ? REL_PATH : '', '/'), '/').'$/', '', str_replace('\\', '/', dirname($_SERVER['PHP_SELF']))), '/'), + 'WriteablePath' => DIRECTORY_SEPARATOR . 'system', + 'SecurityHmacKey' => '', + 'SecurityEncryptionKey' => '', + ), + 'Database' => array( + 'DBErrorBackoffMaxRetryAttempts' => 3, + 'DBErrorBackoffLogicBaseTime' => 100, + 'DBEnableLockRetryDebugging' => 0, + ), ); - return $this->parseSections ? array('Misc' => $ret) : $ret; + return $this->parseSections ? $ret : call_user_func_array('array_merge', $ret); } /** Index: core/kernel/utility/temp_handler.php =================================================================== --- core/kernel/utility/temp_handler.php +++ core/kernel/utility/temp_handler.php @@ -776,8 +776,20 @@ /** @var kDBConnection $connection */ $connection = $this->Application->makeClass( 'kDBConnection', Array (SQL_TYPE, Array ($this->Application, 'handleSQLError')) ); - $connection->debugMode = $this->Application->isDebugMode(); - $connection->Connect(SQL_SERVER, SQL_USER, SQL_PASS, SQL_DB); + $vars = kUtil::getSystemConfig()->getData(); + $config = array( + 'Database' => array( + 'DBErrorBackoffMaxRetryAttempts' => $vars['DBErrorBackoffMaxRetryAttempts'], + 'DBErrorBackoffLogicBaseTime' => $vars['DBErrorBackoffLogicBaseTime'], + 'DBEnableLockRetryDebugging' => $vars['DBEnableLockRetryDebugging'], + + 'DBHost' => SQL_SERVER, + 'DBUser' => SQL_USER, + 'DBUserPassword' => SQL_PASS, + 'DBName' => SQL_DB, + ), + ); + $connection->setup($config); } return $connection; Index: core/units/helpers/deployment_helper.php =================================================================== --- core/units/helpers/deployment_helper.php +++ core/units/helpers/deployment_helper.php @@ -671,16 +671,23 @@ /** * Error handler for sql errors. * - * @param integer $code Error code. - * @param string $msg Error message. - * @param string $sql SQL query. + * @param integer $code Error code. + * @param string $msg Error message. + * @param string $sql SQL query. + * @param boolean|null $throw_exception Throw an exception. * * @return void * @throws Exception When SQL error happens. + * @throws RuntimeException When requested. */ - public function handleSqlError($code, $msg, $sql) + public function handleSqlError($code, $msg, $sql, $throw_exception = null) { $error_msg = 'FAILED' . PHP_EOL . 'SQL Error #' . $code . ': ' . $msg; + + if ( $throw_exception === true ) { + throw new RuntimeException($error_msg); + } + $this->toLog($error_msg); $this->displayStatus($error_msg);