Bug #104316
openDB compare results in error with SQLite in TYPO3 13.2
0%
Description
Hello,
for testing TYPO3 13.2 I have set DB to SQLite. After upgrading from TYPO3 13.1 I get following error in DB compare:
Core: Exception handler (WEB): Uncaught TYPO3 Exception: Doctrine\DBAL\Schema\Index::_addColumn(): Argument #1 ($column) must be of type string, null given, called in /var/www/vhosts/jweiland-hosting.de/httpdocs/typo3retter.de/13.4/vendor/doctrine/dbal/src/Schema/Index.php on line 60 | TypeError thrown in file /var/www/vhosts/jweiland-hosting.de/httpdocs/typo3retter.de/13.4/vendor/doctrine/dbal/src/Schema/Index.php in line 68. Requested URL: https://nix-da.typo3retter.de/typo3/install.php?install%5Bcontroller%5D=maintenance&install%5Bcontext%5D=backend&install%5Baction%5D=databaseAnalyzerAnalyze
Seems that DB compare tries to create a new index:
at Doctrine\DBAL\Schema\Index->__construct() in /var/www/vhosts/900410.jweiland-hosting.de/httpdocs/typo3retter.de/13.4/vendor/doctrine/dbal/src/Schema/AbstractSchemaManager.php line 733 } $indexes = []; foreach ($result as $indexKey => $data) { $indexes[$indexKey] = new Index( $data['name'], $data['columns'], $data['unique'], $data['primary'],
Stefan
Files
Updated by Stefan Froemken about 2 months ago
I have added tablename and the last index loops of that table:
TABLE: sys_log INDEX arrays: array(6) { ["name"]=> string(7) "primary" ["columns"]=> array(1) { [0]=> string(3) "uid" } ["unique"]=> bool(true) ["primary"]=> bool(true) ["flags"]=> array(0) { } ["options"]=> array(1) { ["lengths"]=> array(1) { [0]=> NULL } } } array(6) { ["name"]=> string(20) "index_level_a22f6b36" ["columns"]=> array(1) { [0]=> string(5) "level" } ["unique"]=> bool(false) ["primary"]=> bool(false) ["flags"]=> array(0) { } ["options"]=> array(1) { ["lengths"]=> array(1) { [0]=> NULL } } } array(6) { ["name"]=> string(22) "index_channel_02b8502e" ["columns"]=> array(1) { [0]=> string(7) "channel" } ["unique"]=> bool(false) ["primary"]=> bool(false) ["flags"]=> array(0) { } ["options"]=> array(1) { ["lengths"]=> array(1) { [0]=> NULL } } } array(6) { ["name"]=> string(19) "errorcount_2a6ec0b4" ["columns"]=> array(2) { [0]=> string(6) "tstamp" [1]=> string(5) "error" } ["unique"]=> bool(false) ["primary"]=> bool(false) ["flags"]=> array(0) { } ["options"]=> array(1) { ["lengths"]=> array(2) { [0]=> NULL [1]=> NULL } } } array(6) { ["name"]=> string(19) "combined_1_9d80faf5" ["columns"]=> array(3) { [0]=> string(6) "tstamp" [1]=> string(4) "type" [2]=> string(6) "userid" } ["unique"]=> bool(false) ["primary"]=> bool(false) ["flags"]=> array(0) { } ["options"]=> array(1) { ["lengths"]=> array(3) { [0]=> NULL [1]=> NULL [2]=> NULL } } } array(6) { ["name"]=> string(16) "request_a5c72ce4" ["columns"]=> array(1) { [0]=> string(10) "request_id" } ["unique"]=> bool(false) ["primary"]=> bool(false) ["flags"]=> array(0) { } ["options"]=> array(1) { ["lengths"]=> array(1) { [0]=> NULL } } } array(6) { ["name"]=> string(18) "user_auth_014b30af" ["columns"]=> array(3) { [0]=> string(4) "type" [1]=> string(6) "action" [2]=> string(6) "tstamp" } ["unique"]=> bool(false) ["primary"]=> bool(false) ["flags"]=> array(0) { } ["options"]=> array(1) { ["lengths"]=> array(3) { [0]=> NULL [1]=> NULL [2]=> NULL } } } array(6) { ["name"]=> string(18) "recuidIdx_c290c39d" ["columns"]=> array(1) { [0]=> string(6) "recuid" } ["unique"]=> bool(false) ["primary"]=> bool(false) ["flags"]=> array(0) { } ["options"]=> array(1) { ["lengths"]=> array(1) { [0]=> NULL } } } array(6) { ["name"]=> string(14) "event_63aff1ce" ["columns"]=> array(2) { [0]=> string(6) "userid" [1]=> string(9) "event_pid" } ["unique"]=> bool(false) ["primary"]=> bool(false) ["flags"]=> array(0) { } ["options"]=> array(1) { ["lengths"]=> array(2) { [0]=> NULL [1]=> NULL } } }
Updated by Stefan Froemken about 2 months ago
Backtrace:
(1/1) TypeError Doctrine\DBAL\Schema\Index::_addColumn(): Argument #1 ($column) must be of type string, null given, called in /var/www/vhosts/900410.jweiland-hosting.de/httpdocs/typo3retter.de/13.4/vendor/doctrine/dbal/src/Schema/Index.php on line 60 in /var/www/vhosts/900410.jweiland-hosting.de/httpdocs/typo3retter.de/13.4/vendor/doctrine/dbal/src/Schema/Index.php line 68 $this->addFlag($flag); } } protected function _addColumn(string $column): void { $this->_columns[$column] = new Identifier($column); } at Doctrine\DBAL\Schema\Index->_addColumn() in /var/www/vhosts/900410.jweiland-hosting.de/httpdocs/typo3retter.de/13.4/vendor/doctrine/dbal/src/Schema/Index.php line 60 $this->_isUnique = $isUnique; $this->_isPrimary = $isPrimary; foreach ($columns as $column) { $this->_addColumn($column); } foreach ($flags as $flag) { $this->addFlag($flag); at Doctrine\DBAL\Schema\Index->__construct() in /var/www/vhosts/900410.jweiland-hosting.de/httpdocs/typo3retter.de/13.4/vendor/doctrine/dbal/src/Schema/AbstractSchemaManager.php line 733 } $indexes = []; foreach ($result as $indexKey => $data) { $indexes[$indexKey] = new Index( $data['name'], $data['columns'], $data['unique'], $data['primary'], at Doctrine\DBAL\Schema\AbstractSchemaManager->_getPortableTableIndexesList() in /var/www/vhosts/900410.jweiland-hosting.de/httpdocs/typo3retter.de/13.4/vendor/doctrine/dbal/src/Schema/SQLiteSchemaManager.php line 161 $indexBuffer[] = $idx; } } return parent::_getPortableTableIndexesList($indexBuffer, $tableName); } /** * {@inheritDoc} at Doctrine\DBAL\Schema\SQLiteSchemaManager->_getPortableTableIndexesList() in /var/www/vhosts/900410.jweiland-hosting.de/httpdocs/typo3retter.de/13.4/vendor/doctrine/dbal/src/Schema/AbstractSchemaManager.php line 211 $tables[] = new Table( $tableName, $this->_getPortableTableColumnList($tableName, $database, $tableColumns), $this->_getPortableTableIndexesList($indexColumnsByTable[$tableName] ?? [], $tableName), [], $this->_getPortableTableForeignKeysList($foreignKeyColumnsByTable[$tableName] ?? []), $tableOptionsByTable[$tableName] ?? [], ); at Doctrine\DBAL\Schema\AbstractSchemaManager->listTables() in /var/www/vhosts/900410.jweiland-hosting.de/httpdocs/typo3retter.de/13.4/vendor/doctrine/dbal/src/Schema/AbstractSchemaManager.php line 802 if ($this->platform->supportsSequences()) { $sequences = $this->listSequences(); } $tables = $this->listTables(); return new Schema($tables, $sequences, $this->createSchemaConfig(), $schemaNames); } at Doctrine\DBAL\Schema\AbstractSchemaManager->introspectSchema() in /var/www/vhosts/900410.jweiland-hosting.de/httpdocs/typo3retter.de/13.4/vendor/typo3/cms-core/Classes/Database/Schema/ConnectionMigrator.php line 241 ); } // Build the schema definitions $fromSchema = $this->connection->createSchemaManager()->introspectSchema(); $toSchema = $this->buildExpectedSchemaDefinitions($this->connectionName); // Add current table options to the fromSchema $tableOptions = $this->getTableOptions($this->getSchemaTableNames($fromSchema)); at TYPO3\CMS\Core\Database\Schema\ConnectionMigrator->buildSchemaDiff() in /var/www/vhosts/900410.jweiland-hosting.de/httpdocs/typo3retter.de/13.4/vendor/typo3/cms-core/Classes/Database/Schema/ConnectionMigrator.php line 103 * suggestions in the form of SQL statements. */ public function getUpdateSuggestions(bool $remove = false): array { $schemaDiff = $this->buildSchemaDiff(); if ($remove === false) { return array_merge_recursive( ['add' => [], 'create_table' => [], 'change' => [], 'change_currentValue' => []], $this->getNewFieldUpdateSuggestions($schemaDiff), at TYPO3\CMS\Core\Database\Schema\ConnectionMigrator->getUpdateSuggestions() in /var/www/vhosts/900410.jweiland-hosting.de/httpdocs/typo3retter.de/13.4/vendor/typo3/cms-core/Classes/Database/Schema/SchemaMigrator.php line 65 $updateSuggestions = []; foreach ($this->connectionPool->getConnectionNames() as $connectionName) { $connection = $this->connectionPool->getConnectionByName($connectionName); $connectionMigrator = ConnectionMigrator::create($connectionName, $connection, $tables); $updateSuggestions[$connectionName] = $connectionMigrator->getUpdateSuggestions($remove); } return $updateSuggestions; } at TYPO3\CMS\Core\Database\Schema\SchemaMigrator->getUpdateSuggestions() in /var/www/vhosts/900410.jweiland-hosting.de/httpdocs/typo3retter.de/13.4/vendor/typo3/cms-install/Classes/Controller/MaintenanceController.php line 233 $suggestions = []; try { $sqlReader = $container->get(SqlReader::class); $sqlStatements = $sqlReader->getCreateTableStatementArray($sqlReader->getTablesDefinitionString()); $addCreateChange = $this->schemaMigrator->getUpdateSuggestions($sqlStatements); // Aggregate the per-connection statements into one flat array $addCreateChange = array_merge_recursive(...array_values($addCreateChange)); if (!empty($addCreateChange['create_table'])) { at TYPO3\CMS\Install\Controller\MaintenanceController->databaseAnalyzerAnalyzeAction() in /var/www/vhosts/900410.jweiland-hosting.de/httpdocs/typo3retter.de/13.4/vendor/typo3/cms-install/Classes/Middleware/Maintenance.php line 233 'Unknown action method ' . $action . ' in controller ' . $controllerName, 1505216027 ); } $response = $controller->$action($request); } return $response; } at TYPO3\CMS\Install\Middleware\Maintenance->process() in /var/www/vhosts/900410.jweiland-hosting.de/httpdocs/typo3retter.de/13.4/vendor/typo3/cms-core/Classes/Http/MiddlewareDispatcher.php line 111 } public function handle(ServerRequestInterface $request): ResponseInterface { return $this->middleware->process($request, $this->next); } }; } at Psr\Http\Server\RequestHandlerInterface@anonymous/var/www/vhosts/900410.jweiland-hosting.de/httpdocs/typo3retter.de/13.4/vendor/typo3/cms-core/Classes/Http/MiddlewareDispatcher.php:93$53e->handle() in /var/www/vhosts/900410.jweiland-hosting.de/httpdocs/typo3retter.de/13.4/vendor/typo3/cms-core/Classes/Middleware/NormalizedParamsAttribute.php line 44 $normalizedParams = $request->getAttribute('normalizedParams', null); if ($normalizedParams === null) { $request = $request->withAttribute('normalizedParams', NormalizedParams::createFromRequest($request)); } return $handler->handle($request); } } at TYPO3\CMS\Core\Middleware\NormalizedParamsAttribute->process() in /var/www/vhosts/900410.jweiland-hosting.de/httpdocs/typo3retter.de/13.4/vendor/typo3/cms-core/Classes/Http/MiddlewareDispatcher.php line 162 if (!$middleware instanceof MiddlewareInterface) { throw new \InvalidArgumentException(get_class($middleware) . ' does not implement ' . MiddlewareInterface::class, 1516821342); } return $middleware->process($request, $this->next); } }; } } at Psr\Http\Server\RequestHandlerInterface@anonymous/var/www/vhosts/900410.jweiland-hosting.de/httpdocs/typo3retter.de/13.4/vendor/typo3/cms-core/Classes/Http/MiddlewareDispatcher.php:128$53f->handle() in /var/www/vhosts/900410.jweiland-hosting.de/httpdocs/typo3retter.de/13.4/vendor/typo3/cms-core/Classes/Http/MiddlewareDispatcher.php line 70 * Invoke the middleware stack */ public function handle(ServerRequestInterface $request): ResponseInterface { return $this->tip->handle($request); } /** * Seed the middleware stack with the inner request handler at TYPO3\CMS\Core\Http\MiddlewareDispatcher->handle() in /var/www/vhosts/900410.jweiland-hosting.de/httpdocs/typo3retter.de/13.4/vendor/typo3/cms-core/Classes/Http/AbstractApplication.php line 80 public function handle(ServerRequestInterface $request): ResponseInterface { try { $response = $this->requestHandler->handle($request); } catch (ImmediateResponseException $exception) { $response = $exception->getResponse(); } return $response; at TYPO3\CMS\Core\Http\AbstractApplication->handle() in /var/www/vhosts/900410.jweiland-hosting.de/httpdocs/typo3retter.de/13.4/vendor/typo3/cms-install/Classes/Http/Application.php line 46 public function handle(ServerRequestInterface $request): ResponseInterface { $this->initializeContext(); $request = $request->withAttribute('applicationType', SystemEnvironmentBuilder::REQUESTTYPE_INSTALL); return parent::handle($request) ->withHeader('X-Frame-Options', 'SAMEORIGIN'); } /** at TYPO3\CMS\Install\Http\Application->handle() in /var/www/vhosts/900410.jweiland-hosting.de/httpdocs/typo3retter.de/13.4/vendor/typo3/cms-core/Classes/Http/AbstractApplication.php line 92 * Set up the application and shut it down afterwards */ final public function run() { $response = $this->handle(ServerRequestFactory::fromGlobals()); $this->sendResponse($response); } } at TYPO3\CMS\Core\Http\AbstractApplication->run() in /var/www/vhosts/900410.jweiland-hosting.de/httpdocs/typo3retter.de/13.4/public/typo3/install.php line 19 call_user_func(static function () { $classLoader = require dirname(dirname(__DIR__)).'/vendor/autoload.php'; \TYPO3\CMS\Core\Core\SystemEnvironmentBuilder::run(1); \TYPO3\CMS\Core\Core\Bootstrap::init($classLoader, true)->get(\TYPO3\CMS\Install\Http\Application::class)->run(); }); at {closure}() in /var/www/vhosts/900410.jweiland-hosting.de/httpdocs/typo3retter.de/13.4/public/typo3/install.php line 20 call_user_func(static function () { $classLoader = require dirname(dirname(__DIR__)).'/vendor/autoload.php'; \TYPO3\CMS\Core\Core\SystemEnvironmentBuilder::run(1); \TYPO3\CMS\Core\Core\Bootstrap::init($classLoader, true)->get(\TYPO3\CMS\Install\Http\Application::class)->run(); });
Updated by Stefan Froemken about 2 months ago
- File cms-dc1190ef.sqlite.gz cms-dc1190ef.sqlite.gz added
Updated by Stefan Bürk about 2 months ago
Looks like Doctrine DBAL retrieves one column for one index without a name, and it fails
during buiding the "current database scheme" - which is fully in hand of Doctrine DBAL,
at least for now.
It is not related to our custom Database Analyzer implementations. Needs deeper investigation,
and thus thanks for the SQLite file. We may be able to provide a direct fix in core until it
get's merged and released in upstream Doctrine, if that is really the case.
Thanks for reporting and providing the Stack Trace and SQLite file which is really really helpful.
Updated by Stefan Froemken about 2 months ago
I have downloaded the sqlite file to debug locally.
It's table: sys_refindexIndex Columns:
- ref_table
- ref_uid
- tablename
- workspace
- null
- null
- null
- null
Updated by Stefan Froemken about 2 months ago
It's the lookup_ref index which was inserted with version 13.2 three months ago. With this patch four new columns were added to sys_refindex. It seems that the new index was first applied, instead of adding the four new columns first in DB compare.
Updated by Stefan Froemken about 2 months ago
- File Bildschirmfoto 2024-07-05 um 21.37.10.png Bildschirmfoto 2024-07-05 um 21.37.10.png added
- File Bildschirmfoto 2024-07-05 um 21.29.33.png Bildschirmfoto 2024-07-05 um 21.29.33.png added
I have removed the wrong index (see picture). Go in DB Compare again (See second picture) and press apply. Some error ocurrs
Updated by Garvin Hicking about 2 months ago
@ StefanBürk: Just noting my thoughts here. Looks as if the order of applying the queries is not the one as shown in the screenshot. As if "create table" is executed before the "alter table" column additions. Stefan Froemken mentioned that if you first select all "alter table" statements and execute, and then the rest, the structure is properly applied...?
Updated by Stefan Bürk about 2 months ago
Garvin Hicking wrote in #note-9:
@ StefanBürk: Just noting my thoughts here. Looks as if the order of applying the queries is not the one as shown in the screenshot. As if "create table" is executed before the "alter table" column additions. Stefan Froemken mentioned that if you first select all "alter table" statements and execute, and then the rest, the structure is properly applied...?
Thanks Garvin - I just had a quick look before and asked for additional stack trace. That may explain the "order" and what eventually leaded to it - but not really. This still means that the SQLite file (Database) returns an index definition for an index in the database having no column(names) set. Usually, that should be prevented on creation (why did sqlite allowed it than? And also that may need a `safety guard` in doctrine to avoid that TypeError in that case.
Note that this is one of many reason I want to refactor the database analyzer stack and ensure that these operation are only executable in the right order - and not unsafe splitted pre/post sql statements which has been splittet into dedicated statements an no-one realizes that they are grouped and reflects a "single task required to do in steps". We will see.
Updated by Garvin Hicking about 2 months ago
· Edited
Totally agree, my first reply to Stefan was "SQLite should not be able to do that". Having an index on non-existing columns is not something I'd expect to get. It should lead to an error IMO.
Maybe we can catch that specific exception for now and emit as a partial error, but continue executing the other statements. Then the columns should be created, the error reported, if you re-try then, the indexes might get creatable? I can offer to take a look so you don't need to, next week?
With your bigger picture in mind this should be what you spend time on 😅
Updated by Garvin Hicking about 2 months ago
- Status changed from New to Accepted
- Target version changed from next-patchlevel to 13 LTS
I've checked this out for a bit now but I sadly don't see a way out of this without involving DBAL. The error is so deep within the Schema Manager of DBAL that we can't fix the index state from outside.
Kind of related bug reports:
https://github.com/doctrine/dbal/issues/6392
https://github.com/doctrine/dbal/issues/5306
There it's about MySQL, but both apply due to a "NULL" index name like in SQLite. Do you want me to create an issue and nag Alexander Turek about it, or will you? :-D
Updated by Markus Klein about 1 month ago
From my stack trace:
at Doctrine\DBAL\Schema\AbstractSchemaManager->_getPortableTableIndexesList(array(array('key_name' => 'primary', 'primary' => true, 'non_unique' => false, 'column_name' => 'hash'), array('key_name' => 'lookup_ref_7061b779', 'primary' => false, 'non_unique' => true, 'column_name' => 'ref_table'), array('key_name' => 'lookup_ref_7061b779', 'primary' => false, 'non_unique' => true, 'column_name' => 'ref_uid'), array('key_name' => 'lookup_ref_7061b779', 'primary' => false, 'non_unique' => true, 'column_name' => 'tablename'), array('key_name' => 'lookup_ref_7061b779', 'primary' => false, 'non_unique' => true, 'column_name' => 'workspace'), array('key_name' => 'lookup_ref_7061b779', 'primary' => false, 'non_unique' => true, 'column_name' => null), array('key_name' => 'lookup_ref_7061b779', 'primary' => false, 'non_unique' => true, 'column_name' => null), array('key_name' => 'lookup_ref_7061b779', 'primary' => false, 'non_unique' => true, 'column_name' => null), array('key_name' => 'lookup_ref_7061b779', 'primary' => false, 'non_unique' => true, 'column_name' => null), array('key_name' => 'lookup_rec_000b43f6', 'primary' => false, 'non_unique' => true, 'column_name' => 'tablename'), array('key_name' => 'lookup_rec_000b43f6', 'primary' => false, 'non_unique' => true, 'column_name' => 'recuid'), array('key_name' => 'lookup_uid_37958fdc', 'primary' => false, 'non_unique' => true, 'column_name' => 'ref_table'), array('key_name' => 'lookup_uid_37958fdc', 'primary' => false, 'non_unique' => true, 'column_name' => 'ref_uid'), array('key_name' => 'lookup_string_13590c2e', 'primary' => false, 'non_unique' => true, 'column_name' => 'ref_string')), 'sys_refindex')
Updated by Stefan Bürk 26 days ago
- Related to Bug #95577: Database compare fails on index creation when disabling file metadata translation added