From 805627be170a2f05c9194f123499c47a8a75e257 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mike=20Schw=C3=B6rer?= Date: Thu, 20 Oct 2022 14:56:27 +0200 Subject: [PATCH] Add check-db to egg and save parents as json-objects --- data/css_compress/compress.py | 2 +- www/commands/extendedgitgraph_refresh.php | 9 +- www/data/css/styles.css | 4 + www/data/css/styles.min.css | 1 + www/data/css/styles_admin.scss | 1 + www/extern/egg/EGGDatabase.php | 92 ++++++++++++++++++++- www/extern/egg/ExtendedGitGraph2.php | 14 ++++ www/extern/egg/db_init.sql | 11 +++ www/internals/modules/mikeschergitgraph.php | 8 ++ www/internals/modules/selftest.php | 76 ++++++++++++++++- www/pages/admin.php | 12 +-- www/shell/extendedgitgraph_refresh.php | 20 ++++- 12 files changed, 234 insertions(+), 16 deletions(-) diff --git a/data/css_compress/compress.py b/data/css_compress/compress.py index a3b7795..5855aa4 100644 --- a/data/css_compress/compress.py +++ b/data/css_compress/compress.py @@ -10,7 +10,7 @@ import shutil def findnext(strdata, start, searchchr): depth = 0 for i in range(start, len(strdata)): - if strdata[i] == searchchr: return i; + if strdata[i] == searchchr: return i def findclose(strdata, start): diff --git a/www/commands/extendedgitgraph_refresh.php b/www/commands/extendedgitgraph_refresh.php index 3da7e52..51b12ef 100644 --- a/www/commands/extendedgitgraph_refresh.php +++ b/www/commands/extendedgitgraph_refresh.php @@ -14,8 +14,15 @@ if (!$r1) echo 'EGG::update failed.'; } -$r2 = $SITE->modules->ExtendedGitGraph()->updateCache(); +$r2 = $SITE->modules->ExtendedGitGraph()->checkDatabaseConsistency(); if (!$r2) +{ + http_response_code(500); + echo 'EGG::checkDatabaseConsistency failed.'; +} + +$r3 = $SITE->modules->ExtendedGitGraph()->updateCache(); +if (!$r3) { http_response_code(500); echo 'EGG::updateCache failed.'; diff --git a/www/data/css/styles.css b/www/data/css/styles.css index 70980b1..2326daf 100644 --- a/www/data/css/styles.css +++ b/www/data/css/styles.css @@ -869,6 +869,10 @@ html, body { min-width: 200px; } +.kvl_250 div span:first-child { + min-width: 250px; +} + .kvl_300 div span:first-child { min-width: 300px; } diff --git a/www/data/css/styles.min.css b/www/data/css/styles.min.css index 1a98af9..9f02ded 100644 --- a/www/data/css/styles.min.css +++ b/www/data/css/styles.min.css @@ -167,6 +167,7 @@ html,body{margin:0;padding:0;height:100%} .keyvaluelist div span:first-child{font-weight:bold;min-width:500px;align-self:start} .kvl_100 div span:first-child{min-width:100px} .kvl_200 div span:first-child{min-width:200px} +.kvl_250 div span:first-child{min-width:250px} .kvl_300 div span:first-child{min-width:300px} .selftest_parent{width:800px;max-width:100%} .selftest_outputchild{border:1px solid #888;background:#f8f8f8;color:#000;font-family:Consolas,Monaco,"Courier New",Menlo,monospace;white-space:pre;max-height:600px;overflow-x:auto;overflow-y:auto} diff --git a/www/data/css/styles_admin.scss b/www/data/css/styles_admin.scss index 46fd2f5..ee33dd3 100644 --- a/www/data/css/styles_admin.scss +++ b/www/data/css/styles_admin.scss @@ -39,6 +39,7 @@ .kvl_100 div span:first-child { min-width: 100px; } .kvl_200 div span:first-child { min-width: 200px; } +.kvl_250 div span:first-child { min-width: 250px; } .kvl_300 div span:first-child { min-width: 300px; } .selftest_parent { diff --git a/www/extern/egg/EGGDatabase.php b/www/extern/egg/EGGDatabase.php index 290bb8c..1198475 100644 --- a/www/extern/egg/EGGDatabase.php +++ b/www/extern/egg/EGGDatabase.php @@ -258,7 +258,8 @@ class EGGDatabase foreach ($commits as $commit) { - $strparents = implode(";", $commit->Parents); + $jparents = "[]"; + if (count($commit->Parents) > 0) $jparents = '["'.implode('","', $commit->Parents).'"]'; $this->sql_exec_pre_prep($stmtAddCommit, [ @@ -275,7 +276,7 @@ class EGGDatabase [":cm", $commit->CommitterEmail, PDO::PARAM_STR], [":msg", $commit->Message, PDO::PARAM_STR], [":dat", $commit->Date, PDO::PARAM_STR], - [":prt", $strparents, PDO::PARAM_STR], + [":prt", $jparents, PDO::PARAM_STR], ]); $dbid = $this->sql_query_assoc_pre_prep($stmtGetID, @@ -527,7 +528,7 @@ class EGGDatabase $c->CommitterEmail = $row['committer_email']; $c->Message = $row['message']; $c->Date = $row['date']; - $c->Parents = array_filter(explode(';', $row['parent_commits']), fn($p) => $p !== ''); + $c->Parents = json_decode($row['parent_commits']); $r []= $c; } return $r; @@ -555,9 +556,92 @@ class EGGDatabase $c->CommitterEmail = $row['committer_email']; $c->Message = $row['message']; $c->Date = $row['date']; - $c->Parents = array_filter(explode(';', $row['parent_commits']), fn($p) => $p !== ''); + $c->Parents = json_decode($row['parent_commits']); $r []= $c; } return $r; } + + public function checkDatabase(): array{ + + $result = []; + + // ======== [1] invalid json in parent_commits ========== + { + $errors = $this->sql_query_assoc_prep("SELECT * FROM metadata WHERE NOT json_valid(parent_commits)", []); + + foreach ($errors as $e) { + $result []= "Found commit-metadata entry {".$e['hash']."} with invalid json in 'parent_commits'"; + } + } + + // ======== [2] metadata with missing parent ========== + { + $errors = $this->sql_query_assoc_prep(" +SELECT md1.*, md2.hash as check_hash FROM ( + + SELECT + md1.*, mdp.value AS parent + FROM + metadata AS md1, json_each(md1.parent_commits) AS mdp + +) AS md1 + +LEFT JOIN metadata AS md2 ON md1.parent = md2.hash + +WHERE md1.parent != '' AND check_hash IS NULL", []); + + foreach ($errors as $e) { + $result []= "Found commit-metadata entry {".$e['hash']."} with an reference to a not-existing parent '".$e['parent']."'"; + } + } + + // ======== [3] repositories without branches ========== + { + $errors = $this->sql_query_assoc_prep("SELECT repositories.*, (SELECT COUNT(*) FROM branches WHERE repositories.id = branches.repo_id) AS branch_count FROM repositories WHERE branch_count = 0", []); + + foreach ($errors as $e) { + $result []= "Found repository [".$e['id']."]'".$e['name']."' without any branches"; + } + } + + + // ======== [4] branches without commits ========== + { + $errors = $this->sql_query_assoc_prep("SELECT branches.*, (SELECT COUNT(*) FROM commits WHERE branches.id = commits.branch_id) AS commit_count FROM branches WHERE commit_count = 0", []); + + foreach ($errors as $e) { + $result []= "Found branch [".$e['id']."]'".$e['name']."' without any commits"; + } + } + + // ======== [5] commits with missing metadata ========== + { + $errors = $this->sql_query_assoc_prep("SELECT commits.*, metadata.hash AS mdh FROM commits LEFT JOIN metadata ON commits.hash = metadata.hash WHERE mdh IS NULL", []); + + foreach ($errors as $e) { + $result []= "Missing metadata for commit ".$e['id']." | {".$e['hash']."}"; + } + } + + // ======== [6] metadata with missing commits ========== + { + $errors = $this->sql_query_assoc_prep("SELECT metadata.*, commits.hash AS ch FROM metadata LEFT JOIN commits ON commits.hash = metadata.hash WHERE ch IS NULL", []); + + foreach ($errors as $e) { + $result []= "Missing commit for metadata entry {".$e['hash']."}"; + } + } + + // ======== [7] missing branch-head in commits ========== + { + $errors = $this->sql_query_assoc_prep("SELECT branches.*, commits.id AS cid FROM branches LEFT JOIN commits ON branches.head = commits.hash WHERE cid IS NULL", []); + + foreach ($errors as $e) { + $result []= "Missing head-commit {".$e['head']."} for branch ".$e['id'].": '".$e['name']."'"; + } + } + + return $result; + } } \ No newline at end of file diff --git a/www/extern/egg/ExtendedGitGraph2.php b/www/extern/egg/ExtendedGitGraph2.php index c41225f..065d8b9 100644 --- a/www/extern/egg/ExtendedGitGraph2.php +++ b/www/extern/egg/ExtendedGitGraph2.php @@ -177,4 +177,18 @@ class ExtendedGitGraph2 implements ILogger { foreach($this->logger as $lgr) $lgr->proclog($text); } + + /** + * @return string[] + */ + public function checkDatabase(): array + { + $this->db->open(); + + $r = $this->db->checkDatabase(); + + $this->db->close(); + + return $r; + } } \ No newline at end of file diff --git a/www/extern/egg/db_init.sql b/www/extern/egg/db_init.sql index 2652478..2e35898 100644 --- a/www/extern/egg/db_init.sql +++ b/www/extern/egg/db_init.sql @@ -70,3 +70,14 @@ CREATE VIEW allbranches AS FROM branches LEFT JOIN repositories ON branches.repo_id = repositories.id; +/*----SPLIT----*/ + +CREATE INDEX "branches_repo_id" ON "branches" ("repo_id"); + +/*----SPLIT----*/ + +CREATE INDEX "commits_branch_id" ON "commits" ("branch_id"); + +/*----SPLIT----*/ + +CREATE INDEX "commits_hash" ON "commits" ("hash"); diff --git a/www/internals/modules/mikeschergitgraph.php b/www/internals/modules/mikeschergitgraph.php index cf2c66f..62f2e63 100644 --- a/www/internals/modules/mikeschergitgraph.php +++ b/www/internals/modules/mikeschergitgraph.php @@ -49,4 +49,12 @@ class MikescherGitGraph implements IWebsiteModule return ['result'=>'ok', 'message' => '']; } + + /** + * @return string[] + */ + public function checkDatabaseConsistency(): array + { + return $this->extgitgraph->checkDatabase(); + } } \ No newline at end of file diff --git a/www/internals/modules/selftest.php b/www/internals/modules/selftest.php index ddbd34f..42cd8c2 100644 --- a/www/internals/modules/selftest.php +++ b/www/internals/modules/selftest.php @@ -30,7 +30,8 @@ class SelfTest implements IWebsiteModule 'modules::updateslog' => 'Program Updates (data)', 'modules::webapps' => 'Webapps (data)', 'modules::highscores' => 'Highscores (data)', - 'backend::git' => 'Git Repository' + 'egg::db-check' => 'ExtendedGitGraph (db-check)', + 'backend::git' => 'Git Repository', ]; private $methods = []; @@ -118,6 +119,8 @@ class SelfTest implements IWebsiteModule $this->addCheckConsistency("modules::webapps::webapps-check-consistency", function(){ return Website::inst()->modules->WebApps(); }); $this->addCheckConsistency("modules::highscores::highscores-check-consistency", function(){ return Website::inst()->modules->Highscores(); }); + $this->addLambdaStatus("egg::db-check::check-db-consistency", function(){ return Website::inst()->modules->ExtendedGitGraph()->checkDatabaseConsistency(); }); + $ajaxsecret = Website::inst()->config['ajax_secret']; $this->addMethodPathResponse( "api::default::base-test-2", 200, '{}', '/api/test'); @@ -362,6 +365,77 @@ class SelfTest implements IWebsiteModule ]; } + private function addLambdaStatus(string $name, Closure $fun) + { + $this->methods []= + [ + 'name' => $name, + 'func' => function() use ($name, $fun) + { + try + { + $result = $fun(); + + if (empty($result)) return + [ + 'result' => self::STATUS_OK, + 'message' => 'OK', + 'long' => 'Okay', + 'exception' => null, + ]; + + if (isset($result['result']) && isset($result['message'])) { + if ($result['result'] === 'err') return + [ + 'result' => self::STATUS_ERROR, + 'message' => $result['message'], + 'long' => isset($result['long']) ? $result['long'] : null, + 'exception' => null, + ]; + + if ($result['result'] === 'warn') return + [ + 'result' => self::STATUS_WARN, + 'message' => $result['message'], + 'long' => isset($result['long']) ? $result['long'] : null, + 'exception' => null, + ]; + + if ($result['result'] === 'ok') return + [ + 'result' => self::STATUS_OK, + 'message' => 'OK', + 'long' => isset($result['long']) ? $result['long'] : null, + 'exception' => null, + ]; + } + + if (is_array($result) && is_string($result[0])) { + return + [ + 'result' => self::STATUS_ERROR, + 'message' => count($result) . " errors occured", + 'long' => implode("\n", $result), + 'exception' => null, + ]; + } + + throw new Exception("Unknown result: " . print_r($result, true)); + } + catch (Throwable $e) + { + return + [ + 'result' => self::STATUS_ERROR, + 'message' => str_max_len($e->getMessage(), 48), + 'long' => formatException($e), + 'exception' => $e, + ]; + } + } + ]; + } + private function addMethodPathResponse(string $name, int $status, string $json_expected, string $path) { $this->methods []= diff --git a/www/pages/admin.php b/www/pages/admin.php index 9f714ac..7e9d368 100644 --- a/www/pages/admin.php +++ b/www/pages/admin.php @@ -57,7 +57,7 @@ $connected = true; try { $SITE->modules->Database(); } catch (Exception $e) { $c
Full Selftest -
+
modules->SelfTest()->listMethodGroups() as $group): $stid++; ?>
@@ -84,7 +84,7 @@ $connected = true; try { $SITE->modules->Database(); } catch (Exception $e) { $c $me = $real_ip == $self_ip ?> -
+
Registered IP:
Current IP:
@@ -112,7 +112,7 @@ $connected = true; try { $SITE->modules->Database(); } catch (Exception $e) { $c
-
+
Total users: modules->AlephNoteStatistics()->getTotalUserCount(); ?>
Users on latest version: modules->AlephNoteStatistics()->getUserCountFromLastVersion(); ?>
Active users: modules->AlephNoteStatistics()->getActiveUserCount(32); ?>
@@ -133,7 +133,7 @@ $connected = true; try { $SITE->modules->Database(); } catch (Exception $e) { $c
Statics
-
+
Blog entries: modules->Blog()->listAll()); ?>
Book entries: modules->Books()->listAll()); ?>
Euler entries: modules->Euler()->listAll()); ?>
@@ -195,7 +195,7 @@ $connected = true; try { $SITE->modules->Database(); } catch (Exception $e) { $c
Configuration
-
+
config as $key => $value) { @@ -212,7 +212,7 @@ $connected = true; try { $SITE->modules->Database(); } catch (Exception $e) { $c
Configuration['extendedgitgraph']
-
+
config['extendedgitgraph'] as $key => $value) { diff --git a/www/shell/extendedgitgraph_refresh.php b/www/shell/extendedgitgraph_refresh.php index 64b037d..6f6f249 100644 --- a/www/shell/extendedgitgraph_refresh.php +++ b/www/shell/extendedgitgraph_refresh.php @@ -34,11 +34,25 @@ if (!$r1) } echo "\n"; -echo "============================= Start Update-Cache =============================\n"; +echo "============================= Check Consistency =============================\n"; echo "\n"; -$r2 = $egg->updateCache(); -if (!$r2) +$r2 = $egg->checkDatabaseConsistency(); +if (count($r2) > 0) +{ + echo "EGG::updateCache failed."; + foreach ($r2 as $msg) { + echo " > $msg"; + } + exit(99); +} + +echo "\n"; +echo "============================= Update Cache =============================\n"; +echo "\n"; + +$r3 = $egg->updateCache(); +if (!$r3) { echo "EGG::updateCache failed."; exit(99);