Add check-db to egg and save parents as json-objects

data/css_compress/compress.py

@@ -10,7 +10,7 @@ import shutil
 def findnext(strdata, start, searchchr):
     depth = 0
     for i in range(start, len(strdata)):
-        if strdata[i] == searchchr: return i;
+        if strdata[i] == searchchr: return i
 
 
 def findclose(strdata, start):

www/commands/extendedgitgraph_refresh.php

@@ -14,8 +14,15 @@ if (!$r1)
 	echo 'EGG::update failed.';
 }
 
-$r2 = $SITE->modules->ExtendedGitGraph()->updateCache();
+$r2 = $SITE->modules->ExtendedGitGraph()->checkDatabaseConsistency();
 if (!$r2)
+{
+	http_response_code(500);
+	echo 'EGG::checkDatabaseConsistency failed.';
+}
+
+$r3 = $SITE->modules->ExtendedGitGraph()->updateCache();
+if (!$r3)
 {
 	http_response_code(500);
 	echo 'EGG::updateCache failed.';

www/data/css/styles.css

@@ -869,6 +869,10 @@ html, body {
   min-width: 200px;
 }
 
+.kvl_250 div span:first-child {
+  min-width: 250px;
+}
+
 .kvl_300 div span:first-child {
   min-width: 300px;
 }

www/data/css/styles.min.css

@@ -167,6 +167,7 @@ html,body{margin:0;padding:0;height:100%}
 .keyvaluelist div span:first-child{font-weight:bold;min-width:500px;align-self:start}
 .kvl_100 div span:first-child{min-width:100px}
 .kvl_200 div span:first-child{min-width:200px}
+.kvl_250 div span:first-child{min-width:250px}
 .kvl_300 div span:first-child{min-width:300px}
 .selftest_parent{width:800px;max-width:100%}
 .selftest_outputchild{border:1px solid #888;background:#f8f8f8;color:#000;font-family:Consolas,Monaco,"Courier New",Menlo,monospace;white-space:pre;max-height:600px;overflow-x:auto;overflow-y:auto}

www/data/css/styles_admin.scss

@@ -39,6 +39,7 @@
 
 .kvl_100 div span:first-child { min-width: 100px; }
 .kvl_200 div span:first-child { min-width: 200px; }
+.kvl_250 div span:first-child { min-width: 250px; }
 .kvl_300 div span:first-child { min-width: 300px; }
 
 .selftest_parent {

www/extern/egg/EGGDatabase.php

@@ -258,7 +258,8 @@ class EGGDatabase
 
 		foreach ($commits as $commit)
 		{
-			$strparents = implode(";", $commit->Parents);
+			$jparents = "[]";
+			if (count($commit->Parents) > 0) $jparents = '["'.implode('","', $commit->Parents).'"]';
 
 			$this->sql_exec_pre_prep($stmtAddCommit,
 				[
@@ -275,7 +276,7 @@ class EGGDatabase
 					[":cm",   $commit->CommitterEmail, PDO::PARAM_STR],
 					[":msg",  $commit->Message,        PDO::PARAM_STR],
 					[":dat",  $commit->Date,           PDO::PARAM_STR],
-					[":prt",  $strparents,             PDO::PARAM_STR],
+					[":prt",  $jparents,               PDO::PARAM_STR],
 				]);
 
 			$dbid = $this->sql_query_assoc_pre_prep($stmtGetID,
@@ -527,7 +528,7 @@ class EGGDatabase
 			$c->CommitterEmail = $row['committer_email'];
 			$c->Message        = $row['message'];
 			$c->Date           = $row['date'];
-			$c->Parents        = array_filter(explode(';', $row['parent_commits']), fn($p) => $p !== '');
+			$c->Parents        = json_decode($row['parent_commits']);
 			$r []= $c;
 		}
 		return $r;
@@ -555,9 +556,92 @@ class EGGDatabase
 			$c->CommitterEmail = $row['committer_email'];
 			$c->Message        = $row['message'];
 			$c->Date           = $row['date'];
-			$c->Parents        = array_filter(explode(';', $row['parent_commits']), fn($p) => $p !== '');
+			$c->Parents        = json_decode($row['parent_commits']);
 			$r []= $c;
 		}
 		return $r;
 	}
+
+	public function checkDatabase(): array{
+
+		$result = [];
+
+		// ======== [1] invalid json in parent_commits ==========
+		{
+			$errors = $this->sql_query_assoc_prep("SELECT * FROM metadata WHERE NOT json_valid(parent_commits)", []);
+
+			foreach ($errors as $e) {
+				$result []= "Found commit-metadata entry {".$e['hash']."} with invalid json in 'parent_commits'";
+			}
+		}
+
+		// ======== [2] metadata with missing parent ==========
+		{
+			$errors = $this->sql_query_assoc_prep("
+SELECT md1.*, md2.hash as check_hash FROM (
+	
+	SELECT 
+		md1.*, mdp.value AS parent 
+	FROM 
+		metadata AS md1, json_each(md1.parent_commits) AS mdp 
+
+) AS md1
+	
+LEFT JOIN metadata AS md2 ON md1.parent = md2.hash
+
+WHERE md1.parent != '' AND check_hash IS NULL", []);
+
+			foreach ($errors as $e) {
+				$result []= "Found commit-metadata entry {".$e['hash']."} with an reference to a not-existing parent '".$e['parent']."'";
+			}
+		}
+
+		// ======== [3] repositories without branches ==========
+		{
+			$errors = $this->sql_query_assoc_prep("SELECT repositories.*, (SELECT COUNT(*) FROM branches WHERE repositories.id = branches.repo_id) AS branch_count FROM repositories WHERE branch_count = 0", []);
+
+			foreach ($errors as $e) {
+				$result []= "Found repository [".$e['id']."]'".$e['name']."' without any branches";
+			}
+		}
+
+
+		// ======== [4] branches without commits ==========
+		{
+			$errors = $this->sql_query_assoc_prep("SELECT branches.*, (SELECT COUNT(*) FROM commits WHERE branches.id = commits.branch_id) AS commit_count FROM branches WHERE commit_count = 0", []);
+
+			foreach ($errors as $e) {
+				$result []= "Found branch [".$e['id']."]'".$e['name']."' without any commits";
+			}
+		}
+
+		// ======== [5] commits with missing metadata ==========
+		{
+			$errors = $this->sql_query_assoc_prep("SELECT commits.*, metadata.hash AS mdh FROM commits LEFT JOIN metadata ON commits.hash = metadata.hash WHERE mdh IS NULL", []);
+
+			foreach ($errors as $e) {
+				$result []= "Missing metadata for commit ".$e['id']." | {".$e['hash']."}";
+			}
+		}
+
+		// ======== [6] metadata with missing commits ==========
+		{
+			$errors = $this->sql_query_assoc_prep("SELECT metadata.*, commits.hash AS ch FROM metadata LEFT JOIN commits ON commits.hash = metadata.hash WHERE ch IS NULL", []);
+
+			foreach ($errors as $e) {
+				$result []= "Missing commit for metadata entry {".$e['hash']."}";
+			}
+		}
+
+		// ======== [7] missing branch-head in commits ==========
+		{
+			$errors = $this->sql_query_assoc_prep("SELECT branches.*, commits.id AS cid FROM branches LEFT JOIN commits ON branches.head = commits.hash WHERE cid IS NULL", []);
+
+			foreach ($errors as $e) {
+				$result []= "Missing head-commit {".$e['head']."} for branch ".$e['id'].": '".$e['name']."'";
+			}
+		}
+
+		return $result;
+	}
 }

www/extern/egg/ExtendedGitGraph2.php

@@ -177,4 +177,18 @@ class ExtendedGitGraph2 implements ILogger
 	{
 		foreach($this->logger as $lgr) $lgr->proclog($text);
 	}
+
+	/**
+	 * @return string[]
+	 */
+	public function checkDatabase(): array
+	{
+		$this->db->open();
+
+		$r = $this->db->checkDatabase();
+
+		$this->db->close();
+
+		return $r;
+	}
 }

www/extern/egg/db_init.sql

@@ -70,3 +70,14 @@ CREATE VIEW allbranches AS
 	FROM branches
 	LEFT JOIN repositories ON branches.repo_id  = repositories.id;
 
+/*----SPLIT----*/
+
+CREATE INDEX "branches_repo_id" ON "branches" ("repo_id");
+
+/*----SPLIT----*/
+
+CREATE INDEX "commits_branch_id" ON "commits" ("branch_id");
+
+/*----SPLIT----*/
+
+CREATE INDEX "commits_hash" ON "commits" ("hash");

www/internals/modules/mikeschergitgraph.php

@@ -49,4 +49,12 @@ class MikescherGitGraph implements IWebsiteModule
 
 		return ['result'=>'ok', 'message' => ''];
 	}
+
+	/**
+	 * @return string[]
+	 */
+	public function checkDatabaseConsistency(): array
+	{
+		return $this->extgitgraph->checkDatabase();
+	}
 }

www/internals/modules/selftest.php

@@ -30,7 +30,8 @@ class SelfTest implements IWebsiteModule
 		'modules::updateslog'   => 'Program Updates (data)',
 		'modules::webapps'      => 'Webapps (data)',
 		'modules::highscores'   => 'Highscores (data)',
-		'backend::git'          => 'Git Repository'
+		'egg::db-check'         => 'ExtendedGitGraph (db-check)',
+		'backend::git'          => 'Git Repository',
 	];
 
 	private $methods = [];
@@ -118,6 +119,8 @@ class SelfTest implements IWebsiteModule
 		$this->addCheckConsistency("modules::webapps::webapps-check-consistency",           function(){ return Website::inst()->modules->WebApps(); });
 		$this->addCheckConsistency("modules::highscores::highscores-check-consistency",     function(){ return Website::inst()->modules->Highscores(); });
 
+		$this->addLambdaStatus("egg::db-check::check-db-consistency", function(){ return Website::inst()->modules->ExtendedGitGraph()->checkDatabaseConsistency(); });
+
 		$ajaxsecret = Website::inst()->config['ajax_secret'];
 
 		$this->addMethodPathResponse(   "api::default::base-test-2",   200,  '{}', '/api/test');
@@ -362,6 +365,77 @@ class SelfTest implements IWebsiteModule
 		];
 	}
 
+	private function addLambdaStatus(string $name, Closure $fun)
+	{
+		$this->methods []=
+			[
+				'name' => $name,
+				'func' => function() use ($name, $fun)
+				{
+					try
+					{
+						$result = $fun();
+
+						if (empty($result)) return
+						[
+							'result' => self::STATUS_OK,
+							'message' => 'OK',
+							'long' => 'Okay',
+							'exception' => null,
+						];
+
+						if (isset($result['result']) && isset($result['message'])) {
+							if ($result['result'] === 'err') return
+							[
+								'result' => self::STATUS_ERROR,
+								'message' => $result['message'],
+								'long' => isset($result['long']) ? $result['long'] : null,
+								'exception' => null,
+							];
+
+							if ($result['result'] === 'warn') return
+							[
+								'result' => self::STATUS_WARN,
+								'message' => $result['message'],
+								'long' => isset($result['long']) ? $result['long'] : null,
+								'exception' => null,
+							];
+
+							if ($result['result'] === 'ok') return
+							[
+								'result' => self::STATUS_OK,
+								'message' => 'OK',
+								'long' => isset($result['long']) ? $result['long'] : null,
+								'exception' => null,
+							];
+						}
+
+						if (is_array($result) && is_string($result[0])) {
+							return
+							[
+								'result' => self::STATUS_ERROR,
+								'message' => count($result) . " errors occured",
+								'long' => implode("\n", $result),
+								'exception' => null,
+							];
+						}
+
+						throw new Exception("Unknown result: " . print_r($result, true));
+					}
+					catch (Throwable $e)
+					{
+						return
+							[
+								'result' => self::STATUS_ERROR,
+								'message' => str_max_len($e->getMessage(), 48),
+								'long' => formatException($e),
+								'exception' => $e,
+							];
+					}
+				}
+			];
+	}
+
 	private function addMethodPathResponse(string $name, int $status, string $json_expected, string $path)
 	{
 		$this->methods []=

www/pages/admin.php

@@ -57,7 +57,7 @@ $connected = true; try { $SITE->modules->Database(); } catch (Exception $e) { $c
         <div class="bc_data">
             <a id="btnFullSelftest" class="button" href="#">Full Selftest</a>
 
-            <div class="keyvaluelist kvl_200 selftest_parent <?= ($SITE->isProd() ? 'selftest_parallel' : 'selftest_sequential') ?>">
+            <div class="keyvaluelist kvl_250 selftest_parent <?= ($SITE->isProd() ? 'selftest_parallel' : 'selftest_sequential') ?>">
                 <?php $stid=1000; foreach ($SITE->modules->SelfTest()->listMethodGroups() as $group): $stid++; ?>
                     <div class="selftest_tabchild" onclick="showSelfTestOutput('#selftest_tab_<?= $stid; ?>', '#selftest_out_<?= $stid; ?>')">
                         <span><?= $group['name']; ?></span>
@@ -84,7 +84,7 @@ $connected = true; try { $SITE->modules->Database(); } catch (Exception $e) { $c
 		$me = $real_ip == $self_ip
 		?>
 
-        <div class="bc_data keyvaluelist kvl_200">
+        <div class="bc_data keyvaluelist kvl_250">
             <div><span>Registered IP:</span> <span><?php echo $self_ip; ?></span></div>
             <div><span>Current IP:</span> <span><?php echo $real_ip; ?></span></div>
         </div>
@@ -112,7 +112,7 @@ $connected = true; try { $SITE->modules->Database(); } catch (Exception $e) { $c
 
 		<?php if ($connected): ?>
             <div class="bc_data">
-                <div class="keyvaluelist kvl_200">
+                <div class="keyvaluelist kvl_250">
                     <div><span>Total users:</span> <span><?php echo $SITE->modules->AlephNoteStatistics()->getTotalUserCount(); ?></span></div>
                     <div><span>Users on latest version:</span> <span><?php echo $SITE->modules->AlephNoteStatistics()->getUserCountFromLastVersion(); ?></span></div>
                     <div><span>Active users:</span> <span><?php echo $SITE->modules->AlephNoteStatistics()->getActiveUserCount(32); ?></span></div>
@@ -133,7 +133,7 @@ $connected = true; try { $SITE->modules->Database(); } catch (Exception $e) { $c
     <div class="boxedcontent">
         <div class="bc_header">Statics</div>
 
-        <div class="bc_data keyvaluelist kvl_200">
+        <div class="bc_data keyvaluelist kvl_250">
             <div><span>Blog entries:</span> <span><?php    echo count($SITE->modules->Blog()->listAll()); ?></span></div>
             <div><span>Book entries:</span> <span><?php    echo count($SITE->modules->Books()->listAll()); ?></span></div>
             <div><span>Euler entries:</span> <span><?php   echo count($SITE->modules->Euler()->listAll()); ?></span></div>
@@ -195,7 +195,7 @@ $connected = true; try { $SITE->modules->Database(); } catch (Exception $e) { $c
 
     <div class="boxedcontent">
         <div class="bc_header">Configuration</div>
-        <div class="bc_data keyvaluelist kvl_200">
+        <div class="bc_data keyvaluelist kvl_250">
 			<?php
 			foreach ($SITE->config as $key => $value)
 			{
@@ -212,7 +212,7 @@ $connected = true; try { $SITE->modules->Database(); } catch (Exception $e) { $c
 
     <div class="boxedcontent">
         <div class="bc_header">Configuration['extendedgitgraph']</div>
-        <div class="bc_data keyvaluelist kvl_200">
+        <div class="bc_data keyvaluelist kvl_250">
 			<?php
 			foreach ($SITE->config['extendedgitgraph'] as $key => $value)
 			{

www/shell/extendedgitgraph_refresh.php

@@ -34,11 +34,25 @@ if (!$r1)
 }
 
 echo "\n";
-echo "============================= Start Update-Cache =============================\n";
+echo "============================= Check Consistency =============================\n";
 echo "\n";
 
-$r2 = $egg->updateCache();
+$r2 = $egg->checkDatabaseConsistency();
-if (!$r2)
+if (count($r2) > 0)
+{
+	echo "EGG::updateCache failed.";
+    foreach ($r2 as $msg) {
+		echo "    > $msg";
+    }
+	exit(99);
+}
+
+echo "\n";
+echo "============================= Update Cache =============================\n";
+echo "\n";
+
+$r3 = $egg->updateCache();
+if (!$r3)
 {
 	echo "EGG::updateCache failed.";
 	exit(99);