Mikescher/www.mikescher.com
1
0
Fork 0
Code Actions

Add check-db to egg and save parents as json-objects

Browse Source
This commit is contained in:
Mike Schwörer 2022-10-20 14:56:27 +02:00
parent 16051928c1
commit 805627be17
Signed by: Mikescher
GPG Key ID: D3C7172E0A70F8CF
12 changed files with 234 additions and 16 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
data/css_compress/compress.py
View File

@ -10,7 +10,7 @@ import shutil
def findnext(strdata, start, searchchr):
depth = 0
for i in range(start, len(strdata)):
if strdata[i] == searchchr: return i;
if strdata[i] == searchchr: return i
def findclose(strdata, start):

9
www/commands/extendedgitgraph_refresh.php
View File

@ -14,8 +14,15 @@ if (!$r1)
echo 'EGG::update failed.';
}
$r2 = $SITE->modules->ExtendedGitGraph()->updateCache();
$r2 = $SITE->modules->ExtendedGitGraph()->checkDatabaseConsistency();
if (!$r2)
{
http_response_code(500);
echo 'EGG::checkDatabaseConsistency failed.';
}
$r3 = $SITE->modules->ExtendedGitGraph()->updateCache();
if (!$r3)
{
http_response_code(500);
echo 'EGG::updateCache failed.';

4
www/data/css/styles.css
View File

@ -869,6 +869,10 @@ html, body {
min-width: 200px;
}
.kvl_250 div span:first-child {
min-width: 250px;
}
.kvl_300 div span:first-child {
min-width: 300px;
}

1
www/data/css/styles.min.css vendored
View File

@ -167,6 +167,7 @@ html,body{margin:0;padding:0;height:100%}
.keyvaluelist div span:first-child{font-weight:bold;min-width:500px;align-self:start}
.kvl_100 div span:first-child{min-width:100px}
.kvl_200 div span:first-child{min-width:200px}
.kvl_250 div span:first-child{min-width:250px}
.kvl_300 div span:first-child{min-width:300px}
.selftest_parent{width:800px;max-width:100%}
.selftest_outputchild{border:1px solid #888;background:#f8f8f8;color:#000;font-family:Consolas,Monaco,"Courier New",Menlo,monospace;white-space:pre;max-height:600px;overflow-x:auto;overflow-y:auto}

1
www/data/css/styles_admin.scss
View File

@ -39,6 +39,7 @@
.kvl_100 div span:first-child { min-width: 100px; }
.kvl_200 div span:first-child { min-width: 200px; }
.kvl_250 div span:first-child { min-width: 250px; }
.kvl_300 div span:first-child { min-width: 300px; }
.selftest_parent {

92
www/extern/egg/EGGDatabase.php vendored
View File

@ -258,7 +258,8 @@ class EGGDatabase
foreach ($commits as $commit)
{
$strparents = implode(";", $commit->Parents);
$jparents = "[]";
if (count($commit->Parents) > 0) $jparents = '["'.implode('","', $commit->Parents).'"]';
$this->sql_exec_pre_prep($stmtAddCommit,
[
@ -275,7 +276,7 @@ class EGGDatabase
[":cm", $commit->CommitterEmail, PDO::PARAM_STR],
[":msg", $commit->Message, PDO::PARAM_STR],
[":dat", $commit->Date, PDO::PARAM_STR],
[":prt", $strparents, PDO::PARAM_STR],
[":prt", $jparents, PDO::PARAM_STR],
]);
$dbid = $this->sql_query_assoc_pre_prep($stmtGetID,
@ -527,7 +528,7 @@ class EGGDatabase
$c->CommitterEmail = $row['committer_email'];
$c->Message = $row['message'];
$c->Date = $row['date'];
$c->Parents = array_filter(explode(';', $row['parent_commits']), fn($p) => $p !== '');
$c->Parents = json_decode($row['parent_commits']);
$r []= $c;
}
return $r;
@ -555,9 +556,92 @@ class EGGDatabase
$c->CommitterEmail = $row['committer_email'];
$c->Message = $row['message'];
$c->Date = $row['date'];
$c->Parents = array_filter(explode(';', $row['parent_commits']), fn($p) => $p !== '');
$c->Parents = json_decode($row['parent_commits']);
$r []= $c;
}
return $r;
}
public function checkDatabase(): array{
$result = [];
// ======== [1] invalid json in parent_commits ==========
{
$errors = $this->sql_query_assoc_prep("SELECT * FROM metadata WHERE NOT json_valid(parent_commits)", []);
foreach ($errors as $e) {
$result []= "Found commit-metadata entry {".$e['hash']."} with invalid json in 'parent_commits'";
}
}
// ======== [2] metadata with missing parent ==========
{
$errors = $this->sql_query_assoc_prep("
SELECT md1.*, md2.hash as check_hash FROM (
SELECT
md1.*, mdp.value AS parent
FROM
metadata AS md1, json_each(md1.parent_commits) AS mdp
) AS md1
LEFT JOIN metadata AS md2 ON md1.parent = md2.hash
WHERE md1.parent != '' AND check_hash IS NULL", []);
foreach ($errors as $e) {
$result []= "Found commit-metadata entry {".$e['hash']."} with an reference to a not-existing parent '".$e['parent']."'";
}
}
// ======== [3] repositories without branches ==========
{
$errors = $this->sql_query_assoc_prep("SELECT repositories.*, (SELECT COUNT(*) FROM branches WHERE repositories.id = branches.repo_id) AS branch_count FROM repositories WHERE branch_count = 0", []);
foreach ($errors as $e) {
$result []= "Found repository [".$e['id']."]'".$e['name']."' without any branches";
}
}
// ======== [4] branches without commits ==========
{
$errors = $this->sql_query_assoc_prep("SELECT branches.*, (SELECT COUNT(*) FROM commits WHERE branches.id = commits.branch_id) AS commit_count FROM branches WHERE commit_count = 0", []);
foreach ($errors as $e) {
$result []= "Found branch [".$e['id']."]'".$e['name']."' without any commits";
}
}
// ======== [5] commits with missing metadata ==========
{
$errors = $this->sql_query_assoc_prep("SELECT commits.*, metadata.hash AS mdh FROM commits LEFT JOIN metadata ON commits.hash = metadata.hash WHERE mdh IS NULL", []);
foreach ($errors as $e) {
$result []= "Missing metadata for commit ".$e['id']." | {".$e['hash']."}";
}
}
// ======== [6] metadata with missing commits ==========
{
$errors = $this->sql_query_assoc_prep("SELECT metadata.*, commits.hash AS ch FROM metadata LEFT JOIN commits ON commits.hash = metadata.hash WHERE ch IS NULL", []);
foreach ($errors as $e) {
$result []= "Missing commit for metadata entry {".$e['hash']."}";
}
}
// ======== [7] missing branch-head in commits ==========
{
$errors = $this->sql_query_assoc_prep("SELECT branches.*, commits.id AS cid FROM branches LEFT JOIN commits ON branches.head = commits.hash WHERE cid IS NULL", []);
foreach ($errors as $e) {
$result []= "Missing head-commit {".$e['head']."} for branch ".$e['id'].": '".$e['name']."'";
}
}
return $result;
}
}

14
www/extern/egg/ExtendedGitGraph2.php vendored
View File

@ -177,4 +177,18 @@ class ExtendedGitGraph2 implements ILogger
{
foreach($this->logger as $lgr) $lgr->proclog($text);
}
/**
* @return string[]
*/
public function checkDatabase(): array
{
$this->db->open();
$r = $this->db->checkDatabase();
$this->db->close();
return $r;
}
}

11
www/extern/egg/db_init.sql vendored
View File

@ -70,3 +70,14 @@ CREATE VIEW allbranches AS
FROM branches
LEFT JOIN repositories ON branches.repo_id = repositories.id;
/*----SPLIT----*/
CREATE INDEX "branches_repo_id" ON "branches" ("repo_id");
/*----SPLIT----*/
CREATE INDEX "commits_branch_id" ON "commits" ("branch_id");
/*----SPLIT----*/
CREATE INDEX "commits_hash" ON "commits" ("hash");

8
www/internals/modules/mikeschergitgraph.php
View File

@ -49,4 +49,12 @@ class MikescherGitGraph implements IWebsiteModule
return ['result'=>'ok', 'message' => ''];
}
/**
* @return string[]
*/
public function checkDatabaseConsistency(): array
{
return $this->extgitgraph->checkDatabase();
}
}

76
www/internals/modules/selftest.php
View File

@ -30,7 +30,8 @@ class SelfTest implements IWebsiteModule
'modules::updateslog' => 'Program Updates (data)',
'modules::webapps' => 'Webapps (data)',
'modules::highscores' => 'Highscores (data)',
'backend::git' => 'Git Repository'
'egg::db-check' => 'ExtendedGitGraph (db-check)',
'backend::git' => 'Git Repository',
];
private $methods = [];
@ -118,6 +119,8 @@ class SelfTest implements IWebsiteModule
$this->addCheckConsistency("modules::webapps::webapps-check-consistency", function(){ return Website::inst()->modules->WebApps(); });
$this->addCheckConsistency("modules::highscores::highscores-check-consistency", function(){ return Website::inst()->modules->Highscores(); });
$this->addLambdaStatus("egg::db-check::check-db-consistency", function(){ return Website::inst()->modules->ExtendedGitGraph()->checkDatabaseConsistency(); });
$ajaxsecret = Website::inst()->config['ajax_secret'];
$this->addMethodPathResponse( "api::default::base-test-2", 200, '{}', '/api/test');
@ -362,6 +365,77 @@ class SelfTest implements IWebsiteModule
];
}
private function addLambdaStatus(string $name, Closure $fun)
{
$this->methods []=
[
'name' => $name,
'func' => function() use ($name, $fun)
{
try
{
$result = $fun();
if (empty($result)) return
[
'result' => self::STATUS_OK,
'message' => 'OK',
'long' => 'Okay',
'exception' => null,
];
if (isset($result['result']) && isset($result['message'])) {
if ($result['result'] === 'err') return
[
'result' => self::STATUS_ERROR,
'message' => $result['message'],
'long' => isset($result['long']) ? $result['long'] : null,
'exception' => null,
];
if ($result['result'] === 'warn') return
[
'result' => self::STATUS_WARN,
'message' => $result['message'],
'long' => isset($result['long']) ? $result['long'] : null,
'exception' => null,
];
if ($result['result'] === 'ok') return
[
'result' => self::STATUS_OK,
'message' => 'OK',
'long' => isset($result['long']) ? $result['long'] : null,
'exception' => null,
];
}
if (is_array($result) && is_string($result[0])) {
return
[
'result' => self::STATUS_ERROR,
'message' => count($result) . " errors occured",
'long' => implode("\n", $result),
'exception' => null,
];
}
throw new Exception("Unknown result: " . print_r($result, true));
}
catch (Throwable $e)
{
return
[
'result' => self::STATUS_ERROR,
'message' => str_max_len($e->getMessage(), 48),
'long' => formatException($e),
'exception' => $e,
];
}
}
];
}
private function addMethodPathResponse(string $name, int $status, string $json_expected, string $path)
{
$this->methods []=

12
www/pages/admin.php
View File

@ -57,7 +57,7 @@ $connected = true; try { $SITE->modules->Database(); } catch (Exception $e) { $c
<div class="bc_data">
<a id="btnFullSelftest" class="button" href="#">Full Selftest</a>
<div class="keyvaluelist kvl_200 selftest_parent <?= ($SITE->isProd() ? 'selftest_parallel' : 'selftest_sequential') ?>">
<div class="keyvaluelist kvl_250 selftest_parent <?= ($SITE->isProd() ? 'selftest_parallel' : 'selftest_sequential') ?>">
<?php $stid=1000; foreach ($SITE->modules->SelfTest()->listMethodGroups() as $group): $stid++; ?>
<div class="selftest_tabchild" onclick="showSelfTestOutput('#selftest_tab_<?= $stid; ?>', '#selftest_out_<?= $stid; ?>')">
<span><?= $group['name']; ?></span>
@ -84,7 +84,7 @@ $connected = true; try { $SITE->modules->Database(); } catch (Exception $e) { $c
$me = $real_ip == $self_ip
?>
<div class="bc_data keyvaluelist kvl_200">
<div class="bc_data keyvaluelist kvl_250">
<div><span>Registered IP:</span> <span><?php echo $self_ip; ?></span></div>
<div><span>Current IP:</span> <span><?php echo $real_ip; ?></span></div>
</div>
@ -112,7 +112,7 @@ $connected = true; try { $SITE->modules->Database(); } catch (Exception $e) { $c
<?php if ($connected): ?>
<div class="bc_data">
<div class="keyvaluelist kvl_200">
<div class="keyvaluelist kvl_250">
<div><span>Total users:</span> <span><?php echo $SITE->modules->AlephNoteStatistics()->getTotalUserCount(); ?></span></div>
<div><span>Users on latest version:</span> <span><?php echo $SITE->modules->AlephNoteStatistics()->getUserCountFromLastVersion(); ?></span></div>
<div><span>Active users:</span> <span><?php echo $SITE->modules->AlephNoteStatistics()->getActiveUserCount(32); ?></span></div>
@ -133,7 +133,7 @@ $connected = true; try { $SITE->modules->Database(); } catch (Exception $e) { $c
<div class="boxedcontent">
<div class="bc_header">Statics</div>
<div class="bc_data keyvaluelist kvl_200">
<div class="bc_data keyvaluelist kvl_250">
<div><span>Blog entries:</span> <span><?php echo count($SITE->modules->Blog()->listAll()); ?></span></div>
<div><span>Book entries:</span> <span><?php echo count($SITE->modules->Books()->listAll()); ?></span></div>
<div><span>Euler entries:</span> <span><?php echo count($SITE->modules->Euler()->listAll()); ?></span></div>
@ -195,7 +195,7 @@ $connected = true; try { $SITE->modules->Database(); } catch (Exception $e) { $c
<div class="boxedcontent">
<div class="bc_header">Configuration</div>
<div class="bc_data keyvaluelist kvl_200">
<div class="bc_data keyvaluelist kvl_250">
<?php
foreach ($SITE->config as $key => $value)
{
@ -212,7 +212,7 @@ $connected = true; try { $SITE->modules->Database(); } catch (Exception $e) { $c
<div class="boxedcontent">
<div class="bc_header">Configuration['extendedgitgraph']</div>
<div class="bc_data keyvaluelist kvl_200">
<div class="bc_data keyvaluelist kvl_250">
<?php
foreach ($SITE->config['extendedgitgraph'] as $key => $value)
{

20
www/shell/extendedgitgraph_refresh.php
View File

@ -34,11 +34,25 @@ if (!$r1)
}
echo "\n";
echo "============================= Start Update-Cache =============================\n";
echo "============================= Check Consistency =============================\n";
echo "\n";
$r2 = $egg->updateCache();
if (!$r2)
$r2 = $egg->checkDatabaseConsistency();
if (count($r2) > 0)
{
echo "EGG::updateCache failed.";
foreach ($r2 as $msg) {
echo " > $msg";
}
exit(99);
}
echo "\n";
echo "============================= Update Cache =============================\n";
echo "\n";
$r3 = $egg->updateCache();
if (!$r3)
{
echo "EGG::updateCache failed.";
exit(99);