From ba96d8f72631c6ff07afd4f406d48a63e6c30bc3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mike=20Schw=C3=B6rer=20=28Macbook=29?= Date: Sat, 28 Dec 2019 14:00:11 +0100 Subject: [PATCH] upgrade to extendedGitGraph 2 (WIP) --- www/ajax/egh_redraw.php | 7 +- www/ajax/egh_refresh.php | 9 +- www/ajax/egh_status.php | 12 +- www/extern/egh/ConnectionGitea.php | 39 -- www/extern/egh/ConnectionGithub.php | 166 -------- www/extern/egh/EGGDatabase.php | 430 ++++++++++++++++++++ www/extern/egh/EGHRemoteConfig.php | 27 -- www/extern/egh/EGHRenderer.php | 224 ----------- www/extern/egh/ExtendedGitGraph.php | 189 --------- www/extern/egh/ExtendedGitGraph2.php | 145 +++++++ www/extern/egh/Logger.php | 82 ++++ www/extern/egh/Models.php | 83 ++++ www/extern/egh/OutputGenerator.php | 284 +++++++++++++ www/extern/egh/RemoteSource.php | 575 +++++++++++++++++++++++++++ www/extern/egh/SingleCommitInfo.php | 31 -- www/extern/egh/Utils.php | 166 +++++++- www/extern/egh/db_init.sql | 37 ++ www/extern/egh/db_queryyear.sql | 10 + www/internals/mikeschergitgraph.php | 36 +- www/pages/about.php | 2 +- 20 files changed, 1840 insertions(+), 714 deletions(-) delete mode 100644 www/extern/egh/ConnectionGitea.php delete mode 100644 www/extern/egh/ConnectionGithub.php create mode 100644 www/extern/egh/EGGDatabase.php delete mode 100644 www/extern/egh/EGHRemoteConfig.php delete mode 100644 www/extern/egh/EGHRenderer.php delete mode 100644 www/extern/egh/ExtendedGitGraph.php create mode 100644 www/extern/egh/ExtendedGitGraph2.php create mode 100644 www/extern/egh/Logger.php create mode 100644 www/extern/egh/Models.php create mode 100644 www/extern/egh/OutputGenerator.php create mode 100644 www/extern/egh/RemoteSource.php delete mode 100644 www/extern/egh/SingleCommitInfo.php create mode 100644 www/extern/egh/db_init.sql create mode 100644 www/extern/egh/db_queryyear.sql diff --git a/www/ajax/egh_redraw.php b/www/ajax/egh_redraw.php index b7d6597..f46193c 100644 --- a/www/ajax/egh_redraw.php +++ b/www/ajax/egh_redraw.php @@ -1,14 +1,11 @@ init(); -$v->updateFromCache(); -$v->generate(); +$v->update(); -file_put_contents(__DIR__ . '/../dynamic/egh.html', $v->getAll()); \ No newline at end of file diff --git a/www/ajax/egh_refresh.php b/www/ajax/egh_refresh.php index e67a198..78eaa61 100644 --- a/www/ajax/egh_refresh.php +++ b/www/ajax/egh_refresh.php @@ -1,15 +1,12 @@ init(); -$v->updateFromRemotes(); -$v->generate(); - -file_put_contents(__DIR__ . '/../dynamic/egh.html', $v->getAll()); +$v->update(); +$v->updateCache(); diff --git a/www/ajax/egh_status.php b/www/ajax/egh_status.php index c8cc1af..03e9fac 100644 --- a/www/ajax/egh_status.php +++ b/www/ajax/egh_status.php @@ -2,9 +2,17 @@ if (session_status() !== PHP_SESSION_ACTIVE) session_start(); -if (key_exists('ajax_progress_egh_refresh', $_SESSION)) - echo $_SESSION['ajax_progress_egh_refresh']; +global $CONFIG; + +if (isset($_GET['clear'])) +{ + if (key_exists($CONFIG['extendedgitgraph']['session_var'], $_SESSION)) $_SESSION[$CONFIG['extendedgitgraph']['session_var']] = ''; +} + +if (key_exists($CONFIG['extendedgitgraph']['session_var'], $_SESSION)) + echo $_SESSION[$CONFIG['extendedgitgraph']['session_var']]; else echo '[[ NO SESSION STARTED ]]'; + return; \ No newline at end of file diff --git a/www/extern/egh/ConnectionGitea.php b/www/extern/egh/ConnectionGitea.php deleted file mode 100644 index 922c7b0..0000000 --- a/www/extern/egh/ConnectionGitea.php +++ /dev/null @@ -1,39 +0,0 @@ -owner = $owner; - } - - public function setURL($giteaurl) { - $this->url = $giteaurl; - } - - /* @return SingleCommitInfo[] */ - public function getDataUser($cfg) - { - $result = []; //TODO - - return $result; - } - - /* @return SingleCommitInfo[] */ - public function getDataRepository($cfg) - { - $result = []; //TODO - - return $result; - } -} \ No newline at end of file diff --git a/www/extern/egh/ConnectionGithub.php b/www/extern/egh/ConnectionGithub.php deleted file mode 100644 index bd2aa3c..0000000 --- a/www/extern/egh/ConnectionGithub.php +++ /dev/null @@ -1,166 +0,0 @@ -owner = $owner; - } - - public function setAPIToken($token) { - $this->token = $token; - } - - public function queryAPIToken($client_id, $client_secret) { - $url = Utils::sharpFormat(self::URL_OAUTH_TOKEN, ['id'=>$client_id, 'secret'=>$client_secret, 'code'=>'egh']); - $result = file_get_contents($url); - - $result = str_replace('access_token=', '', $result); - $result = str_replace('&scope=&token_type=bearer', '', $result); - - $this->setAPIToken($result); - } - - /** - * @param $cfg EGHRemoteConfig - * @return SingleCommitInfo[] - */ - public function getDataUser($cfg) - { - $repos = $this->listRepositoriesByUser($cfg->Param); - - $result = []; - foreach ($repos as $repo) - { - $commits = $this->listCommitsFromRepo($repo, $cfg->Author); - foreach ($commits as $c) $result []= $c; - } - - return $result; - } - - - /** - * @param $cfg EGHRemoteConfig - * @return SingleCommitInfo[] - */ - public function getDataRepository($cfg) - { - return $this->listCommitsFromRepo($cfg->Param, $cfg->Author); - } - - /** - * @param $user string - * @return string[] - */ - private function listRepositoriesByUser($user) - { - $result = []; - - $page = 1; - $url = Utils::sharpFormat(self::API_REPOSITORIESLIST, ['user'=>$user, 'page'=>$page, 'token'=>$this->token]); - - $json = $this->getJSON($url); - - while (! empty($json)) { - foreach ($json as $result_repo) { - $result []= $result_repo->{'full_name'}; - $this->owner->out("Found Repo: " . $result_repo->{'full_name'}); - } - - $page++; - $url = Utils::sharpFormat(self::API_REPOSITORIESLIST, ['user'=>$user, 'page'=>$page, 'token'=>$this->token]); - $json = $this->getJSON($url); - } - - return $result; - } - - /** - * @param $repo string - * @param $user string - * @return SingleCommitInfo[] - */ - private function listCommitsFromRepo($repo, $user) - { - $page = 1; - $url = Utils::sharpFormat(self::API_COMMITSLIST, ['repo'=>$repo, 'author'=>$user, 'page'=>$page, 'token'=>$this->token]); - - $result = $this->getJSON($url); - - $commit_list = []; - - while (! empty($result)) { - foreach ($result as $rc) $commit_list[] = new SingleCommitInfo(DateTime::createFromFormat(DateTime::ISO8601, $rc->{'commit'}->{'author'}->{'date'}), 'github', $user, $repo); - $this->owner->out("Found " . count($result) . " Commits in " . $repo); - - $page++; - $url = Utils::sharpFormat(self::API_COMMITSLIST, [ 'repo'=>$repo, 'author'=>$user, 'page'=>$page, 'token'=>$this->token ]); - $result = $this->getJSON($url); - } - - return $commit_list; - } - public function getJSON($url) { - if (array_key_exists('HTTP_USER_AGENT', $_SERVER)) { - $options = - [ - 'http' => - [ - 'user_agent' => $_SERVER['HTTP_USER_AGENT'], - 'header' => 'Authorization: token ' . $this->token, - ], - 'https' => - [ - 'user_agent' => $_SERVER['HTTP_USER_AGENT'], - 'header' => 'Authorization: token ' . $this->token, - ], - ]; - } else { - $options = - [ - 'http' => - [ - 'user_agent' => 'ExtendedGitGraph_for_mikescher.com', - 'header' => 'Authorization: token ' . $this->token, - ], - 'https' => - [ - 'user_agent' => 'ExtendedGitGraph_for_mikescher.com', - 'header' => 'Authorization: token ' . $this->token, - ], - ]; - } - - $context = stream_context_create($options); - - $response = @file_get_contents($url, false, $context); - - if ($response === false) - { - $this->owner->out("Error recieving json: '" . $url . "'"); - return []; - } - - return json_decode($response); - } - -} \ No newline at end of file diff --git a/www/extern/egh/EGGDatabase.php b/www/extern/egh/EGGDatabase.php new file mode 100644 index 0000000..67ec66d --- /dev/null +++ b/www/extern/egh/EGGDatabase.php @@ -0,0 +1,430 @@ +path = $path; + $this->logger = $log; + } + + public function open() + { + $exists = file_exists($this->path); + + if (!$exists) $this->logger->proclog("No database file found. Creating new at '" . $this->path . "'"); + + $dsn = "sqlite:" . $this->path; + $opt = + [ + PDO::ATTR_ERRMODE => PDO::ERRMODE_EXCEPTION, + PDO::ATTR_DEFAULT_FETCH_MODE => PDO::FETCH_ASSOC, + ]; + + $this->pdo = new PDO($dsn, null, null, $opt); + + if(!$exists) $this->init(); + } + + private function init() + { + $this->logger->proclog("Initialize new database '" . $this->path . "'"); + + $this->query_from_file(__DIR__."/db_init.sql"); + } + + public function close() + { + $this->pdo = null; // https://stackoverflow.com/questions/18277233 + } + + /** @param $path string */ + public function query_from_file(string $path) + { + $sql = file_get_contents($path); + + $cmds = explode("/*----SPLIT----*/", $sql); + + foreach ($cmds as $cmd) $this->pdo->exec($cmd); + } + + public function sql_query_assoc(string $query) + { + $r = $this->pdo->query($query)->fetchAll(PDO::FETCH_ASSOC); + + return $r; + } + + public function sql_query_assoc_prep(string $query, array $params) + { + $stmt = $this->pdo->prepare($query); + + foreach ($params as $p) + { + if (strpos($query, $p[0]) !== FALSE) $stmt->bindValue($p[0], $p[1], $p[2]); + } + + $stmt->execute(); + $r = $stmt->fetchAll(PDO::FETCH_ASSOC); + + return $r; + } + + public function sql_exec_prep(string $query, array $params) + { + $stmt = $this->pdo->prepare($query); + + foreach ($params as $p) + { + if (strpos($query, $p[0]) !== FALSE) $stmt->bindValue($p[0], $p[1], $p[2]); + } + + $stmt->execute(); + + return $stmt->rowCount(); + } + + /** + * @param string $url + * @param string $name + * @param string $source + * @return Repository + * @throws Exception + */ + public function getOrCreateRepository(string $url, string $name, string $source) + { + $repo = $this->sql_query_assoc_prep("SELECT * FROM repositories WHERE url = :url", + [ + [":url", $url, PDO::PARAM_STR], + ]); + + if (count($repo) === 0) + { + $this->sql_exec_prep("INSERT INTO repositories (url, name, source, last_update, last_change) VALUES (:url, :nam, :src, :lu, :lc)", + [ + [":url", $url, PDO::PARAM_STR], + [":nam", $name, PDO::PARAM_STR], + [":src", $source, PDO::PARAM_STR], + [":lu", Utils::sqlnow(), PDO::PARAM_STR], + [":lc", Utils::sqlnow(), PDO::PARAM_STR], + ]); + + $repo = $this->sql_query_assoc_prep("SELECT * FROM repositories WHERE url = :url", + [ + [":url", $url, PDO::PARAM_STR], + ]); + + if (count($repo) === 0) throw new Exception("No repo after insert [" . $source . "|" . $name . "]"); + + $this->logger->proclog("Inserted (new) repository [" . $source . "|" . $name . "] into database"); + } + + $r = new Repository(); + $r->ID = $repo[0]['id']; + $r->URL = $repo[0]['url']; + $r->Name = $repo[0]['name']; + $r->Source = $repo[0]['source']; + $r->LastUpdate = $repo[0]['last_update']; + $r->LastChange = $repo[0]['last_change']; + return $r; + } + + /** + * @param string $source + * @param Repository $repo + * @param string $name + * @return Branch + * @throws Exception + */ + public function getOrCreateBranch(string $source, Repository $repo, string $name) + { + $branch = $this->sql_query_assoc_prep("SELECT * FROM branches WHERE repo_id = :rid AND name = :nam", + [ + [":rid", $repo->ID, PDO::PARAM_INT], + [":nam", $name, PDO::PARAM_STR], + ]); + + if (count($branch) === 0) + { + $this->sql_exec_prep("INSERT INTO branches (repo_id, name, head, last_update, last_change) VALUES (:rid, :nam, :sha, :lu, :lc)", + [ + [":rid", $repo->ID, PDO::PARAM_INT], + [":nam", $name, PDO::PARAM_STR], + [":sha", null, PDO::PARAM_STR], + [":lu", Utils::sqlnow(), PDO::PARAM_STR], + [":lc", Utils::sqlnow(), PDO::PARAM_STR], + ]); + + $branch = $this->sql_query_assoc_prep("SELECT * FROM branches WHERE repo_id = :rid AND name = :nam", + [ + [":rid", $repo->ID, PDO::PARAM_INT], + [":nam", $name, PDO::PARAM_STR], + ]); + + if (count($branch) === 0) throw new Exception("No branch after insert [" . $source . "|" . $repo->Name . "|" . $name . "]"); + + $this->logger->proclog("Inserted (new) branch [" . $source . "|" . $repo->Name . "|" . $name . "] into database"); + } + + $r = new Branch(); + $r->ID = $branch[0]['id']; + $r->Name = $branch[0]['name']; + $r->Repo = $repo; + $r->Head = $branch[0]['head']; + $r->LastUpdate = $branch[0]['last_update']; + $r->LastChange = $branch[0]['last_change']; + return $r; + } + + /** + * @param string $source + * @param Repository $repo + * @param Branch $branch + * @param Commit[] $commits + */ + public function insertNewCommits(string $source, Repository $repo, Branch $branch, array $commits) { + $this->logger->proclog("Inserted " . count($commits) . " (new) commits into [" . $source . "|" . $repo->Name . "|" . $branch->Name . "]"); + + foreach ($commits as $commit) + { + $strparents = implode(";", $commit->Parents); + + $this->sql_exec_prep("INSERT INTO commits ([branch_id], [hash], [author_name], [author_email], [committer_name], [committer_email], [message], [date], [parent_commits]) VALUES (:brid, :sha, :an, :am, :cn, :cm, :msg, :dat, :prt)", + [ + [":brid", $branch->ID, PDO::PARAM_INT], + [":sha", $commit->Hash, PDO::PARAM_STR], + [":an", $commit->AuthorName, PDO::PARAM_STR], + [":am", $commit->AuthorEmail, PDO::PARAM_STR], + [":cn", $commit->CommitterName, PDO::PARAM_STR], + [":cm", $commit->CommitterEmail, PDO::PARAM_STR], + [":msg", $commit->Message, PDO::PARAM_STR], + [":dat", $commit->Date, PDO::PARAM_STR], + [":prt", $strparents, PDO::PARAM_STR], + ]); + + $dbid = $this->sql_query_assoc_prep("SELECT id FROM commits WHERE [branch_id] = :brid AND [Hash] = :sha", + [ + [":brid", $branch->ID, PDO::PARAM_INT], + [":sha", $commit->Hash, PDO::PARAM_STR], + ]); + + $commit->ID = $dbid[0]['id']; + } + } + + /** + * @param Branch $branch + * @param string $head + */ + public function setBranchHead(Branch $branch, string $head) { + $this->sql_exec_prep("UPDATE branches SET head = :head WHERE id = :id", + [ + [":id", $branch->ID, PDO::PARAM_INT], + [":head", $head, PDO::PARAM_STR], + ]); + + $branch->Head = $head; + } + + public function setUpdateDateOnRepository(Repository $repo) { + $now = Utils::sqlnow(); + $this->sql_exec_prep("UPDATE repositories SET last_update = :now WHERE id = :id", + [ + [":id", $repo->ID, PDO::PARAM_INT], + [":now", $now, PDO::PARAM_STR], + ]); + $repo->LastUpdate = $now; + } + + public function setChangeDateOnRepository(Repository $repo) { + $now = Utils::sqlnow(); + $this->sql_exec_prep("UPDATE repositories SET last_change = :now WHERE id = :id", + [ + [":id", $repo->ID, PDO::PARAM_INT], + [":now", $now, PDO::PARAM_STR], + ]); + $repo->LastChange = $now; + } + + public function setUpdateDateOnBranch(Branch $branch) { + $now = Utils::sqlnow(); + $this->sql_exec_prep("UPDATE branches SET last_update = :now WHERE id = :id", + [ + [":id", $branch->ID, PDO::PARAM_INT], + [":now", $now, PDO::PARAM_STR], + ]); + $branch->LastUpdate = $now; + } + + public function setChangeDateOnBranch(Branch $branch) { + $now = Utils::sqlnow(); + $this->sql_exec_prep("UPDATE branches SET last_change = :now WHERE id = :id", + [ + [":id", $branch->ID, PDO::PARAM_INT], + [":now", $now, PDO::PARAM_STR], + ]); + $branch->LastChange = $now; + } + + /** + * @param string $source + * @param Repository $repo + * @param Branch[] $branches + */ + public function deleteOtherBranches(string $source, Repository $repo, array $branches) + { + $db = $this->sql_query_assoc_prep("SELECT id, repo_id, name FROM branches WHERE repo_id = :rid", + [ + [":rid", $repo->ID, PDO::PARAM_STR] + ]); + + foreach ($db as $dbname) + { + $exist = false; + foreach ($branches as $brnch) if ($brnch->ID === $dbname['id']) $exist = true; + + if (!$exist) $this->deleteBranchRecursive($source, $repo->Name, $dbname['name'], $dbname['id']); + } + } + + /** + * @param string $source + * @param Repository[] $repos + */ + public function deleteOtherRepositories(string $source, array $repos) + { + $db = $this->sql_query_assoc_prep("SELECT id, url, name FROM repositories WHERE source = :src", + [ + [":src", $source, PDO::PARAM_STR] + ]); + + foreach ($db as $dbname) + { + $exist = false; + foreach ($repos as $rep) if ($rep->ID === $dbname['id']) $exist = true; + + if (!$exist) $this->deleteRepositoryRecursive($source, $dbname['name'], $dbname['id']); + } + } + + /** + * @param string $source + * @param string $name + * @param int $id + */ + private function deleteRepositoryRecursive(string $source, string $name, int $id) + { + $this->logger->proclog("Delete repository [".$source."|" . $name . "] from database (no longer exists)"); + + $branches = $this->sql_query_assoc_prep("SELECT id FROM branches WHERE repo_id = :rid", [ [":rid", $id, PDO::PARAM_INT] ]); + + $this->sql_exec_prep("DELETE FROM repositories WHERE id = :id", [[":id", $id, PDO::PARAM_INT]]); + $this->sql_exec_prep("DELETE FROM branches WHERE repo_id = :rid", [[":rid", $id, PDO::PARAM_INT]]); + foreach ($branches as $branch) $this->sql_exec_prep("DELETE FROM commits WHERE branch_id = :bid", [[":bid", $branch["id"], PDO::PARAM_INT]]); + } + + /** + * @param string $source + * @param string $reponame + * @param string $name + * @param int $id + */ + private function deleteBranchRecursive(string $source, string $reponame, string $name, int $id) + { + $this->logger->proclog("Delete branch [" . $source . "|" . $reponame . "|" . $name . "] from database (no longer exists)"); + + $this->sql_exec_prep("DELETE FROM branches WHERE id = :bid", [[":bid", $id, PDO::PARAM_INT]]); + $this->sql_exec_prep("DELETE FROM commits WHERE branch_id = :bid", [[":bid", $id, PDO::PARAM_INT]]); + } + + /** + * @param Branch $branch + */ + public function deleteAllCommits(Branch $branch) { + $this->sql_exec_prep("DELETE FROM commits WHERE branch_id = :bid", [[":bid", $branch->ID, PDO::PARAM_INT]]); + } + + public function deleteOldSources(array $sources) + { + $dbsources = $this->sql_query_assoc_prep("SELECT source FROM repositories GROUP BY source", []); + + foreach ($dbsources as $dbsrc) + { + $exist = false; + foreach ($sources as $src) if ($src === $dbsrc['source']) $exist=true; + + if (!$exist) + { + $this->logger->proclog("Delete source [" . $dbsrc['source'] . "] from database (no longer exists)"); + $repos = $this->sql_query_assoc_prep("SELECT source,name,id FROM repositories WHERE source = :src", [ [":src", $dbsrc['source'], PDO::PARAM_STR] ]); + + foreach ($repos as $r) $this->deleteRepositoryRecursive($r['source'], $r['name'], $r['id']); + } + } + } + + /** + * @param int $year + * @param string[] $identities + * @return array + */ + public function getCommitCountOfYearByDate(int $year, array $identities): array + { + $sql = file_get_contents(__DIR__ . "/db_queryyear.sql"); + + $cond = "(1=0)"; + $prep = + [ + [":year", "".$year, PDO::PARAM_STR] + ]; + $i=0; + foreach ($identities as $ident) + { + $cond .= " OR (mail1 = :_".$i."_)"; + $prep []= [":_".$i."_", $ident, PDO::PARAM_STR]; + $i++; + $cond .= " OR (mail2 = :_".$i."_)"; + $prep []= [":_".$i."_", $ident, PDO::PARAM_STR]; + $i++; + } + + $sql = str_replace("/*{INDETITY_COND}*/", $cond, $sql); + + $rows = $this->sql_query_assoc_prep($sql, $prep); + + $r = []; + foreach ($rows as $row) $r[$row['commitdate']] = $row['count']; + + return $r; + } + + /** + * @return int[] + */ + public function getAllYears(): array + { + $rows = $this->sql_query_assoc("SELECT d FROM (SELECT cast(strftime('%Y', commits.date) as decimal) AS d FROM commits) GROUP BY d ORDER BY d"); + $r = []; + foreach ($rows as $row) $r []= $row['d']; + return $r; + } +} \ No newline at end of file diff --git a/www/extern/egh/EGHRemoteConfig.php b/www/extern/egh/EGHRemoteConfig.php deleted file mode 100644 index 1a11566..0000000 --- a/www/extern/egh/EGHRemoteConfig.php +++ /dev/null @@ -1,27 +0,0 @@ -Type = $typ; - $this->URL = $url; - $this->Author = $usr; - $this->Param = $param; - } - -} \ No newline at end of file diff --git a/www/extern/egh/EGHRenderer.php b/www/extern/egh/EGHRenderer.php deleted file mode 100644 index 6258fad..0000000 --- a/www/extern/egh/EGHRenderer.php +++ /dev/null @@ -1,224 +0,0 @@ - ['#F5F5F5', '#DBDEE0', '#C2C7CB', '#AAB0B7', '#9099A2', '#77828E', '#5E6B79', '#455464', '#2C3E50'], - 'standard' => ["#ebedf0", "#c6e48b", "#7bc96f", "#239a3b", "#196127"], - 'modern' => ["#afaca8", "#d6e685", "#8cc665", "#44a340", "#1e6823"], - 'gray' => ["#eeeeee", "#bdbdbd", "#9e9e9e", "#616161", "#212121"], - 'red' => ["#eeeeee", "#ff7171", "#ff0000", "#b70000", "#830000"], - 'blue' => ["#eeeeee", "#6bcdff", "#00a1f3", "#0079b7", "#003958"], - 'purple' => ["#eeeeee", "#d2ace6", "#aa66cc", "#660099", "#4f2266"], - 'orange' => ["#eeeeee", "#ffcc80", "#ffa726", "#fb8c00", "#e65100"], - 'halloween' => ["#eeeeee", "#ffee4a", "#ffc501", "#fe9600", "#03001c"], - ]; - - const MONTHS = ['Jan', 'Feb', 'Mar', 'Apr', 'May', 'Jun', 'Jul', 'Aug', 'Sep', 'Oct', 'Nov', 'Dec']; - const DAYS = ['M', 'T', 'W', 'T', 'F', 'S', 'S']; - - /* @var ExtendedGitGraph */ - private $owner; - /* @var SingleCommitInfo[] */ - public $data; - /* @var string */ - public $colorScheme = 'standard'; - /* @var int[] */ - public $yearList; - /* @var array */ - public $commitMap; // date('Y-m-d') -> count - - public function __construct($egh) { - $this->owner = $egh; - } - - /** - * @param $data SingleCommitInfo[] - */ - public function init($data) - { - $this->data = $data; - - $this->yearList = $this->getYears($data); - $this->owner->out("Found " . count($this->yearList) . " year to generate."); - - $this->commitMap = $this->generateCommitMap($data, $this->yearList); - $this->owner->out("Commitmap with ".count($this->commitMap)." entries generated."); - } - - /** - * @param $data SingleCommitInfo[] - * @return int[] - */ - public function getYears($data) { - $years = array(); - - foreach ($data as $commit) { - if(! in_array($commit->Timestamp->format('Y'), $years)) - $years[] = intval($commit->Timestamp->format('Y')); - } - - asort($years); - - return $years; - } - - /** - * @param $data SingleCommitInfo[] - * @param $yearList int[] - * @return array - */ - private function generateCommitMap($data, $yearList) - { - $result = []; - - foreach ($yearList as $year) - { - $ymap = []; - - $date = new DateTime($year . '-01-01'); - while($date->format('Y') == $year) - { - $ymap[$date->format('Y-m-d')] = 0; - $date = $date->modify("+1 day"); - } - - foreach ($data as $commit) - { - if(array_key_exists($commit->Timestamp->format('Y-m-d'), $ymap)) $ymap[$commit->Timestamp->format('Y-m-d')]++; - } - - $result = array_merge($result, $ymap); - } - - - return $result; - } - - /** - * @param $year int - * @return int - */ - private function getMaxCommitCount($year) - { - $max = 0; - foreach ($this->commitMap as $date => $count) if (Utils::startsWith($date, strval($year))) $max = max($max, $count); - return $max; - } - - /** - * @param $year int - * @return string - */ - public function render($year) - { - $now = new DateTime(); - $date = new DateTime($year . '-01-01'); - $monthlist = array_fill(0, 12, [0, 0]); - $colors = self::COLOR_SCHEMES[$this->colorScheme]; - - $ymapmax = $this->getMaxCommitCount($year); - $exponent = log(0.98/(count($colors)-1), 1/$ymapmax); // (1/max)^n = 0.98 // => 1 commit erreicht immer genau die erste stufe - - $html = ''; - - $html .= '
' . "\n"; - $html .= '' . "\n"; - $html .= '' . "\n"; - $html .= '' . "\n"; - - $week = 0; - $wday = 0; - while($date->format('Y') == $year) - { - if ($date > $now) // THE FUTURE, SPONGEBOB - { - while ($date->format('d') != $date->format('t')) - { - if ($date->format('N') == 1 && $date->format('z') > 0) $week++; - $date = $date->modify("+1 day"); - } - $monthlist[$date->format('m') - 1][1] = $week + ($wday / 7); - - $date = $date->modify("+1 year"); // Kill - continue; - } - - $c_count = $this->commitMap[$date->format('Y-m-d')]; - $color_idx = min((count($colors)-1), ceil(pow($c_count/$ymapmax, $exponent) * (count($colors)-1))); - $color = $colors[$color_idx]; - - $wday = ($date->format('N') - 1); - - if ($date->format('N') == 1 && $date->format('z') > 0) - { - $html .= '' . "\n"; - $week++; - $html .= '' . "\n"; - } - - if ($date->format('d') == 1) - { - $monthlist[$date->format('m') - 1][0] = $week + ($wday / 7); - } - else if ($date->format('d') == $date->format('t')) - { - $monthlist[$date->format('m') - 1][1] = $week + ($wday / 7); - } - - $html .= '' . "\n"; - - $date = $date->modify("+1 day"); - } - - $html .= '' . "\n"; - - for($i = 0; $i < 12; $i++) - { - if ($monthlist[$i][1]-$monthlist[$i][0] > 0) - { - $posx = (($monthlist[$i][0]+$monthlist[$i][1])/2) * self::DIST_X; - $html .= '' . self::MONTHS[$i] . '' . "\n"; - } - } - - for($i = 0; $i < 7; $i++) { - $html .= '' . self::DAYS[$i] . '' . "\n"; - } - - $html .= '' . $year . '' . "\n"; - - $html .= '' . "\n"; - $html .= '' . "\n"; - $html .= '
' . "\n"; - $html .= '  ' . "\n"; - $html .= '
' . "\n"; - $html .= '
' . "\n"; - $html .= 'extendedGitGraph' . "\n"; - $html .= '
' . "\n"; - $html .= '
' . "\n"; - - - return $html; - } -} \ No newline at end of file diff --git a/www/extern/egh/ExtendedGitGraph.php b/www/extern/egh/ExtendedGitGraph.php deleted file mode 100644 index 4f7e5a8..0000000 --- a/www/extern/egh/ExtendedGitGraph.php +++ /dev/null @@ -1,189 +0,0 @@ -filenamecache = $filename_cache; - $this->remoteconfigs = []; - $this->ConnectionGithub = new ConnectionGithub($this); - $this->ConnectionGitea = new ConnectionGitea($this); - $this->outputMode = $outmode; - $this->logFilePath = $logfile; - } - - public function addRemote($type, $url, $user, $param) { - $this->remoteconfigs []= new EGHRemoteConfig($type, $url, $user, $param); - } - - public function setColorScheme($s) { - $this->colorScheme = $s; - } - - public function out($txt) - { - if ($txt !== '') $txt = '[' . date('H:i:s') . '] ' . $txt; - - if ($this->outputMode === self::OUT_SESSION) - { - if (session_status() !== PHP_SESSION_ACTIVE) session_start(); - - $_SESSION[self::PROGRESS_SESSION_COOKIE] .= $txt . "\r\n"; - session_commit(); - } - - print $txt; - print "\r\n"; - - $logfile = Utils::sharpFormat($this->logFilePath, ['num'=>'']); - file_put_contents($logfile, $txt.PHP_EOL , FILE_APPEND | LOCK_EX); - } - - public function init() - { - if ($this->outputMode === self::OUT_SESSION) - { - if (session_status() !== PHP_SESSION_ACTIVE) session_start(); - $_SESSION[self::PROGRESS_SESSION_COOKIE] = ''; - session_commit(); - } - - $f3 = Utils::sharpFormat($this->logFilePath, ['num'=>'_3']); - $f2 = Utils::sharpFormat($this->logFilePath, ['num'=>'_2']); - $f1 = Utils::sharpFormat($this->logFilePath, ['num'=>'_1']); - $f0 = Utils::sharpFormat($this->logFilePath, ['num'=>'' ]); - - if (file_exists($f3)) @unlink($f3); - if (file_exists($f2)) @rename($f2, $f3); - if (file_exists($f1)) @rename($f1, $f2); - if (file_exists($f0)) @rename($f0, $f1); - if (file_exists($f0)) @unlink($f0); - - $this->out('EXTENDED_GIT_GRAPH started'); - $this->out(''); - } - - public function updateFromRemotes() - { - $data = []; - - foreach ($this->remoteconfigs as $cfg) - { - if ($cfg->Type === 'github-user') - $data = array_merge($data, $this->ConnectionGithub->getDataUser($cfg)); - else if ($cfg->Type === 'github-repository') - $data = array_merge($data, $this->ConnectionGithub->getDataRepository($cfg)); - else if ($cfg->Type === 'gitea-user') - $data = array_merge($data, $this->ConnectionGitea->getDataUser($cfg)); - else if ($cfg->Type === 'gitea-repository') - $data = array_merge($data, $this->ConnectionGitea->getDataRepository($cfg)); - else - $this->out("Unknown type: " . $cfg->Type); - } - - $this->out("Found " . count($data) . " commits."); - - file_put_contents($this->filenamecache, serialize($data)); - - $this->queriedData = $data; - - } - - public function updateFromCache() - { - if (file_exists($this->filenamecache)) - $this->queriedData = unserialize(file_get_contents($this->filenamecache)); - else - $this->queriedData = []; - } - - public function generate() - { - $renderer = new EGHRenderer($this); - $renderer->colorScheme = $this->colorScheme; - - $renderer->init($this->queriedData); - - $this->renderedHTML = []; - foreach ($renderer->yearList as $y) $this->renderedHTML[$y] = $renderer->render($y); - } - - /** - * @param $url string - * @return array|mixed - */ - public function getJSON($url) { - if (array_key_exists('HTTP_USER_AGENT', $_SERVER)) { - $options = - [ - 'http' => ['user_agent'=> $_SERVER['HTTP_USER_AGENT']], - 'https' => ['user_agent'=> $_SERVER['HTTP_USER_AGENT']], - ]; - } else { - $options = - [ - 'http' => ['user_agent'=> 'ExtendedGitGraph_for_mikescher.com'], - 'https' => ['user_agent'=> 'ExtendedGitGraph_for_mikescher.com'], - ]; - } - - $context = stream_context_create($options); - - $response = @file_get_contents($url, false, $context); - - if ($response === false) - { - $this->out("Error recieving json: '" . $url . "'"); - return []; - } - - return json_decode($response); - } - - public function get() - { - return $this->renderedHTML; - } - - public function getAll() - { - $all = ''; - foreach ($this->get() as $year => $html) $all .= $html . "\n"; - return $all; - } -} \ No newline at end of file diff --git a/www/extern/egh/ExtendedGitGraph2.php b/www/extern/egh/ExtendedGitGraph2.php new file mode 100644 index 0000000..355922e --- /dev/null +++ b/www/extern/egh/ExtendedGitGraph2.php @@ -0,0 +1,145 @@ +logger = []; + if ($config['output_session']) $this->logger []= new SessionLogger($config['session_var']); + if ($config['output_stdout']) $this->logger []= new OutputLogger(); + if ($config['output_logfile']) $this->logger []= new FileLogger($config['logfile'], $config['logfile_count']); + + $this->sources = []; + + $sourcenames = []; + foreach ($config['remotes'] as $rmt) + { + $newsrc = null; + if ($rmt['type'] === 'github') + $newsrc = new GithubConnection($this, $rmt['name'], $rmt['url'], $rmt['filter'], $rmt['exclusions'], $rmt['oauth_id'], $rmt['oauth_secret'], $rmt['token_cache'] ); + else if ($rmt['type'] === 'gitea') + $newsrc = new GiteaConnection($this, $rmt['name'], $rmt['url'], $rmt['filter'], $rmt['exclusions'], $rmt['username'], $rmt['password'] ); + else + throw new Exception("Unknown remote-type: " . $rmt['type']); + + if (array_key_exists($newsrc->getName(), $sourcenames)) throw new Exception("Duplicate source name: " . $newsrc->getName()); + + $this->sources []= $newsrc; + $sourcenames []= $newsrc->getName(); + } + + $this->db = new EGGDatabase($config['data_cache_file'], $this); + + $this->outputter = new FullRenderer($this, $config['identities'], $config['output_cache_files']); + } + + public function update() + { + try + { + $this->db->open(); + + $this->proclog("Start incremental data update"); + $this->proclog(); + + foreach ($this->sources as $src) + { + $this->proclog("======= UPDATE " . $src->getName() . " ======="); + + $src->update($this->db); + + $this->proclog(); + } + + $this->db->deleteOldSources(array_map(function (IRemoteSource $v){ return $v->getName(); }, $this->sources)); + + $this->proclog("Update finished."); + + $this->db->close(); + } + catch (Exception $exception) + { + $this->proclog("(!) FATAL ERROR -- UNCAUGHT EXCEPTION THROWN"); + $this->proclog(); + $this->proclog($exception->getMessage()); + $this->proclog(); + $this->proclog($exception->getTraceAsString()); + } + } + + public function updateCache(): string + { + try + { + $this->db->open(); + + $this->proclog("Start update cache"); + $this->proclog(); + + $data = $this->outputter->updateCache($this->db); + + $this->db->close(); + $this->proclog("UpdateCache finished."); + + return $data; + } + catch (Exception $exception) + { + $this->proclog("(!) FATAL ERROR -- UNCAUGHT EXCEPTION THROWN"); + $this->proclog(); + $this->proclog($exception->getMessage()); + $this->proclog(); + $this->proclog($exception->getTraceAsString()); + } + } + + /** + * @return string|null + */ + public function loadFromCache() + { + try + { + $this->db->open(); + + $data = $this->outputter->loadFromCache(); + + return $data; + } + catch (Exception $exception) + { + $this->proclog("(!) FATAL ERROR -- UNCAUGHT EXCEPTION THROWN"); + $this->proclog(); + $this->proclog($exception->getMessage()); + $this->proclog(); + $this->proclog($exception->getTraceAsString()); + } + } + + public function proclog($text = '') + { + foreach($this->logger as $lgr) $lgr->proclog($text); + } +} \ No newline at end of file diff --git a/www/extern/egh/Logger.php b/www/extern/egh/Logger.php new file mode 100644 index 0000000..727b011 --- /dev/null +++ b/www/extern/egh/Logger.php @@ -0,0 +1,82 @@ +0;$i--) + { + $f2 = Utils::sharpFormat($filename, [ 'num' => '_'.( $i ) ]); + $f1 = Utils::sharpFormat($filename, [ 'num' => '_'.( $i-1 ) ]); + if ($i-1 === 0) $f1 = Utils::sharpFormat($filename, [ 'num' => '' ]); + + if (file_exists($f2)) @unlink($f2); + if (file_exists($f1)) @rename($f1, $f2); + } + + + $f0 = Utils::sharpFormat($filename, ['num'=>'' ]); + if (file_exists($f0)) @unlink($f0); + $this->path = $f0; + } + + public function proclog($text) + { + if ($text !== '') $text = '[' . date('H:i:s') . '] ' . $text; + + file_put_contents($this->path, $text . PHP_EOL , FILE_APPEND | LOCK_EX); + } +} + +class SessionLogger implements ILogger +{ + /** @var string $sessionvar */ + private $sessionvar; + + /** @var string $sessionvar */ + public function __construct($sessionvar) + { + $this->sessionvar = $sessionvar; + + if (session_status() !== PHP_SESSION_DISABLED) + { + if (session_status() !== PHP_SESSION_ACTIVE) session_start(); + $_SESSION[$sessionvar] = ''; + session_commit(); + } + } + + public function proclog($text) + { + if (session_status() === PHP_SESSION_DISABLED) return; + + $_SESSION[$this->sessionvar] .= $text . "\r\n"; + session_commit(); + } +} + +class OutputLogger implements ILogger +{ + public function proclog($text) + { + if ($text !== '') $text = '[' . date('H:i:s') . '] ' . $text; + + print $text; + print "\r\n"; + } + +} \ No newline at end of file diff --git a/www/extern/egh/Models.php b/www/extern/egh/Models.php new file mode 100644 index 0000000..c626633 --- /dev/null +++ b/www/extern/egh/Models.php @@ -0,0 +1,83 @@ +logger = $logger; + $this->identities = $identities; + $this->cache_files_path = $cfpath; + } + + /** + * @inheritDoc + */ + public function updateCache(EGGDatabase $db) + { + $years = $db->getAllYears(); + $dyears = []; + + $result = ""; + foreach ($years as $year) + { + $gen = new SingleYearRenderer($this->logger, $year, $this->identities, $this->cache_files_path); + $cc = $gen->updateCache($db); + if ($cc === null) continue; + + $result .= $cc; + $result .= "\n\n\n"; + + $dyears []= $year; + } + + $data = json_encode($dyears); + + $path = Utils::sharpFormat($this->cache_files_path, ['ident' => 'fullrenderer']); + + file_put_contents($path, $data); + $this->logger->proclog("Updated cache file for full renderer"); + + return $result; + } + + /** + * @inheritDoc + */ + public function loadFromCache() + { + $path = Utils::sharpFormat($this->cache_files_path, ['ident' => 'fullrenderer']); + if (!file_exists($path)) + { + $this->logger->proclog("No cache found for [fullrenderer]"); + return null; + } + + $years = json_decode(file_get_contents($path)); + + $result = ""; + + foreach ($years as $year) + { + $gen = new SingleYearRenderer($this->logger, $year, $this->identities, $this->cache_files_path); + $cc = $gen->loadFromCache(); + if ($cc === null) return null; + $result .= $cc; + $result .= "\n\n\n"; + } + return $result; + } +} + +class SingleYearRenderer implements IOutputGenerator +{ + const DIST_X = 13; + const DIST_Y = 13; + const DAY_WIDTH = 11; + const DAY_HEIGHT = 11; + + const MONTHS = ['Jan', 'Feb', 'Mar', 'Apr', 'May', 'Jun', 'Jul', 'Aug', 'Sep', 'Oct', 'Nov', 'Dec']; + const DAYS = ['M', 'T', 'W', 'T', 'F', 'S', 'S']; + + /** @var ILogger $logger */ + private $logger; + + /** @var int $year */ + private $year; + + /** @var string[] $identities */ + private $identities; + + /** @var string $cache_files_path */ + private $cache_files_path; + + /** + * @param ILogger $logger + * @param int $year + * @param string[] $identities + * @param string $cfpath + */ + public function __construct(ILogger $logger, int $year, array $identities, string $cfpath) + { + $this->logger = $logger; + $this->year = $year; + $this->identities = $identities; + $this->cache_files_path = $cfpath; + } + + /** + * @inheritDoc + */ + public function loadFromCache() + { + $path = Utils::sharpFormat($this->cache_files_path, ['ident' => 'singleyear_'.$this->year]); + if (!file_exists($path)) + { + $this->logger->proclog("No cache found for [".('singleyear_'.$this->year)."]"); + return null; + } + return file_get_contents($path); + } + + /** + * @inheritDoc + */ + public function updateCache(EGGDatabase $db) + { + $this->logger->proclog("Generate cache file for year ".$this->year); + $data = $this->generate($db); + + if ($data === null) { + $this->logger->proclog("No data for year ".$this->year); + $this->logger->proclog(""); + return null; + } + + $path = Utils::sharpFormat($this->cache_files_path, ['ident' => 'singleyear_'.$this->year]); + + file_put_contents($path, $data); + $this->logger->proclog("Updated cache file for year ".$this->year); + $this->logger->proclog(""); + + return $data; + } + + /** + * @param EGGDatabase $db + * @return string|null + * @throws Exception + */ + private function generate(EGGDatabase $db) + { + $dbdata = $db->getCommitCountOfYearByDate($this->year, $this->identities); + + if (Utils::array_value_max(0, $dbdata) === 0) return null; + + $now = new DateTime(); + $date = new DateTime($this->year . '-01-01'); + $ymapmax = Utils::array_value_max(1, $dbdata); + + $monthlist = array_fill(0, 12, [0, 0]); + + $exponent9 = log(0.98/((9)-1), 1/$ymapmax); + $exponent5 = log(0.98/((5)-1), 1/$ymapmax); + + + $html = ''; + + $html .= '
' . "\n"; + $html .= '' . "\n"; + $html .= '' . "\n"; + $html .= '' . "\n"; + + $week = 0; + $wday = 0; + while($date->format('Y') == $this->year) + { + if ($date > $now) // THE FUTURE, SPONGEBOB + { + while ($date->format('d') != $date->format('t')) + { + if ($date->format('N') == 1 && $date->format('z') > 0) $week++; + $date = $date->modify("+1 day"); + } + $monthlist[$date->format('m') - 1][1] = $week + ($wday / 7); + + $date = $date->modify("+1 year"); // Kill + continue; + } + + $c_count = array_key_exists($date->format('Y-m-d'), $dbdata) ? $dbdata[$date->format('Y-m-d')] : 0; + $color_idx9 = min(((9)-1), ceil(pow($c_count/$ymapmax, $exponent9) * ((9)-1))); + $color_idx5 = min(((5)-1), ceil(pow($c_count/$ymapmax, $exponent5) * ((5)-1))); + + $wday = ($date->format('N') - 1); + + if ($date->format('N') == 1 && $date->format('z') > 0) + { + $html .= '' . "\n"; + $week++; + $html .= '' . "\n"; + } + + if ($date->format('d') == 1) + { + $monthlist[$date->format('m') - 1][0] = $week + ($wday / 7); + } + else if ($date->format('d') == $date->format('t')) + { + $monthlist[$date->format('m') - 1][1] = $week + ($wday / 7); + } + + $html .= '' . "\n"; + + $date = $date->modify("+1 day"); + } + + $html .= '' . "\n"; + + for($i = 0; $i < 12; $i++) + { + if ($monthlist[$i][1]-$monthlist[$i][0] > 0) + { + $posx = (($monthlist[$i][0]+$monthlist[$i][1])/2) * self::DIST_X; + $html .= '' . self::MONTHS[$i] . '' . "\n"; + } + } + + for($i = 0; $i < 7; $i++) { + $html .= '' . self::DAYS[$i] . '' . "\n"; + } + + $html .= '' . $this->year . '' . "\n"; + + $html .= '' . "\n"; + $html .= '' . "\n"; + $html .= '
' . "\n"; + $html .= '  ' . "\n"; + $html .= '
' . "\n"; + $html .= '
' . "\n"; + $html .= 'extendedGitGraph' . "\n"; + $html .= '
' . "\n"; + $html .= '
' . "\n"; + + + return $html; + } +} \ No newline at end of file diff --git a/www/extern/egh/RemoteSource.php b/www/extern/egh/RemoteSource.php new file mode 100644 index 0000000..558320d --- /dev/null +++ b/www/extern/egh/RemoteSource.php @@ -0,0 +1,575 @@ +logger = $logger; + $this->name = $name; + $this->filter = $filter; + $this->exclusions = $exclusions; + } + + /** @inheritDoc + * @throws Exception + */ + public function update(EGGDatabase $db) + { + $this->preUpdate(); + + $repos = $this->listAndUpdateRepositories($db); + + foreach ($repos as $repo) + { + $branches = $this->listAndUpdateBranches($db, $repo); + $db->setUpdateDateOnRepository($repo); + + $repo_changed = false; + foreach ($branches as $branch) + { + if ($branch->HeadFromAPI === $branch->Head) + { + $db->setUpdateDateOnBranch($branch); + $this->logger->proclog("Branch: [" . $this->name . "|" . $repo->Name . "|" . $branch->Name . "] is up to date"); + continue; + } + + $commits = $this->listAndUpdateCommits($db, $repo, $branch); + $db->setUpdateDateOnBranch($branch); + if (count($commits) === 0) + { + $this->logger->proclog("Branch: [" . $this->name . "|" . $repo->Name . "|" . $branch->Name . "] has no new commits"); + continue; + } + + $this->logger->proclog("Found " . count($commits) . " new commits in Branch: [" . $this->name . "|" . $repo->Name . "|" . $branch->Name . "]"); + + $repo_changed = true; + $db->setChangeDateOnBranch($branch); + } + + if ($repo_changed) $db->setChangeDateOnRepository($repo); + } + + $this->postUpdate(); + } + + /** + * @throws Exception + */ + protected abstract function preUpdate(); + + /** + * @throws Exception + */ + protected abstract function postUpdate(); + + /** + * @param string $user + * @param int $page + * @return array + */ + protected abstract function queryRepositories($user, $page); + + /** + * @param string $reponame + * @return array + */ + protected abstract function queryBranches($reponame); + + /** + * @param string $reponame + * @param string $branchname + * @param string $startsha + * @return array + */ + protected abstract function queryCommits($reponame, $branchname, $startsha); + + /** + * @param mixed $data + * @return array + * @throws Exception + */ + protected abstract function readRepository($data); + + /** + * @param mixed $data + * @return array + * @throws Exception + */ + protected abstract function readBranch($data); + + /** + * @param mixed $data + * @return array + * @throws Exception + */ + protected abstract function readCommit($data); + + /** + * @param EGGDatabase $db + * @return Repository[] + * @throws Exception + */ + private function listAndUpdateRepositories(EGGDatabase $db) { + $f = explode('/', $this->filter); + + $result = []; + + $page = 1; + $json = $this->queryRepositories($f[0], $page); + + while (! empty($json)) + { + foreach ($json as $result_repo) + { + $jdata = $this->readRepository($result_repo); + + if (!Utils::isRepoFilterMatch($this->filter, $this->exclusions, $jdata['full_name'])) + { + $this->logger->proclog("Skip Repo: " . $jdata['full_name']); + continue; + } + + $this->logger->proclog("Found Repo in Remote: " . $jdata['full_name']); + + $result []= $db->getOrCreateRepository($jdata['html_url'], $jdata['full_name'], $this->name); + } + + $page++; + $json = $this->queryRepositories($f[0], $page); + } + + $db->deleteOtherRepositories($this->name, $result); + + return $result; + } + + /** + * @param EGGDatabase $db + * @param Repository $repo + * @return Branch[] + * @throws Exception + */ + private function listAndUpdateBranches(EGGDatabase $db, Repository $repo) { + + $result = []; + + $json = $this->queryBranches($repo->Name); + + foreach ($json as $result_branch) { + $jdata = $this->readBranch($result_branch); + + if ($jdata === null) continue; + + $bname = $jdata['name']; + $bhead = $jdata['sha']; + + $this->logger->proclog("Found Branch in Remote: [" . $repo->Name . "] " . $bname); + + $b = $db->getOrCreateBranch($this->name, $repo, $bname); + $b->HeadFromAPI = $bhead; + $result []= $b; + } + + $db->deleteOtherBranches($this->name, $repo, $result); + + return $result; + } + + /** + * @param EGGDatabase $db + * @param Repository $repo + * @param Branch $branch + * @return Commit[] + * @throws Exception + */ + private function listAndUpdateCommits(EGGDatabase $db, Repository $repo, Branch $branch) { + + $newcommits = []; + + if ($branch->HeadFromAPI === null) return []; + + $target = $branch->Head; + + $next_sha = [ $branch->HeadFromAPI ]; + $visited = [ $branch->HeadFromAPI ]; + + $json = $this->queryCommits($repo->Name, $branch->Name, $next_sha[0]); + + for (;;) + { + foreach ($json as $result_commit) + { + $jdata = $this->readCommit($result_commit); + + $sha = $jdata['sha']; + $author_name = $jdata['author_name']; + $author_email = $jdata['author_email']; + $committer_name = $jdata['committer_name']; + $committer_email = $jdata['committer_email']; + $message = $jdata['message']; + $date = $jdata['date']; + + $parents = $jdata['parents']; + + if (($rmshakey = array_search($sha, $next_sha)) !== false) unset($next_sha[$rmshakey]); + + if (in_array($sha, $visited)) continue; + $visited []= $sha; + + if ($sha === $target && count($next_sha) === 0) + { + if (count($newcommits) === 0) + { + $this->logger->proclog("Found no new commits for: [" . $this->name . "|" . $repo->Name . "|" . $branch->Name . "] (HEAD at {" . substr($branch->HeadFromAPI, 0, 8) . "})"); + return []; + } + + $db->insertNewCommits($this->name, $repo, $branch, $newcommits); + $db->setBranchHead($branch, $branch->HeadFromAPI); + + return $newcommits; + } + + $commit = new Commit(); + $commit->Branch = $branch; + $commit->Hash = $sha; + $commit->AuthorName = $author_name; + $commit->AuthorEmail = $author_email; + $commit->CommitterName = $committer_name; + $commit->CommitterEmail = $committer_email; + $commit->Message = $message; + $commit->Date = $date; + $commit->Parents = $parents; + + $newcommits []= $commit; + + foreach ($parents as $p) + { + $next_sha []= $p; + } + } + + $next_sha = array_values($next_sha); // fix numeric keys + if (count($next_sha) === 0) break; + + $json = $this->queryCommits($repo->Name, $branch->Name, $next_sha[0]); + } + + $this->logger->proclog("HEAD pointer in Branch: [" . $this->name . "|" . $repo->Name . "|" . $branch->Name . "] no longer matches. Re-query all " . count($newcommits) . " commits (old HEAD := {".substr($branch->Head, 0, 8)."})"); + + $db->deleteAllCommits($branch); + + if (count($newcommits) === 0) return []; + + $db->insertNewCommits($this->name, $repo, $branch, $newcommits); + $db->setBranchHead($branch, $branch->HeadFromAPI); + + return $newcommits; + } + + /** @inheritDoc */ + public function getName() { return $this->name; } + + /** @inheritDoc */ + public abstract function toString(); +} + +class GithubConnection extends StandardGitConnection +{ + const API_OAUTH_AUTH = 'https://github.com/login/oauth/authorize?client_id=%s'; + const URL_OAUTH_TOKEN = 'https://github.com/login/oauth/access_token?client_id={id}&client_secret={secret}&code={code}'; + + const API_RATELIMIT = 'https://api.github.com/rate_limit'; + const API_REPOSITORIESLIST = 'https://api.github.com/users/{user}/repos?page={page}&per_page=100'; + const API_COMMITSLIST = 'https://api.github.com/repos/{repo}/commits?per_page=100&sha={sha}'; + const API_BRANCHLIST = 'https://api.github.com/repos/{repo}/branches'; + + /** @var string $url */ + private $url; + + /** @var string $oauth_id */ + private $oauth_id; + + /** @var string $oauth_secret */ + private $oauth_secret; + + /** @var string $apitokenpath */ + private $apitokenpath; + + /** @var string $apitoken */ + private $apitoken; + + /** + * @param ILogger $logger + * @param string $name + * @param string $url + * @param string $filter + * @param string[] exclusions + * @param string $oauth_id + * @param string $oauth_secret + * @param string $apitokenpath + */ + public function __construct(ILogger $logger, string $name, string $url, string $filter, array $exclusions, string $oauth_id, string $oauth_secret, string $apitokenpath) + { + parent::__construct($logger, $name, $filter, $exclusions); + + $this->url = $url; + $this->oauth_id = $oauth_id; + $this->oauth_secret = $oauth_secret; + $this->apitokenpath = $apitokenpath; + + if ($this->apitokenpath !== null && file_exists($this->apitokenpath)) + $this->apitoken = file_get_contents($this->apitokenpath); + else + $this->apitoken = null; + } + + /** + * @throws Exception + */ + public function queryAPIToken() { + $url = Utils::sharpFormat(self::URL_OAUTH_TOKEN, ['id'=>$this->oauth_id, 'secret'=>$this->oauth_secret, 'code'=>'egg']); + $fullresult = $result = file_get_contents($url); + + $result = str_replace('access_token=', '', $result); + $result = str_replace('&scope=&token_type=bearer', '', $result); + + $this->logger->proclog("Updated Github API token"); + + if (Utils::startsWith($result, "error=")) throw new Exception($fullresult); + + if ($result!=='' && $result !== null && $this->apitokenpath !== null) + file_put_contents($this->apitokenpath, $result); + + $this->apitoken = $result; + } + + /** @inheritDoc */ + protected function preUpdate() + { + if ($this->apitoken === null) $this->queryAPIToken(); + } + + /** @inheritDoc */ + protected function postUpdate() + { + // + } + + /** @inheritDoc */ + protected function queryRepositories($user, $page) + { + $url = Utils::sharpFormat(self::API_REPOSITORIESLIST, ['user'=>$user, 'page'=>$page]); + return Utils::getJSONWithTokenAuth($this->logger, $url, $this->apitoken); + } + + /** @inheritDoc */ + protected function queryBranches($reponame) + { + $url = Utils::sharpFormat(self::API_BRANCHLIST, ['repo'=>$reponame]); + return Utils::getJSONWithTokenAuth($this->logger, $url, $this->apitoken); + } + + /** @inheritDoc */ + protected function queryCommits($reponame, $branchname, $startsha) + { + $url = Utils::sharpFormat(self::API_COMMITSLIST, [ 'repo'=>$reponame, 'sha'=>$startsha ]); + $this->logger->proclog("Query commits from: [" . $this->name . "|" . $reponame . "|" . $branchname . "] continuing at {" . substr($startsha, 0, 8) . "}"); + return Utils::getJSONWithTokenAuth($this->logger, $url, $this->apitoken); + } + + /** @inheritDoc */ + protected function readRepository($data) + { + return + [ + 'full_name' => $data->{'full_name'}, + 'html_url' => $data->{'html_url'}, + ]; + } + + /** @inheritDoc */ + protected function readBranch($data) + { + if (isset($data->{'block'})) return null; + + return + [ + 'name' => $data->{'name'}, + 'sha' => $data->{'commit'}->{'sha'}, + ]; + } + + /** @inheritDoc */ + protected function readCommit($data) + { + return + [ + 'sha' => $data->{'sha'}, + + 'author_name' => $data->{'commit'}->{'author'}->{'name'}, + 'author_email' => $data->{'commit'}->{'author'}->{'email'}, + + 'committer_name' => $data->{'commit'}->{'committer'}->{'name'}, + 'committer_email' => $data->{'commit'}->{'committer'}->{'email'}, + + 'message' => $data->{'commit'}->{'message'}, + 'date' => (new DateTime($data->{'commit'}->{'author'}->{'date'}))->format("Y-m-d H:i:s"), + + 'parents' => array_map(function ($v){ return $v->{'sha'}; }, $data->{'parents'}), + ]; + } + + /** @inheritDoc */ + public function toString() { return "[Github|".$this->filter."]"; } +} + +class GiteaConnection extends StandardGitConnection +{ + const API_BASE_URL = '/api/v1'; + + const API_USER_REPO_LIST = '/users/{user}/repos'; + const API_BRANCH_LIST = '/repos/{repo}/branches'; + const API_COMMIT_LIST = '/repos/{repo}/commits?sha={sha}'; + + /** @var string $url */ + private $url; + + /** @var string $username */ + private $username; + + /** @var string $password */ + private $password; + + /** + * @param ILogger $logger + * @param string $name + * @param string $url + * @param string $filter + * @param string[] $exclusions + * @param string $username + * @param string $password + */ + public function __construct(ILogger $logger, string $name, string $url, string $filter, array $exclusions, string $username, string $password) + { + parent::__construct($logger, $name, $filter, $exclusions); + + $this->url = $url; + $this->username = $username; + $this->password = $password; + } + + /** @inheritDoc */ + protected function preUpdate() + { + // + } + + /** @inheritDoc */ + protected function postUpdate() + { + // + } + + /** @inheritDoc */ + protected function queryRepositories($user, $page) + { + if ($page > 1) return []; + $url = Utils::sharpFormat(Utils::urlCombine($this->url, self::API_BASE_URL, self::API_USER_REPO_LIST), ['user'=>$user ]); + return Utils::getJSONWithTokenBasicAuth($this->logger, $url, $this->username, $this->password); + } + + /** @inheritDoc */ + protected function queryBranches($reponame) + { + $url = Utils::sharpFormat(Utils::urlCombine($this->url, self::API_BASE_URL, self::API_BRANCH_LIST), ['repo'=>$reponame]); + return Utils::getJSONWithTokenBasicAuth($this->logger, $url, $this->username, $this->password); + } + + /** @inheritDoc */ + protected function queryCommits($reponame, $branchname, $startsha) + { + $url = Utils::sharpFormat(Utils::urlCombine($this->url, self::API_BASE_URL, self::API_COMMIT_LIST), [ 'repo'=>$reponame, 'sha'=>$startsha ]); + $this->logger->proclog("Query commits from: [" . $this->name . "|" . $reponame . "|" . $branchname . "] continuing at {" . substr($startsha, 0, 8) . "}"); + return Utils::getJSONWithTokenBasicAuth($this->logger, $url, $this->username, $this->password); + } + + /** @inheritDoc */ + protected function readRepository($data) + { + return + [ + 'full_name' => $data->{'full_name'}, + 'html_url' => $data->{'html_url'}, + ]; + } + + /** @inheritDoc */ + protected function readBranch($data) + { + return + [ + 'name' => $data->{'name'}, + 'sha' => $data->{'commit'}->{'id'}, + ]; + } + + /** @inheritDoc */ + protected function readCommit($data) + { + return + [ + 'sha' => $data->{'commit'}->{'tree'}->{'sha'}, + + 'author_name' => $data->{'commit'}->{'author'}->{'name'}, + 'author_email' => $data->{'commit'}->{'author'}->{'email'}, + + 'committer_name' => $data->{'commit'}->{'committer'}->{'name'}, + 'committer_email' => $data->{'commit'}->{'committer'}->{'email'}, + + 'message' => $data->{'commit'}->{'message'}, + 'date' => (new DateTime($data->{'commit'}->{'author'}->{'date'}))->format("Y-m-d H:i:s"), + + 'parents' => array_map(function ($v){ return $v->{'sha'}; }, $data->{'parents'}), + ]; + } + + /** @inheritDoc */ + public function toString() { return "[Gitea|".$this->url."|".$this->filter."]"; } +} \ No newline at end of file diff --git a/www/extern/egh/SingleCommitInfo.php b/www/extern/egh/SingleCommitInfo.php deleted file mode 100644 index 8da17bb..0000000 --- a/www/extern/egh/SingleCommitInfo.php +++ /dev/null @@ -1,31 +0,0 @@ -Timestamp = $ts; - $this->SourcePlatform = $src; - $this->SourceUser = $usr; - $this->SourceRepository = $repo; - } - - -} \ No newline at end of file diff --git a/www/extern/egh/Utils.php b/www/extern/egh/Utils.php index 99ddaea..ae8002f 100644 --- a/www/extern/egh/Utils.php +++ b/www/extern/egh/Utils.php @@ -2,7 +2,12 @@ class Utils { - public static function sharpFormat($str, $args) + /** + * @param string $str + * @param string[] $args + * @return string + */ + public static function sharpFormat(string $str, array $args) { foreach ($args as $key => $val) { @@ -11,9 +16,164 @@ class Utils return $str; } - public static function startsWith($haystack, $needle) + /** + * @param string $haystack + * @param string $needle + * @return bool + */ + public static function startsWith(string $haystack, string $needle) { $length = strlen($needle); return (substr($haystack, 0, $length) === $needle); } -} \ No newline at end of file + + /** + * @param string $haystack + * @param string $needle + * @return bool + */ + public static function endsWith(string $haystack, string $needle) + { + $length = strlen($needle); + return ($length === 0) || (substr($haystack, -$length) === $needle); + } + + /** + * @param string $filter + * @param string[] $exclusions + * @param string $name + * @return bool + */ + public static function isRepoFilterMatch(string $filter, array $exclusions, string $name) + { + foreach ($exclusions as $ex) + { + if (strtolower($ex) === strtolower($name)) return false; + } + + $f0 = explode('/', $filter); + $f1 = explode('/', $name); + + if (count($f0) !== 2) return false; + if (count($f1) !== 2) return false; + + if ($f0[0] !== $f1[0] && $f0[0] !== '*') return false; + if ($f0[1] !== $f1[1] && $f0[1] !== '*') return false; + + return true; + } + + /** + * @param ILogger $logger + * @param string $url + * @param string $authtoken + * @return array|mixed + */ + public static function getJSONWithTokenAuth($logger, $url, $authtoken) + { + return Utils::getJSON($logger, $url, 'Authorization: token ' . $authtoken); + } + + /** + * @param ILogger $logger + * @param string $url + * @param string $usr + * @param string $pass + * @return array|mixed + */ + public static function getJSONWithTokenBasicAuth($logger, $url, $usr, $pass) + { + return Utils::getJSON($logger, $url, 'Authorization: Basic ' . base64_encode($usr.':'.$pass)); + } + + /** + * @param ILogger $logger + * @param string $url + * @param string $header + * @return array|mixed + */ + private static function getJSON($logger, $url, $header) + { + //$logger->proclog("[@] " . $url); + + if (array_key_exists('HTTP_USER_AGENT', $_SERVER)) { + $options = + [ + 'http' => + [ + 'user_agent' => $_SERVER['HTTP_USER_AGENT'], + 'header' => $header, + ], + 'https' => + [ + 'user_agent' => $_SERVER['HTTP_USER_AGENT'], + 'header' => $header, + ], + ]; + } else { + $options = + [ + 'http' => + [ + 'user_agent' => 'ExtendedGitGraph_for_mikescher.com', + 'header' => $header, + 'ignore_errors' => true, + ], + 'https' => + [ + 'user_agent' => 'ExtendedGitGraph_for_mikescher.com', + 'header' => $header, + 'ignore_errors' => true, + ], + ]; + } + + $context = stream_context_create($options); + + $response = @file_get_contents($url, false, $context); + + if ($response === false) + { + $logger->proclog("Error recieving json: '" . $url . "'"); + $logger->proclog(print_r(error_get_last(), true)); + return []; + } + + return json_decode($response); + } + + /** + * @return string + */ + public static function sqlnow() + { + return gmdate("Y-m-d H:i:s"); + } + + /** + * @param int $n0 + * @param array $dbdata + * @return int + */ + public static function array_value_max(int $n0, array $dbdata): int + { + foreach ($dbdata as $_ => $val) $n0 = max($n0, $val); + return $n0; + } + + public static function urlCombine(string... $elements) + { + $r = $elements[0]; + $skip = true; + foreach ($elements as $e) + { + if ($skip) { $skip=false; continue; } + + if (Utils::endsWith($r, '/')) $r = substr($r, 0, strlen($r)-1); + if (Utils::startsWith($e, '/')) $e = substr($e, 1); + + $r = $r . '/' . $e; + } + return $r; + } +} diff --git a/www/extern/egh/db_init.sql b/www/extern/egh/db_init.sql new file mode 100644 index 0000000..839755b --- /dev/null +++ b/www/extern/egh/db_init.sql @@ -0,0 +1,37 @@ +CREATE TABLE "repositories" +( + "id" INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT UNIQUE, + "source" TEXT NOT NULL, + "name" TEXT NOT NULL, + "url" TEXT NOT NULL UNIQUE, + "last_update" TEXT NOT NULL, + "last_change" TEXT NOT NULL +); + +/*----SPLIT----*/ + +CREATE TABLE "branches" +( + "id" INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT UNIQUE, + "repo_id" INTEGER NOT NULL, + "name" TEXT NOT NULL, + "head" TEXT, + "last_update" TEXT NOT NULL, + "last_change" TEXT NOT NULL +); + +/*----SPLIT----*/ + +CREATE TABLE "commits" +( + "id" INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT UNIQUE, + "branch_id" INTEGER NOT NULL, + "hash" TEXT NOT NULL, + "author_name" TEXT NOT NULL, + "author_email" TEXT NOT NULL, + "committer_name" TEXT NOT NULL, + "committer_email" TEXT NOT NULL, + "message" TEXT NOT NULL, + "date" TEXT NOT NULL, + "parent_commits" TEXT NOT NULL +); diff --git a/www/extern/egh/db_queryyear.sql b/www/extern/egh/db_queryyear.sql new file mode 100644 index 0000000..5d1c04c --- /dev/null +++ b/www/extern/egh/db_queryyear.sql @@ -0,0 +1,10 @@ +SELECT commitdate AS commitdate, COUNT(*) as count FROM + ( + SELECT + [hash] AS hash, min([author_email]) AS mail1, min([committer_email]) AS mail2, date(min([date])) AS commitdate + FROM COMMITS + GROUP BY hash + HAVING (strftime('%Y', commitdate) = :year AND (/*{INDETITY_COND}*/)) + ) +GROUP BY commitdate +ORDER BY commitdate \ No newline at end of file diff --git a/www/internals/mikeschergitgraph.php b/www/internals/mikeschergitgraph.php index 070f85b..0b4d02d 100644 --- a/www/internals/mikeschergitgraph.php +++ b/www/internals/mikeschergitgraph.php @@ -1,41 +1,35 @@ addRemote('github-user', null, 'Mikescher', 'Mikescher'); - //$v->addRemote('github-user', null, 'Mikescher', 'Sam-Development'); - //$v->addRemote('github-repository', null, 'Mikescher', 'Anastron/ColorRunner'); - $v->addRemote('gitea-repository', null, 'Mikescher', 'Mikescher/server-scripts'); - $v->addRemote('gitea-repository', null, 'Mikescher', 'Mikescher/apache-sites'); - $v->addRemote('gitea-repository', null, 'Mikescher', 'Mikescher/MVU_API'); - - $v->setColorScheme($CONFIG['egh_theme']); - - $v->ConnectionGithub->setAPIToken($CONFIG['egh_token']); - - $v->ConnectionGitea->setURL('https://gogs.mikescher.com'); - - return $v; + return new ExtendedGitGraph2($CONFIG['extendedgitgraph']); } public static function getPathRenderedData() { - return __DIR__ . '/../dynamic/egh.html'; + return __DIR__ . '/../dynamic/egg/cache_fullrenderer.html'; } - public static function includeRender() + /** + * @return string|null + * @throws Exception + */ + public static function get() { - if (file_exists(__DIR__ . '/../dynamic/egh.html')) - include __DIR__ . '/../dynamic/egh.html'; + $d = self::create()->loadFromCache(); + if ($d === null) return ""; + return $d; } public static function checkConsistency() diff --git a/www/pages/about.php b/www/pages/about.php index a092057..706750b 100644 --- a/www/pages/about.php +++ b/www/pages/about.php @@ -47,7 +47,7 @@ global $OPTIONS;
- +