<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.1//EN"
"http://www.w3.org/TR/xhtml11/DTD/xhtml11.dtd">
<html xmlns="http://www.w3.org/1999/xhtml">
<head><meta http-equiv="content-type" content="text/html; charset=utf-8" />
<title>[174459] trunk/Websites/perf.webkit.org</title>
</head>
<body>
<style type="text/css"><!--
#msg dl.meta { border: 1px #006 solid; background: #369; padding: 6px; color: #fff; }
#msg dl.meta dt { float: left; width: 6em; font-weight: bold; }
#msg dt:after { content:':';}
#msg dl, #msg dt, #msg ul, #msg li, #header, #footer, #logmsg { font-family: verdana,arial,helvetica,sans-serif; font-size: 10pt; }
#msg dl a { font-weight: bold}
#msg dl a:link { color:#fc3; }
#msg dl a:active { color:#ff0; }
#msg dl a:visited { color:#cc6; }
h3 { font-family: verdana,arial,helvetica,sans-serif; font-size: 10pt; font-weight: bold; }
#msg pre { overflow: auto; background: #ffc; border: 1px #fa0 solid; padding: 6px; }
#logmsg { background: #ffc; border: 1px #fa0 solid; padding: 1em 1em 0 1em; }
#logmsg p, #logmsg pre, #logmsg blockquote { margin: 0 0 1em 0; }
#logmsg p, #logmsg li, #logmsg dt, #logmsg dd { line-height: 14pt; }
#logmsg h1, #logmsg h2, #logmsg h3, #logmsg h4, #logmsg h5, #logmsg h6 { margin: .5em 0; }
#logmsg h1:first-child, #logmsg h2:first-child, #logmsg h3:first-child, #logmsg h4:first-child, #logmsg h5:first-child, #logmsg h6:first-child { margin-top: 0; }
#logmsg ul, #logmsg ol { padding: 0; list-style-position: inside; margin: 0 0 0 1em; }
#logmsg ul { text-indent: -1em; padding-left: 1em; }#logmsg ol { text-indent: -1.5em; padding-left: 1.5em; }
#logmsg > ul, #logmsg > ol { margin: 0 0 1em 0; }
#logmsg pre { background: #eee; padding: 1em; }
#logmsg blockquote { border: 1px solid #fa0; border-left-width: 10px; padding: 1em 1em 0 1em; background: white;}
#logmsg dl { margin: 0; }
#logmsg dt { font-weight: bold; }
#logmsg dd { margin: 0; padding: 0 0 0.5em 0; }
#logmsg dd:before { content:'\00bb';}
#logmsg table { border-spacing: 0px; border-collapse: collapse; border-top: 4px solid #fa0; border-bottom: 1px solid #fa0; background: #fff; }
#logmsg table th { text-align: left; font-weight: normal; padding: 0.2em 0.5em; border-top: 1px dotted #fa0; }
#logmsg table td { text-align: right; border-top: 1px dotted #fa0; padding: 0.2em 0.5em; }
#logmsg table thead th { text-align: center; border-bottom: 1px solid #fa0; }
#logmsg table th.Corner { text-align: left; }
#logmsg hr { border: none 0; border-top: 2px dashed #fa0; height: 1px; }
#header, #footer { color: #fff; background: #636; border: 1px #300 solid; padding: 6px; }
#patch { width: 100%; }
#patch h4 {font-family: verdana,arial,helvetica,sans-serif;font-size:10pt;padding:8px;background:#369;color:#fff;margin:0;}
#patch .propset h4, #patch .binary h4 {margin:0;}
#patch pre {padding:0;line-height:1.2em;margin:0;}
#patch .diff {width:100%;background:#eee;padding: 0 0 10px 0;overflow:auto;}
#patch .propset .diff, #patch .binary .diff {padding:10px 0;}
#patch span {display:block;padding:0 10px;}
#patch .modfile, #patch .addfile, #patch .delfile, #patch .propset, #patch .binary, #patch .copfile {border:1px solid #ccc;margin:10px 0;}
#patch ins {background:#dfd;text-decoration:none;display:block;padding:0 10px;}
#patch del {background:#fdd;text-decoration:none;display:block;padding:0 10px;}
#patch .lines, .info {color:#888;background:#fff;}
--></style>
<div id="msg">
<dl class="meta">
<dt>Revision</dt> <dd><a href="http://trac.webkit.org/projects/webkit/changeset/174459">174459</a></dd>
<dt>Author</dt> <dd>rniwa@webkit.org</dd>
<dt>Date</dt> <dd>2014-10-08 10:20:32 -0700 (Wed, 08 Oct 2014)</dd>
</dl>
<h3>Log Message</h3>
<pre>Perf dashboard should store commit logs
https://bugs.webkit.org/show_bug.cgi?id=137510
Reviewed by Darin Adler.
For the v2 version of the perf dashboard, we would like to be able to see commit logs in the dashboard itself.
This patch replaces "build_revisions" table with "commits" and "build_commits" relations to store commit logs,
and add JSON APIs to report and retrieve them. It also adds a tools/pull-svn.py to pull commit logs from
a subversion directory. The git version of this script will be added in a follow up patch.
In the new database schema, each revision in each repository is represented by exactly one row in "commits"
instead of one row for each build in "build_revisions". "commits" and "builds" now have a proper many-to-many
relationship via "build_commits" relations.
In order to migrate an existing instance of this application, run the following SQL commands:
BEGIN;
INSERT INTO commits (commit_repository, commit_revision, commit_time)
(SELECT DISTINCT ON (revision_repository, revision_value)
revision_repository, revision_value, revision_time FROM build_revisions);
INSERT INTO build_commits (commit_build, build_commit) SELECT revision_build, commit_id
FROM commits, build_revisions
WHERE commit_repository = revision_repository AND commit_revision = revision_value;
DROP TABLE build_revisions;
COMMIT;
The helper script to submit commit logs can be used as follows:
python ./tools/pull-svn.py "WebKit" https://svn.webkit.org/repository/webkit/ https://perf.webkit.org
feeder-slave feeder-slave-password 60 "webkit-patch find-users"
The above command will pull the subversion server at https://svn.webkit.org/repository/webkit/ every 60 seconds
to retrieve at most 10 commits, and submits the results to https://perf.webkit.org using "feeder-slave" and
"feeder-slave-password" as the builder name and the builder password respectively.
The last, optional, argument is the shell command to convert a subversion account to the corresponding username.
e.g. "webkit-patch find-users rniwa@webkit.org" yields "Ryosuke Niwa" <rniwa@webkit.org> in the stdout.
* init-database.sql: Replaced "build_revisions" relation with "commits" and "build_commits" relations.
* public/api/commits.php: Added. Retrieves a list of commits based on arguments in its path of the form
/api/commits/<repository-name>/<filter>. The behavior of this API depends on <filter> as follows:
- Not specified - It returns every single commit for a given repository.
- Matches "oldest" - It returns the commit with the oldest timestamp.
- Matches "latest" - It returns the commit with the latest timestamp.
- Matches "last-reported" - It returns the commit with the latest timestamp added via report-commits.php.
- Is entirely alphanumeric - It returns the commit whose revision matches the filter.
- Is of the form <alphanumeric>:<alphanumeric> or <alphanumeric>-<alphanumeric> - It retrieves the list
of commits added via report-commits.php between two timestamps retrieved from commits whose revisions
match the two alphanumeric values specified. Because it retrieves commits based on their timestamps,
the list may contain commits that do not appear as neither hash's ancestor in git/mercurial.
(main):
(commit_from_revision):
(fetch_commits_between):
(format_commits):
* public/api/report-commits.php: Added. A JSON API to report new subversion, git, or mercurial commits.
See tests/api-report-commits.js for examples on how to use this API.
* public/api/runs.php: Updated the query to use "commit_builds" and "commits" relations instead of
"build_revisions". Regrettably, the new query is 20% slower but I'm going to wait until the new UI is ready
to optimize this and other JSON APIs.
* public/include/db.php:
(Database::select_or_insert_row):
(Database::update_or_insert_row): Added.
(Database::_select_update_or_insert_row): Extracted from select_or_insert_row. Try to update first and then
insert if the update fails for update_or_insert_row. Preserves the old behavior when $should_update is false.
(Database::select_first_row):
(Database::select_last_row): Added.
(Database::select_first_or_last_row): Extracted from select_first_row. Fixed a bug that we were asserting
$order_by to be not alphanumeric/underscore. Retrieve the last row instead of the first if $descending_order.
* public/include/report-processor.php:
(ReportProcessor::resolve_build_id): Store commits instead of build_revisions. We don't worry about the race
condition for adding "build_commits" rows since we shouldn't have a single tester submitting the same result
concurrently. Even if it happened, it will only result in a PHP error and the database will stay consistent.
* run-tests.js:
(pathToTests): Don't call path.resolve with "undefined" testName; It throws an exception in the latest node.js.
* tests/api-report-commits.js: Added.
* tests/api-report.js: Fixed a test per build_revisions to build_commits/commits replacement.
* tools: Added.
* tools/pull-svn.py: Added. See above for how to use this script.
(main):
(determine_first_revision_to_fetch):
(fetch_revision_from_dasbhoard):
(fetch_commit_and_resolve_author):
(fetch_commit):
(textContent):
(resolve_author_name_from_email):
(submit_commits):</pre>
<h3>Modified Paths</h3>
<ul>
<li><a href="#trunkWebsitesperfwebkitorgChangeLog">trunk/Websites/perf.webkit.org/ChangeLog</a></li>
<li><a href="#trunkWebsitesperfwebkitorginitdatabasesql">trunk/Websites/perf.webkit.org/init-database.sql</a></li>
<li><a href="#trunkWebsitesperfwebkitorgpublicapirunsphp">trunk/Websites/perf.webkit.org/public/api/runs.php</a></li>
<li><a href="#trunkWebsitesperfwebkitorgpublicincludedbphp">trunk/Websites/perf.webkit.org/public/include/db.php</a></li>
<li><a href="#trunkWebsitesperfwebkitorgpublicincludereportprocessorphp">trunk/Websites/perf.webkit.org/public/include/report-processor.php</a></li>
<li><a href="#trunkWebsitesperfwebkitorgruntestsjs">trunk/Websites/perf.webkit.org/run-tests.js</a></li>
<li><a href="#trunkWebsitesperfwebkitorgtestsapireportjs">trunk/Websites/perf.webkit.org/tests/api-report.js</a></li>
</ul>
<h3>Added Paths</h3>
<ul>
<li><a href="#trunkWebsitesperfwebkitorgpublicapicommitsphp">trunk/Websites/perf.webkit.org/public/api/commits.php</a></li>
<li><a href="#trunkWebsitesperfwebkitorgpublicapireportcommitsphp">trunk/Websites/perf.webkit.org/public/api/report-commits.php</a></li>
<li><a href="#trunkWebsitesperfwebkitorgtestsapireportcommitsjs">trunk/Websites/perf.webkit.org/tests/api-report-commits.js</a></li>
<li>trunk/Websites/perf.webkit.org/tools/</li>
<li><a href="#trunkWebsitesperfwebkitorgtoolspullsvnpy">trunk/Websites/perf.webkit.org/tools/pull-svn.py</a></li>
</ul>
</div>
<div id="patch">
<h3>Diff</h3>
<a id="trunkWebsitesperfwebkitorgChangeLog"></a>
<div class="modfile"><h4>Modified: trunk/Websites/perf.webkit.org/ChangeLog (174458 => 174459)</h4>
<pre class="diff"><span>
<span class="info">--- trunk/Websites/perf.webkit.org/ChangeLog 2014-10-08 17:16:25 UTC (rev 174458)
+++ trunk/Websites/perf.webkit.org/ChangeLog 2014-10-08 17:20:32 UTC (rev 174459)
</span><span class="lines">@@ -1,3 +1,110 @@
</span><ins>+2014-10-08 Ryosuke Niwa <rniwa@webkit.org>
+
+ Perf dashboard should store commit logs
+ https://bugs.webkit.org/show_bug.cgi?id=137510
+
+ Reviewed by Darin Adler.
+
+ For the v2 version of the perf dashboard, we would like to be able to see commit logs in the dashboard itself.
+
+ This patch replaces "build_revisions" table with "commits" and "build_commits" relations to store commit logs,
+ and add JSON APIs to report and retrieve them. It also adds a tools/pull-svn.py to pull commit logs from
+ a subversion directory. The git version of this script will be added in a follow up patch.
+
+
+ In the new database schema, each revision in each repository is represented by exactly one row in "commits"
+ instead of one row for each build in "build_revisions". "commits" and "builds" now have a proper many-to-many
+ relationship via "build_commits" relations.
+
+ In order to migrate an existing instance of this application, run the following SQL commands:
+
+ BEGIN;
+
+ INSERT INTO commits (commit_repository, commit_revision, commit_time)
+ (SELECT DISTINCT ON (revision_repository, revision_value)
+ revision_repository, revision_value, revision_time FROM build_revisions);
+
+ INSERT INTO build_commits (commit_build, build_commit) SELECT revision_build, commit_id
+ FROM commits, build_revisions
+ WHERE commit_repository = revision_repository AND commit_revision = revision_value;
+
+ DROP TABLE build_revisions;
+
+ COMMIT;
+
+
+ The helper script to submit commit logs can be used as follows:
+
+ python ./tools/pull-svn.py "WebKit" https://svn.webkit.org/repository/webkit/ https://perf.webkit.org
+ feeder-slave feeder-slave-password 60 "webkit-patch find-users"
+
+ The above command will pull the subversion server at https://svn.webkit.org/repository/webkit/ every 60 seconds
+ to retrieve at most 10 commits, and submits the results to https://perf.webkit.org using "feeder-slave" and
+ "feeder-slave-password" as the builder name and the builder password respectively.
+
+ The last, optional, argument is the shell command to convert a subversion account to the corresponding username.
+ e.g. "webkit-patch find-users rniwa@webkit.org" yields "Ryosuke Niwa" <rniwa@webkit.org> in the stdout.
+
+
+ * init-database.sql: Replaced "build_revisions" relation with "commits" and "build_commits" relations.
+
+ * public/api/commits.php: Added. Retrieves a list of commits based on arguments in its path of the form
+ /api/commits/<repository-name>/<filter>. The behavior of this API depends on <filter> as follows:
+
+ - Not specified - It returns every single commit for a given repository.
+ - Matches "oldest" - It returns the commit with the oldest timestamp.
+ - Matches "latest" - It returns the commit with the latest timestamp.
+ - Matches "last-reported" - It returns the commit with the latest timestamp added via report-commits.php.
+ - Is entirely alphanumeric - It returns the commit whose revision matches the filter.
+ - Is of the form <alphanumeric>:<alphanumeric> or <alphanumeric>-<alphanumeric> - It retrieves the list
+ of commits added via report-commits.php between two timestamps retrieved from commits whose revisions
+ match the two alphanumeric values specified. Because it retrieves commits based on their timestamps,
+ the list may contain commits that do not appear as neither hash's ancestor in git/mercurial.
+ (main):
+ (commit_from_revision):
+ (fetch_commits_between):
+ (format_commits):
+
+ * public/api/report-commits.php: Added. A JSON API to report new subversion, git, or mercurial commits.
+ See tests/api-report-commits.js for examples on how to use this API.
+
+ * public/api/runs.php: Updated the query to use "commit_builds" and "commits" relations instead of
+ "build_revisions". Regrettably, the new query is 20% slower but I'm going to wait until the new UI is ready
+ to optimize this and other JSON APIs.
+
+ * public/include/db.php:
+ (Database::select_or_insert_row):
+ (Database::update_or_insert_row): Added.
+ (Database::_select_update_or_insert_row): Extracted from select_or_insert_row. Try to update first and then
+ insert if the update fails for update_or_insert_row. Preserves the old behavior when $should_update is false.
+
+ (Database::select_first_row):
+ (Database::select_last_row): Added.
+ (Database::select_first_or_last_row): Extracted from select_first_row. Fixed a bug that we were asserting
+ $order_by to be not alphanumeric/underscore. Retrieve the last row instead of the first if $descending_order.
+
+ * public/include/report-processor.php:
+ (ReportProcessor::resolve_build_id): Store commits instead of build_revisions. We don't worry about the race
+ condition for adding "build_commits" rows since we shouldn't have a single tester submitting the same result
+ concurrently. Even if it happened, it will only result in a PHP error and the database will stay consistent.
+
+ * run-tests.js:
+ (pathToTests): Don't call path.resolve with "undefined" testName; It throws an exception in the latest node.js.
+
+ * tests/api-report-commits.js: Added.
+ * tests/api-report.js: Fixed a test per build_revisions to build_commits/commits replacement.
+
+ * tools: Added.
+ * tools/pull-svn.py: Added. See above for how to use this script.
+ (main):
+ (determine_first_revision_to_fetch):
+ (fetch_revision_from_dasbhoard):
+ (fetch_commit_and_resolve_author):
+ (fetch_commit):
+ (textContent):
+ (resolve_author_name_from_email):
+ (submit_commits):
+
</ins><span class="cx"> 2014-09-30 Ryosuke Niwa <rniwa@webkit.org>
</span><span class="cx">
</span><span class="cx"> Update Install.md for Mavericks and fix typos
</span></span></pre></div>
<a id="trunkWebsitesperfwebkitorginitdatabasesql"></a>
<div class="modfile"><h4>Modified: trunk/Websites/perf.webkit.org/init-database.sql (174458 => 174459)</h4>
<pre class="diff"><span>
<span class="info">--- trunk/Websites/perf.webkit.org/init-database.sql 2014-10-08 17:16:25 UTC (rev 174458)
+++ trunk/Websites/perf.webkit.org/init-database.sql 2014-10-08 17:20:32 UTC (rev 174459)
</span><span class="lines">@@ -3,8 +3,9 @@
</span><span class="cx"> DROP TABLE test_configurations CASCADE;
</span><span class="cx"> DROP TYPE test_configuration_type CASCADE;
</span><span class="cx"> DROP TABLE aggregators CASCADE;
</span><del>-DROP TABLE build_revisions CASCADE;
</del><span class="cx"> DROP TABLE builds CASCADE;
</span><ins>+DROP TABLE commits CASCADE;
+DROP TABLE build_commits CASCADE;
</ins><span class="cx"> DROP TABLE builders CASCADE;
</span><span class="cx"> DROP TABLE repositories CASCADE;
</span><span class="cx"> DROP TABLE platforms CASCADE;
</span><span class="lines">@@ -50,15 +51,27 @@
</span><span class="cx"> CONSTRAINT builder_build_time_tuple_must_be_unique UNIQUE(build_builder, build_number, build_time));
</span><span class="cx"> CREATE INDEX build_builder_index ON builds(build_builder);
</span><span class="cx">
</span><del>-CREATE TABLE build_revisions (
- revision_build integer NOT NULL REFERENCES builds ON DELETE CASCADE,
- revision_repository integer NOT NULL REFERENCES repositories ON DELETE CASCADE,
- revision_value varchar(64) NOT NULL,
- revision_time timestamp,
- PRIMARY KEY (revision_repository, revision_build));
-CREATE INDEX revision_build_index ON build_revisions(revision_build);
-CREATE INDEX revision_repository_index ON build_revisions(revision_repository);
</del><ins>+CREATE TABLE commits (
+ commit_id serial PRIMARY KEY,
+ commit_repository integer NOT NULL REFERENCES repositories ON DELETE CASCADE,
+ commit_revision varchar(64) NOT NULL,
+ commit_parent integer REFERENCES commits ON DELETE CASCADE,
+ commit_time timestamp,
+ commit_author_name varchar(128),
+ commit_author_email varchar(320),
+ commit_message text,
+ commit_reported boolean NOT NULL DEFAULT FALSE,
+ CONSTRAINT commit_in_repository_must_be_unique UNIQUE(commit_repository, commit_revision));
+CREATE INDEX commit_time_index ON commits(commit_time);
+CREATE INDEX commit_author_name_index ON commits(commit_author_name);
+CREATE INDEX commit_author_email_index ON commits(commit_author_email);
</ins><span class="cx">
</span><ins>+CREATE TABLE build_commits (
+ commit_build integer NOT NULL REFERENCES builds ON DELETE CASCADE,
+ build_commit integer NOT NULL REFERENCES commits ON DELETE CASCADE
+ PRIMARY KEY (commit_build, build_commit));
+CREATE INDEX build_commits_index ON build_commits(commit_build, build_commit);
+
</ins><span class="cx"> CREATE TABLE aggregators (
</span><span class="cx"> aggregator_id serial PRIMARY KEY,
</span><span class="cx"> aggregator_name varchar(64),
</span></span></pre></div>
<a id="trunkWebsitesperfwebkitorgpublicapicommitsphp"></a>
<div class="addfile"><h4>Added: trunk/Websites/perf.webkit.org/public/api/commits.php (0 => 174459)</h4>
<pre class="diff"><span>
<span class="info">--- trunk/Websites/perf.webkit.org/public/api/commits.php (rev 0)
+++ trunk/Websites/perf.webkit.org/public/api/commits.php 2014-10-08 17:20:32 UTC (rev 174459)
</span><span class="lines">@@ -0,0 +1,84 @@
</span><ins>+<?php
+
+require_once('../include/json-header.php');
+
+function main($paths) {
+ if (count($paths) < 1 || count($paths) > 2)
+ exit_with_error('InvalidRequest');
+
+ $db = new Database;
+ if (!$db->connect())
+ exit_with_error('DatabaseConnectionFailure');
+
+ $repository_name = $paths[0];
+ $repository_row = $db->select_first_row('repositories', 'repository', array('name' => $repository_name));
+ if (!$repository_row)
+ exit_with_error('RepositoryNotFound', array('repositoryName' => $repository_name));
+ $repository_id = $repository_row['repository_id'];
+
+ $filter = array_get($paths, 1);
+ $single_commit = NULL;
+ $commits = array();
+ if (!$filter) {
+ $commits = $db->fetch_table('commits', 'commit_time');
+ } else if ($filter == 'oldest') {
+ $single_commit = $db->select_first_row('commits', 'commit', array('repository' => $repository_id), 'time');
+ } else if ($filter == 'latest') {
+ $single_commit = $db->select_last_row('commits', 'commit', array('repository' => $repository_id), 'time');
+ } else if ($filter == 'last-reported') {
+ $single_commit = $db->select_last_row('commits', 'commit', array('repository' => $repository_id, 'reported' => true), 'time');
+ } else if (ctype_alnum($filter)) {
+ $single_commit = commit_from_revision($db, $repository_id, $repository_name, $filter);
+ } else {
+ $matches = array();
+ if (!preg_match('/([A-Za-z0-9]+)[\:\-]([A-Za-z0-9]+)/', $filter, $matches))
+ exit_with_error('UnknownFilter', array('repositoryName' => $repository_name, 'filter' => $filter));
+
+ $first = commit_from_revision($db, $repository_id, $matches[1])['commit_time'];
+ $second = commit_from_revision($db, $repository_id, $matches[2])['commit_time'];
+ $in_order = $first < $second;
+
+ $commits = fetch_commits_between($db, $repository_id, $in_order ? $first : $second, $in_order ? $second : $first);
+ }
+
+ exit_with_success(array('commits' => format_commits($single_commit ? array($single_commit) : $commits)));
+}
+
+function commit_from_revision($db, $repository_id, $revision) {
+ $all_but_first = substr($revision, 1);
+ if ($revision[0] == 'r' && ctype_digit($all_but_first))
+ $revision = $all_but_first;
+ $commit_info = array('repository' => $repository_id, 'revision' => $revision);
+ $row = $db->select_last_row('commits', 'commit', $commit_info);
+ if (!$row)
+ exit_with_error('UnknownCommit', $commit_info);
+ return $row;
+}
+
+function fetch_commits_between($db, $repository_id, $from, $to) {
+ $commits = $db->query_and_fetch_all('SELECT * FROM commits
+ WHERE commit_repository = $1 AND commit_time >= $2 AND commit_time <= $3 AND commit_reported = true ORDER BY commit_time',
+ array($repository_id, $from, $to));
+ if (!$commits)
+ exit_with_error('FailedToFetchCommits', array('repository' => $repository_id, 'from' => $from, 'to' => $to));
+ return $commits;
+}
+
+function format_commits($commits) {
+ $formatted_commits = array();
+ foreach ($commits as $commit_row) {
+ array_push($formatted_commits, array(
+ 'id' => $commit_row['commit_id'],
+ 'revision' => $commit_row['commit_revision'],
+ 'parent' => $commit_row['commit_parent'],
+ 'time' => $commit_row['commit_time'],
+ 'author' => array('name' => $commit_row['commit_author_name'], 'email' => $commit_row['commit_author_email']),
+ 'message' => $commit_row['commit_message']
+ ));
+ }
+ return $formatted_commits;
+}
+
+main(array_key_exists('PATH_INFO', $_SERVER) ? explode('/', trim($_SERVER['PATH_INFO'], '/')) : array());
+
+?>
</ins></span></pre></div>
<a id="trunkWebsitesperfwebkitorgpublicapireportcommitsphp"></a>
<div class="addfile"><h4>Added: trunk/Websites/perf.webkit.org/public/api/report-commits.php (0 => 174459)</h4>
<pre class="diff"><span>
<span class="info">--- trunk/Websites/perf.webkit.org/public/api/report-commits.php (rev 0)
+++ trunk/Websites/perf.webkit.org/public/api/report-commits.php 2014-10-08 17:20:32 UTC (rev 174459)
</span><span class="lines">@@ -0,0 +1,162 @@
</span><ins>+<?php
+
+require('../include/json-header.php');
+
+function main($post_data) {
+ $db = new Database;
+ if (!$db->connect())
+ exit_with_error('DatabaseConnectionFailure');
+
+ $report = json_decode($post_data, true);
+
+ verify_builder($db, $report);
+
+ $commits = array_get($report, 'commits', array());
+
+ foreach ($commits as $commit_info) {
+ if (!array_key_exists('repository', $commit_info))
+ exit_with_error('MissingRepositoryName', array('commit' => $commit_info));
+ if (!array_key_exists('revision', $commit_info))
+ exit_with_error('MissingRevision', array('commit' => $commit_info));
+ if (!ctype_alnum($commit_info['revision']))
+ exit_with_error('InvalidRevision', array('commit' => $commit_info));
+ if (!array_key_exists('time', $commit_info))
+ exit_with_error('MissingTimestamp', array('commit' => $commit_info));
+ if (!array_key_exists('author', $commit_info) || !is_array($commit_info['author']))
+ exit_with_error('MissingAuthorOrInvalidFormat', array('commit' => $commit_info));
+ }
+
+ $db->begin_transaction();
+ foreach ($commits as $commit_info) {
+ $repository_id = $db->select_or_insert_row('repositories', 'repository', array('name' => $commit_info['repository']));
+ if (!$repository_id) {
+ $db->rollback_transaction();
+ exit_with_error('FailedToInsertRepository', array('commit' => $commit_info));
+ }
+
+ $parent_revision = array_get($commit_info, 'parent');
+ $parent_id = NULL;
+ if ($parent_revision) {
+ $parent_commit = $db->select_first_row('commits', 'commit', array('repository' => $repository_id, 'revision' => $parent_revision));
+ if (!$parent_commit) {
+ $db->rollback_transaction();
+ exit_with_error('FailedToFindParentCommit', array('commit' => $commit_info));
+ }
+ $parent_id = $parent_commit['commit_id'];
+ }
+
+ $data = array(
+ 'repository' => $repository_id,
+ 'revision' => $commit_info['revision'],
+ 'parent' => $parent_id,
+ 'time' => $commit_info['time'],
+ 'author_name' => array_get($commit_info['author'], 'name'),
+ 'author_email' => array_get($commit_info['author'], 'email'),
+ 'message' => $commit_info['message'],
+ 'reported' => true,
+ );
+ $db->update_or_insert_row('commits', 'commit', array('repository' => $repository_id, 'revision' => $data['revision']), $data);
+ }
+ $db->commit_transaction();
+
+ exit_with_success();
+}
+
+function verify_builder($db, $report) {
+ array_key_exists('builderName', $report) or exit_with_error('MissingBuilderName');
+ array_key_exists('builderPassword', $report) or exit_with_error('MissingBuilderPassword');
+
+ $builder_info = array(
+ 'name' => $report['builderName'],
+ 'password_hash' => hash('sha256', $report['builderPassword'])
+ );
+
+ $matched_builder = $db->select_first_row('builders', 'builder', $builder_info);
+ if (!$matched_builder)
+ exit_with_error('BuilderNotFound', array('name' => $builder_info['name']));
+}
+
+main($HTTP_RAW_POST_DATA);
+
+?>
+<?php
+
+require('../include/json-header.php');
+
+function main($post_data) {
+ $db = new Database;
+ if (!$db->connect())
+ exit_with_error('DatabaseConnectionFailure');
+
+ $report = json_decode($post_data, true);
+
+ verify_builder($db, $report);
+
+ $commits = array_get($report, 'commits', array());
+
+ foreach ($commits as $commit_info) {
+ if (!array_key_exists('repository', $commit_info))
+ exit_with_error('MissingRepositoryName', array('commit' => $commit_info));
+ if (!array_key_exists('revision', $commit_info))
+ exit_with_error('MissingRevision', array('commit' => $commit_info));
+ if (!ctype_alnum($commit_info['revision']))
+ exit_with_error('InvalidRevision', array('commit' => $commit_info));
+ if (!array_key_exists('time', $commit_info))
+ exit_with_error('MissingTimestamp', array('commit' => $commit_info));
+ if (!array_key_exists('author', $commit_info) || !is_array($commit_info['author']))
+ exit_with_error('MissingAuthorOrInvalidFormat', array('commit' => $commit_info));
+ }
+
+ $db->begin_transaction();
+ foreach ($commits as $commit_info) {
+ $repository_id = $db->select_or_insert_row('repositories', 'repository', array('name' => $commit_info['repository']));
+ if (!$repository_id) {
+ $db->rollback_transaction();
+ exit_with_error('FailedToInsertRepository', array('commit' => $commit_info));
+ }
+
+ $parent_revision = array_get($commit_info, 'parent');
+ $parent_id = NULL;
+ if ($parent_revision) {
+ $parent_commit = $db->select_first_row('commits', 'commit', array('repository' => $repository_id, 'revision' => $parent_revision));
+ if (!$parent_commit) {
+ $db->rollback_transaction();
+ exit_with_error('FailedToFindParentCommit', array('commit' => $commit_info));
+ }
+ $parent_id = $parent_commit['commit_id'];
+ }
+
+ $data = array(
+ 'repository' => $repository_id,
+ 'revision' => $commit_info['revision'],
+ 'parent' => $parent_id,
+ 'time' => $commit_info['time'],
+ 'author_name' => array_get($commit_info['author'], 'name'),
+ 'author_email' => array_get($commit_info['author'], 'email'),
+ 'message' => $commit_info['message'],
+ 'reported' => true,
+ );
+ $db->update_or_insert_row('commits', 'commit', array('repository' => $repository_id, 'revision' => $data['revision']), $data);
+ }
+ $db->commit_transaction();
+
+ exit_with_success();
+}
+
+function verify_builder($db, $report) {
+ array_key_exists('builderName', $report) or exit_with_error('MissingBuilderName');
+ array_key_exists('builderPassword', $report) or exit_with_error('MissingBuilderPassword');
+
+ $builder_info = array(
+ 'name' => $report['builderName'],
+ 'password_hash' => hash('sha256', $report['builderPassword'])
+ );
+
+ $matched_builder = $db->select_first_row('builders', 'builder', $builder_info);
+ if (!$matched_builder)
+ exit_with_error('BuilderNotFound', array('name' => $builder_info['name']));
+}
+
+main($HTTP_RAW_POST_DATA);
+
+?>
</ins></span></pre></div>
<a id="trunkWebsitesperfwebkitorgpublicapirunsphp"></a>
<div class="modfile"><h4>Modified: trunk/Websites/perf.webkit.org/public/api/runs.php (174458 => 174459)</h4>
<pre class="diff"><span>
<span class="info">--- trunk/Websites/perf.webkit.org/public/api/runs.php 2014-10-08 17:16:25 UTC (rev 174458)
+++ trunk/Websites/perf.webkit.org/public/api/runs.php 2014-10-08 17:20:32 UTC (rev 174459)
</span><span class="lines">@@ -30,12 +30,10 @@
</span><span class="cx">
</span><span class="cx"> function fetch_runs_for_config($db, $config) {
</span><span class="cx"> $raw_runs = $db->query_and_fetch_all('
</span><del>- SELECT test_runs.*, builds.*, array_agg((revision_repository, revision_value, revision_time)) AS revisions
- FROM builds LEFT OUTER JOIN build_revisions ON revision_build = build_id, test_runs
- WHERE run_build = build_id AND run_config = $1
- GROUP BY build_id, build_builder, build_number, build_time, build_latest_revision,
- run_id, run_config, run_build, run_iteration_count_cache,
- run_mean_cache, run_sum_cache, run_square_sum_cache', array($config['config_id']));
</del><ins>+ SELECT test_runs.*, builds.*, array_agg((commit_repository, commit_revision, commit_time)) AS revisions
+ FROM builds LEFT OUTER JOIN build_commits ON commit_build = build_id, test_runs, commits
+ WHERE run_build = build_id AND run_config = $1 AND build_commit = commit_id
+ GROUP BY build_id, run_id', array($config['config_id']));
</ins><span class="cx">
</span><span class="cx"> $formatted_runs = array();
</span><span class="cx"> if (!$raw_runs)
</span></span></pre></div>
<a id="trunkWebsitesperfwebkitorgpublicincludedbphp"></a>
<div class="modfile"><h4>Modified: trunk/Websites/perf.webkit.org/public/include/db.php (174458 => 174459)</h4>
<pre class="diff"><span>
<span class="info">--- trunk/Websites/perf.webkit.org/public/include/db.php 2014-10-08 17:16:25 UTC (rev 174458)
+++ trunk/Websites/perf.webkit.org/public/include/db.php 2014-10-08 17:20:32 UTC (rev 174459)
</span><span class="lines">@@ -104,6 +104,14 @@
</span><span class="cx"> }
</span><span class="cx">
</span><span class="cx"> function select_or_insert_row($table, $prefix, $select_params, $insert_params = NULL, $returning = 'id') {
</span><ins>+ return $this->_select_update_or_insert_row($table, $prefix, $select_params, $insert_params, $returning, FALSE);
+ }
+
+ function update_or_insert_row($table, $prefix, $select_params, $insert_params = NULL, $returning = 'id') {
+ return $this->_select_update_or_insert_row($table, $prefix, $select_params, $insert_params, $returning, TRUE);
+ }
+
+ private function _select_update_or_insert_row($table, $prefix, $select_params, $insert_params, $returning, $should_update) {
</ins><span class="cx"> $values = array();
</span><span class="cx">
</span><span class="cx"> $select_placeholders = array();
</span><span class="lines">@@ -124,23 +132,42 @@
</span><span class="cx">
</span><span class="cx"> $insert_column_names = $this->prefixed_column_names($insert_column_names, $prefix);
</span><span class="cx"> $insert_placeholders = join(', ', $insert_placeholders);
</span><del>- $rows = $this->query_and_fetch_all("INSERT INTO $table ($insert_column_names) SELECT $insert_placeholders WHERE NOT EXISTS
- ($query) RETURNING $returning_column_name", $values);
- if (!$rows)
</del><ins>+
+ // http://stackoverflow.com/questions/1109061/insert-on-duplicate-update-in-postgresql
+ $rows = NULL;
+ if ($should_update) {
+ $rows = $this->query_and_fetch_all("UPDATE $table SET ($insert_column_names) = ($insert_placeholders)
+ WHERE ($select_column_names) = ($select_placeholders) RETURNING $returning_column_name", $values);
+ }
+ if (!$rows) {
+ $rows = $this->query_and_fetch_all("INSERT INTO $table ($insert_column_names) SELECT $insert_placeholders
+ WHERE NOT EXISTS ($query) RETURNING $returning_column_name", $values);
+ }
+ if (!$should_update && !$rows)
</ins><span class="cx"> $rows = $this->query_and_fetch_all($query, $select_values);
</span><span class="cx">
</span><span class="cx"> return $rows ? ($returning == '*' ? $rows[0] : $rows[0][$returning_column_name]) : NULL;
</span><span class="cx"> }
</span><span class="cx">
</span><span class="cx"> function select_first_row($table, $prefix, $params, $order_by = NULL) {
</span><ins>+ return $this->select_first_or_last_row($table, $prefix, $params, $order_by, FALSE);
+ }
+
+ function select_last_row($table, $prefix, $params, $order_by = NULL) {
+ return $this->select_first_or_last_row($table, $prefix, $params, $order_by, TRUE);
+ }
+
+ private function select_first_or_last_row($table, $prefix, $params, $order_by, $descending_order) {
</ins><span class="cx"> $placeholders = array();
</span><span class="cx"> $values = array();
</span><span class="cx"> $column_names = $this->prefixed_column_names($this->prepare_params($params, $placeholders, $values), $prefix);
</span><span class="cx"> $placeholders = join(', ', $placeholders);
</span><span class="cx"> $query = "SELECT * FROM $table WHERE ($column_names) = ($placeholders)";
</span><span class="cx"> if ($order_by) {
</span><del>- assert(!ctype_alnum_underscore($order_by));
</del><ins>+ assert(ctype_alnum_underscore($order_by));
</ins><span class="cx"> $query .= ' ORDER BY ' . $this->prefixed_name($order_by, $prefix);
</span><ins>+ if ($descending_order)
+ $query .= ' DESC';
</ins><span class="cx"> }
</span><span class="cx"> $rows = $this->query_and_fetch_all($query . ' LIMIT 1', $values);
</span><span class="cx">
</span></span></pre></div>
<a id="trunkWebsitesperfwebkitorgpublicincludereportprocessorphp"></a>
<div class="modfile"><h4>Modified: trunk/Websites/perf.webkit.org/public/include/report-processor.php (174458 => 174459)</h4>
<pre class="diff"><span>
<span class="info">--- trunk/Websites/perf.webkit.org/public/include/report-processor.php 2014-10-08 17:16:25 UTC (rev 174458)
+++ trunk/Websites/perf.webkit.org/public/include/report-processor.php 2014-10-08 17:20:32 UTC (rev 174459)
</span><span class="lines">@@ -98,13 +98,24 @@
</span><span class="cx"> if (!$repository_id)
</span><span class="cx"> $this->exit_with_error('FailedToInsertRepository', array('name' => $repository_name));
</span><span class="cx">
</span><del>- $revision_data = array('repository' => $repository_id, 'build' => $build_id, 'value' => $revision_data['revision'],
- 'time' => array_get($revision_data, 'timestamp'));
- $revision_row = $this->db->select_or_insert_row('build_revisions', 'revision', array('repository' => $repository_id, 'build' => $build_id), $revision_data, '*');
- if (!$revision_row)
- $this->exit_with_error('FailedToInsertRevision', $revision_data);
- if ($revision_row['revision_value'] != $revision_data['value'])
- $this->exit_with_error('MismatchingRevisionData', array('existing' => $revision_row, 'new' => $revision_data));
</del><ins>+ $commit_data = array('repository' => $repository_id, 'revision' => $revision_data['revision'], 'time' => array_get($revision_data, 'timestamp'));
+
+ $mismatching_commit = $this->db->query_and_fetch_all('SELECT * FROM build_commits, commits
+ WHERE build_commit = commit_id AND commit_build = $1 AND commit_repository = $2 AND commit_revision != $3 LIMIT 1',
+ array($build_id, $repository_id, $revision_data['revision']));
+ if ($mismatching_commit)
+ $this->exit_with_error('MismatchingCommitRevision', array('build' => $build_id, 'existing' => $mismatching_commit, 'new' => $commit_data));
+
+ $commit_row = $this->db->select_or_insert_row('commits', 'commit',
+ array('repository' => $repository_id, 'revision' => $revision_data['revision']), $commit_data, '*');
+ if (!$commit_row)
+ $this->exit_with_error('FailedToRecordCommit', $commit_data);
+ if (abs($commit_row['commit_time'] - $commit_data['time']) > 1.0)
+ $this->exit_with_error('MismatchingCommitTime', array('existing' => $commit_row, 'new' => $commit_data));
+
+ if (!$this->db->insert_row('build_commits', null,
+ array('commit_build' => $build_id, 'build_commit' => $commit_row['commit_id']), null))
+ $this->exit_with_error('FailedToRelateCommitToBuild', array('commit' => $commit_row, 'build' => $build_id));
</ins><span class="cx"> }
</span><span class="cx">
</span><span class="cx"> return $build_id;
</span></span></pre></div>
<a id="trunkWebsitesperfwebkitorgruntestsjs"></a>
<div class="modfile"><h4>Modified: trunk/Websites/perf.webkit.org/run-tests.js (174458 => 174459)</h4>
<pre class="diff"><span>
<span class="info">--- trunk/Websites/perf.webkit.org/run-tests.js 2014-10-08 17:16:25 UTC (rev 174458)
+++ trunk/Websites/perf.webkit.org/run-tests.js 2014-10-08 17:20:32 UTC (rev 174459)
</span><span class="lines">@@ -30,7 +30,7 @@
</span><span class="cx"> }
</span><span class="cx">
</span><span class="cx"> function pathToTests(testName) {
</span><del>- return path.resolve(__dirname, 'tests', testName);
</del><ins>+ return testName ? path.resolve(__dirname, 'tests', testName) : path.resolve(__dirname, 'tests');
</ins><span class="cx"> }
</span><span class="cx">
</span><span class="cx"> var configurationJSON = require('./config.json');
</span></span></pre></div>
<a id="trunkWebsitesperfwebkitorgtestsapireportcommitsjs"></a>
<div class="addfile"><h4>Added: trunk/Websites/perf.webkit.org/tests/api-report-commits.js (0 => 174459)</h4>
<pre class="diff"><span>
<span class="info">--- trunk/Websites/perf.webkit.org/tests/api-report-commits.js (rev 0)
+++ trunk/Websites/perf.webkit.org/tests/api-report-commits.js 2014-10-08 17:20:32 UTC (rev 174459)
</span><span class="lines">@@ -0,0 +1,219 @@
</span><ins>+describe("/api/report-commits/", function () {
+ var emptyReport = {
+ "builderName": "someBuilder",
+ "builderPassword": "somePassword",
+ };
+ var subversionCommit = {
+ "builderName": "someBuilder",
+ "builderPassword": "somePassword",
+ "commits": [
+ {
+ "repository": "WebKit",
+ "revision": "141977",
+ "time": "2013-02-06T08:55:20.9Z",
+ "author": {"name": "Commit Queue", "email": "commit-queue@webkit.org"},
+ "message": "some message",
+ }
+ ],
+ };
+ var subversionInvalidCommit = {
+ "builderName": "someBuilder",
+ "builderPassword": "somePassword",
+ "commits": [
+ {
+ "repository": "WebKit",
+ "revision": "_141977",
+ "time": "2013-02-06T08:55:20.9Z",
+ "author": {"name": "Commit Queue", "email": "commit-queue@webkit.org"},
+ "message": "some message",
+ }
+ ],
+ };
+ var subversionTwoCommits = {
+ "builderName": "someBuilder",
+ "builderPassword": "somePassword",
+ "commits": [
+ {
+ "repository": "WebKit",
+ "revision": "141977",
+ "time": "2013-02-06T08:55:20.9Z",
+ "author": {"name": "Commit Queue", "email": "commit-queue@webkit.org"},
+ "message": "some message",
+ },
+ {
+ "repository": "WebKit",
+ "parent": "141977",
+ "revision": "141978",
+ "time": "2013-02-06T09:54:56.0Z",
+ "author": {"name": "Mikhail Pozdnyakov", "email": "mikhail.pozdnyakov@intel.com"},
+ "message": "another message",
+ }
+ ]
+ }
+
+ function addBuilder(report, callback) {
+ queryAndFetchAll('INSERT INTO builders (builder_name, builder_password_hash) values ($1, $2)',
+ [report.builderName, sha256(report.builderPassword)], callback);
+ }
+
+ it("should reject error when builder name is missing", function () {
+ postJSON('/api/report-commits/', {}, function (response) {
+ assert.equal(response.statusCode, 200);
+ assert.equal(JSON.parse(response.responseText)['status'], 'MissingBuilderName');
+ notifyDone();
+ });
+ });
+
+ it("should reject when there are no builders", function () {
+ postJSON('/api/report-commits/', emptyReport, function (response) {
+ assert.equal(response.statusCode, 200);
+ assert.notEqual(JSON.parse(response.responseText)['status'], 'OK');
+
+ queryAndFetchAll('SELECT COUNT(*) from commits', [], function (rows) {
+ assert.equal(rows[0].count, 0);
+ notifyDone();
+ });
+ });
+ });
+
+ it("should accept an empty report", function () {
+ addBuilder(emptyReport, function () {
+ postJSON('/api/report-commits/', emptyReport, function (response) {
+ assert.equal(response.statusCode, 200);
+ assert.equal(JSON.parse(response.responseText)['status'], 'OK');
+ notifyDone();
+ });
+ });
+ });
+
+ it("should add a missing repository", function () {
+ addBuilder(subversionCommit, function () {
+ postJSON('/api/report-commits/', subversionCommit, function (response) {
+ assert.equal(response.statusCode, 200);
+ assert.equal(JSON.parse(response.responseText)['status'], 'OK');
+ queryAndFetchAll('SELECT * FROM repositories', [], function (rows) {
+ assert.equal(rows.length, 1);
+ assert.equal(rows[0]['repository_name'], subversionCommit.commits[0]['repository']);
+ notifyDone();
+ });
+ });
+ });
+ });
+
+ it("should store a commit from a valid builder", function () {
+ addBuilder(subversionCommit, function () {
+ postJSON('/api/report-commits/', subversionCommit, function (response) {
+ assert.equal(response.statusCode, 200);
+ assert.equal(JSON.parse(response.responseText)['status'], 'OK');
+ queryAndFetchAll('SELECT * FROM commits', [], function (rows) {
+ assert.equal(rows.length, 1);
+ var reportedData = subversionCommit.commits[0];
+ assert.equal(rows[0]['commit_revision'], reportedData['revision']);
+ assert.equal(rows[0]['commit_time'].toString(), new Date('2013-02-06 08:55:20.9').toString());
+ assert.equal(rows[0]['commit_author_name'], reportedData['author']['name']);
+ assert.equal(rows[0]['commit_author_email'], reportedData['author']['email']);
+ assert.equal(rows[0]['commit_message'], reportedData['message']);
+ notifyDone();
+ });
+ });
+ });
+ });
+
+ it("should reject an invalid revision number", function () {
+ addBuilder(subversionCommit, function () {
+ subversionCommit
+ postJSON('/api/report-commits/', subversionInvalidCommit, function (response) {
+ assert.equal(response.statusCode, 200);
+ assert.notEqual(JSON.parse(response.responseText)['status'], 'OK');
+ queryAndFetchAll('SELECT * FROM commits', [], function (rows) {
+ assert.equal(rows.length, 0);
+ notifyDone();
+ });
+ });
+ });
+ });
+
+ it("should store two commits from a valid builder", function () {
+ addBuilder(subversionTwoCommits, function () {
+ postJSON('/api/report-commits/', subversionTwoCommits, function (response) {
+ assert.equal(response.statusCode, 200);
+ assert.equal(JSON.parse(response.responseText)['status'], 'OK');
+ queryAndFetchAll('SELECT * FROM commits ORDER BY commit_time', [], function (rows) {
+ assert.equal(rows.length, 2);
+ var reportedData = subversionTwoCommits.commits[0];
+ assert.equal(rows[0]['commit_revision'], reportedData['revision']);
+ assert.equal(rows[0]['commit_time'].toString(), new Date('2013-02-06 08:55:20.9').toString());
+ assert.equal(rows[0]['commit_author_name'], reportedData['author']['name']);
+ assert.equal(rows[0]['commit_author_email'], reportedData['author']['email']);
+ assert.equal(rows[0]['commit_message'], reportedData['message']);
+ var reportedData = subversionTwoCommits.commits[1];
+ assert.equal(rows[1]['commit_revision'], reportedData['revision']);
+ assert.equal(rows[1]['commit_time'].toString(), new Date('2013-02-06 09:54:56.0').toString());
+ assert.equal(rows[1]['commit_author_name'], reportedData['author']['name']);
+ assert.equal(rows[1]['commit_author_email'], reportedData['author']['email']);
+ assert.equal(rows[1]['commit_message'], reportedData['message']);
+ notifyDone();
+ });
+ });
+ });
+ });
+
+ it("should update an existing commit if there is one", function () {
+ queryAndFetchAll('INSERT INTO repositories (repository_name) VALUES ($1) RETURNING *', ['WebKit'], function (repositories) {
+ var repositoryId = repositories[0]['repository_id'];
+ var reportedData = subversionCommit.commits[0];
+ queryAndFetchAll('INSERT INTO commits (commit_repository, commit_revision, commit_time) VALUES ($1, $2, $3) RETURNING *',
+ [repositoryId, reportedData['revision'], reportedData['time']], function (existingCommits) {
+ var commitId = existingCommits[0]['commit_id'];
+ assert.equal(existingCommits[0]['commit_author_name'], null);
+ assert.equal(existingCommits[0]['commit_author_email'], null);
+ assert.equal(existingCommits[0]['commit_message'], null);
+ addBuilder(subversionCommit, function () {
+ postJSON('/api/report-commits/', subversionCommit, function (response) {
+ assert.equal(response.statusCode, 200);
+ assert.equal(JSON.parse(response.responseText)['status'], 'OK');
+ queryAndFetchAll('SELECT * FROM commits', [], function (rows) {
+ assert.equal(rows.length, 1);
+ var reportedData = subversionCommit.commits[0];
+ assert.equal(rows[0]['commit_author_name'], reportedData['author']['name']);
+ assert.equal(rows[0]['commit_author_email'], reportedData['author']['email']);
+ assert.equal(rows[0]['commit_message'], reportedData['message']);
+ notifyDone();
+ });
+ });
+ });
+ });
+ });
+ });
+
+ it("should not update an unrelated commit", function () {
+ queryAndFetchAll('INSERT INTO repositories (repository_name) VALUES ($1) RETURNING *', ['WebKit'], function (repositories) {
+ var repositoryId = repositories[0]['repository_id'];
+ var reportedData = subversionTwoCommits.commits[1];
+ queryAndFetchAll('INSERT INTO commits (commit_repository, commit_revision, commit_time) VALUES ($1, $2, $3) RETURNING *',
+ [repositoryId, reportedData['revision'], reportedData['time']], function (existingCommits) {
+ reportedData = subversionTwoCommits.commits[0];
+ queryAndFetchAll('INSERT INTO commits (commit_repository, commit_revision, commit_time) VALUES ($1, $2, $3) RETURNING *',
+ [repositoryId, reportedData['revision'], reportedData['time']], function () {
+ addBuilder(subversionCommit, function () {
+ postJSON('/api/report-commits/', subversionCommit, function (response) {
+ assert.equal(response.statusCode, 200);
+ assert.equal(JSON.parse(response.responseText)['status'], 'OK');
+ queryAndFetchAll('SELECT * FROM commits ORDER BY commit_time', [], function (rows) {
+ assert.equal(rows.length, 2);
+ assert.equal(rows[0]['commit_author_name'], reportedData['author']['name']);
+ assert.equal(rows[0]['commit_author_email'], reportedData['author']['email']);
+ assert.equal(rows[0]['commit_message'], reportedData['message']);
+ assert.equal(rows[1]['commit_author_name'], null);
+ assert.equal(rows[1]['commit_author_email'], null);
+ assert.equal(rows[1]['commit_message'], null);
+ notifyDone();
+ });
+ });
+ });
+ });
+ });
+ });
+ });
+
+});
</ins></span></pre></div>
<a id="trunkWebsitesperfwebkitorgtestsapireportjs"></a>
<div class="modfile"><h4>Modified: trunk/Websites/perf.webkit.org/tests/api-report.js (174458 => 174459)</h4>
<pre class="diff"><span>
<span class="info">--- trunk/Websites/perf.webkit.org/tests/api-report.js 2014-10-08 17:16:25 UTC (rev 174458)
+++ trunk/Websites/perf.webkit.org/tests/api-report.js 2014-10-08 17:20:32 UTC (rev 174459)
</span><span class="lines">@@ -111,14 +111,14 @@
</span><span class="cx">
</span><span class="cx"> var repositoryIdToName = {};
</span><span class="cx"> rows.forEach(function (row) { repositoryIdToName[row['repository_id']] = row['repository_name']; });
</span><del>- queryAndFetchAll('SELECT * FROM build_revisions', [], function (rows) {
</del><ins>+ queryAndFetchAll('SELECT * FROM build_commits, commits WHERE build_commit = commit_id', [], function (rows) {
</ins><span class="cx"> var repositoryNameToRevisionRow = {};
</span><span class="cx"> rows.forEach(function (row) {
</span><del>- repositoryNameToRevisionRow[repositoryIdToName[row['revision_repository']]] = row;
</del><ins>+ repositoryNameToRevisionRow[repositoryIdToName[row['commit_repository']]] = row;
</ins><span class="cx"> });
</span><del>- assert.equal(repositoryNameToRevisionRow['OS X']['revision_value'], '10.8.2 12C60');
- assert.equal(repositoryNameToRevisionRow['WebKit']['revision_value'], '141977');
- assert.equal(repositoryNameToRevisionRow['WebKit']['revision_time'].toString(),
</del><ins>+ assert.equal(repositoryNameToRevisionRow['OS X']['commit_revision'], '10.8.2 12C60');
+ assert.equal(repositoryNameToRevisionRow['WebKit']['commit_revision'], '141977');
+ assert.equal(repositoryNameToRevisionRow['WebKit']['commit_time'].toString(),
</ins><span class="cx"> new Date('2013-02-06 08:55:20.9').toString());
</span><span class="cx"> notifyDone();
</span><span class="cx"> });
</span></span></pre></div>
<a id="trunkWebsitesperfwebkitorgtoolspullsvnpy"></a>
<div class="addfile"><h4>Added: trunk/Websites/perf.webkit.org/tools/pull-svn.py (0 => 174459)</h4>
<pre class="diff"><span>
<span class="info">--- trunk/Websites/perf.webkit.org/tools/pull-svn.py (rev 0)
+++ trunk/Websites/perf.webkit.org/tools/pull-svn.py 2014-10-08 17:20:32 UTC (rev 174459)
</span><span class="lines">@@ -0,0 +1,163 @@
</span><ins>+#!/usr/bin/python
+
+import json
+import re
+import subprocess
+import sys
+import time
+import urllib2
+
+from xml.dom.minidom import parseString as parseXmlString
+
+
+def main(argv):
+ if len(argv) < 7:
+ sys.exit('Usage: pull-svn <repository-name> <repository-URL> <dashboard-URL> <builder-name> <builder-password> <seconds-to-sleep> [<email-to-name-helper>]')
+
+ repository_name = argv[1]
+ repository_url = argv[2]
+ dashboard_url = argv[3]
+ builder_name = argv[4]
+ builder_password = argv[5]
+ seconds_to_sleep = float(argv[6])
+ email_to_name_helper = argv[7] if len(argv) > 7 else None
+
+ print "Submitting revision logs for %s at %s to %s" % (repository_name, repository_url, dashboard_url)
+
+ revision_to_fetch = determine_first_revision_to_fetch(dashboard_url, repository_name)
+ print "Start fetching commits at r%d" % revision_to_fetch
+
+ pending_commits_to_send = []
+
+ while True:
+ commit = fetch_commit_and_resolve_author(repository_name, repository_url, email_to_name_helper, revision_to_fetch)
+
+ if commit:
+ print "Fetched r%d." % revision_to_fetch
+ pending_commits_to_send += [commit]
+ revision_to_fetch += 1
+ else:
+ print "Revision %d not found" % revision_to_fetch
+
+ if not commit or len(pending_commits_to_send) >= 10:
+ if pending_commits_to_send:
+ print "Submitting the above commits to %s..." % dashboard_url
+ submit_commits(pending_commits_to_send, dashboard_url, builder_name, builder_password)
+ print "Successfully submitted."
+ pending_commits_to_send = []
+ time.sleep(seconds_to_sleep)
+
+
+def determine_first_revision_to_fetch(dashboard_url, repository_name):
+ try:
+ last_reported_revision = fetch_revision_from_dasbhoard(dashboard_url, repository_name, 'last-reported')
+ except Exception as error:
+ sys.exit('Failed to fetch the latest reported commit: ' + str(error))
+
+ if last_reported_revision:
+ return last_reported_revision + 1
+
+ # FIXME: This is a problematic if dashboard can get results for revisions older than oldest_revision
+ # in the future because we never refetch older revisions.
+ try:
+ return fetch_revision_from_dasbhoard(dashboard_url, repository_name, 'oldest') or 1
+ except Exception as error:
+ sys.exit('Failed to fetch the oldest commit: ' + str(error))
+
+
+def fetch_revision_from_dasbhoard(dashboard_url, repository_name, filter):
+ result = urllib2.urlopen(dashboard_url + '/api/commits/' + repository_name + '/' + filter).read()
+ parsed_result = json.loads(result)
+ if parsed_result['status'] != 'OK' and parsed_result['status'] != 'RepositoryNotFound':
+ raise Exception(result)
+ commits = parsed_result.get('commits')
+ return int(commits[0]['revision']) if commits else None
+
+
+def fetch_commit_and_resolve_author(repository_name, repository_url, email_to_name_helper, revision_to_fetch):
+ try:
+ commit = fetch_commit(repository_name, repository_url, revision_to_fetch)
+ except Exception as error:
+ sys.exit('Failed to fetch the commit %d: %s' % (revision_to_fetch, str(error)))
+
+ if not commit:
+ return None
+
+ email = commit['author']['email']
+ try:
+ name = resolve_author_name_from_email(email_to_name_helper, email) if email_to_name_helper else None
+ if name:
+ commit['author']['name'] = name
+ except Exception as error:
+ sys.exit('Failed to resolve the author name from an email %s: %s' % (email, str(error)))
+
+ return commit
+
+
+def fetch_commit(repository_name, repository_url, revision):
+ args = ['svn', 'log', '--revision', str(revision), '--xml', repository_url]
+ try:
+ output = subprocess.check_output(args, stderr=subprocess.STDOUT)
+ except subprocess.CalledProcessError as error:
+ if (': No such revision ' + str(revision)) in error.output:
+ return None
+ raise error
+ xml = parseXmlString(output)
+ time = textContent(xml.getElementsByTagName("date")[0])
+ author_email = textContent(xml.getElementsByTagName("author")[0])
+ message = textContent(xml.getElementsByTagName("msg")[0])
+ return {
+ 'repository': repository_name,
+ 'revision': revision,
+ 'time': time,
+ 'author': {'email': author_email},
+ 'message': message,
+ }
+
+
+def textContent(element):
+ text = ''
+ for child in element.childNodes:
+ if child.nodeType == child.TEXT_NODE:
+ text += child.data
+ else:
+ text += textContent(child)
+ return text
+
+
+name_email_compound_regex = re.compile(r'^\s*(?P<name>(\".+\"|[^<]+?))\s*\<(?P<email>.+)\>\s*$')
+
+
+def resolve_author_name_from_email(helper, email):
+ output = subprocess.check_output(helper + ' ' + email, shell=True)
+ match = name_email_compound_regex.match(output)
+ if match:
+ return match.group('name').strip('"')
+ return output.strip()
+
+
+def submit_commits(commits, dashboard_url, builder_name, builder_password):
+ try:
+ payload = json.dumps({
+ 'builderName': builder_name,
+ 'builderPassword': builder_password,
+ 'commits': commits,
+ })
+ request = urllib2.Request(dashboard_url + '/api/report-commits')
+ request.add_header('Content-Type', 'application/json')
+ request.add_header('Content-Length', len(payload))
+
+ output = urllib2.urlopen(request, payload).read()
+ try:
+ result = json.loads(output)
+ except Exception, error:
+ raise Exception(error, output)
+
+ if result.get('status') != 'OK':
+ raise Exception(result)
+ except Exception as error:
+ sys.exit('Failed to submit commits: %s' % str(error))
+
+
+if __name__ == "__main__":
+ main(sys.argv)
</ins></span></pre>
</div>
</div>
</body>
</html>