<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.1//EN"
"http://www.w3.org/TR/xhtml11/DTD/xhtml11.dtd">
<html xmlns="http://www.w3.org/1999/xhtml">
<head><meta http-equiv="content-type" content="text/html; charset=utf-8" />
<title>[182363] trunk</title>
</head>
<body>
<style type="text/css"><!--
#msg dl.meta { border: 1px #006 solid; background: #369; padding: 6px; color: #fff; }
#msg dl.meta dt { float: left; width: 6em; font-weight: bold; }
#msg dt:after { content:':';}
#msg dl, #msg dt, #msg ul, #msg li, #header, #footer, #logmsg { font-family: verdana,arial,helvetica,sans-serif; font-size: 10pt; }
#msg dl a { font-weight: bold}
#msg dl a:link { color:#fc3; }
#msg dl a:active { color:#ff0; }
#msg dl a:visited { color:#cc6; }
h3 { font-family: verdana,arial,helvetica,sans-serif; font-size: 10pt; font-weight: bold; }
#msg pre { overflow: auto; background: #ffc; border: 1px #fa0 solid; padding: 6px; }
#logmsg { background: #ffc; border: 1px #fa0 solid; padding: 1em 1em 0 1em; }
#logmsg p, #logmsg pre, #logmsg blockquote { margin: 0 0 1em 0; }
#logmsg p, #logmsg li, #logmsg dt, #logmsg dd { line-height: 14pt; }
#logmsg h1, #logmsg h2, #logmsg h3, #logmsg h4, #logmsg h5, #logmsg h6 { margin: .5em 0; }
#logmsg h1:first-child, #logmsg h2:first-child, #logmsg h3:first-child, #logmsg h4:first-child, #logmsg h5:first-child, #logmsg h6:first-child { margin-top: 0; }
#logmsg ul, #logmsg ol { padding: 0; list-style-position: inside; margin: 0 0 0 1em; }
#logmsg ul { text-indent: -1em; padding-left: 1em; }#logmsg ol { text-indent: -1.5em; padding-left: 1.5em; }
#logmsg > ul, #logmsg > ol { margin: 0 0 1em 0; }
#logmsg pre { background: #eee; padding: 1em; }
#logmsg blockquote { border: 1px solid #fa0; border-left-width: 10px; padding: 1em 1em 0 1em; background: white;}
#logmsg dl { margin: 0; }
#logmsg dt { font-weight: bold; }
#logmsg dd { margin: 0; padding: 0 0 0.5em 0; }
#logmsg dd:before { content:'\00bb';}
#logmsg table { border-spacing: 0px; border-collapse: collapse; border-top: 4px solid #fa0; border-bottom: 1px solid #fa0; background: #fff; }
#logmsg table th { text-align: left; font-weight: normal; padding: 0.2em 0.5em; border-top: 1px dotted #fa0; }
#logmsg table td { text-align: right; border-top: 1px dotted #fa0; padding: 0.2em 0.5em; }
#logmsg table thead th { text-align: center; border-bottom: 1px solid #fa0; }
#logmsg table th.Corner { text-align: left; }
#logmsg hr { border: none 0; border-top: 2px dashed #fa0; height: 1px; }
#header, #footer { color: #fff; background: #636; border: 1px #300 solid; padding: 6px; }
#patch { width: 100%; }
#patch h4 {font-family: verdana,arial,helvetica,sans-serif;font-size:10pt;padding:8px;background:#369;color:#fff;margin:0;}
#patch .propset h4, #patch .binary h4 {margin:0;}
#patch pre {padding:0;line-height:1.2em;margin:0;}
#patch .diff {width:100%;background:#eee;padding: 0 0 10px 0;overflow:auto;}
#patch .propset .diff, #patch .binary .diff {padding:10px 0;}
#patch span {display:block;padding:0 10px;}
#patch .modfile, #patch .addfile, #patch .delfile, #patch .propset, #patch .binary, #patch .copfile {border:1px solid #ccc;margin:10px 0;}
#patch ins {background:#dfd;text-decoration:none;display:block;padding:0 10px;}
#patch del {background:#fdd;text-decoration:none;display:block;padding:0 10px;}
#patch .lines, .info {color:#888;background:#fff;}
--></style>
<div id="msg">
<dl class="meta">
<dt>Revision</dt> <dd><a href="http://trac.webkit.org/projects/webkit/changeset/182363">182363</a></dd>
<dt>Author</dt> <dd>antti@apple.com</dd>
<dt>Date</dt> <dd>2015-04-05 11:45:38 -0700 (Sun, 05 Apr 2015)</dd>
</dl>
<h3>Log Message</h3>
<pre>Bloom filter should support longer hashes
https://bugs.webkit.org/show_bug.cgi?id=143419
Reviewed by Dan Bernstein.
Source/WebKit2:
Use the hash digest directly in the contents filter instead of going via shortHash.
* NetworkProcess/cache/NetworkCacheKey.h:
(WebKit::NetworkCache::Key::hash):
(WebKit::NetworkCache::Key::shortHash): Deleted.
(WebKit::NetworkCache::Key::toShortHash): Deleted.
No longer needed.
* NetworkProcess/cache/NetworkCacheStorage.cpp:
(WebKit::NetworkCache::Storage::synchronize):
(WebKit::NetworkCache::Storage::addToContentsFilter):
(WebKit::NetworkCache::Storage::mayContain):
* NetworkProcess/cache/NetworkCacheStorage.h:
Source/WTF:
It currently supports 'unsigned' hash only which is inconvenient and doesn't have enough independent bits for larger filters.
Support arbitrarily sized hashes of type std::array<uint8_t, hashSize> (like SHA1::Digest and MD5::Digest).
* wtf/BloomFilter.h:
(WTF::BloomFilter<keyBits>::keysFromHash):
(WTF::BloomFilter<keyBits>::mayContain):
(WTF::BloomFilter<keyBits>::add):
Tools:
* TestWebKitAPI/Tests/WTF/BloomFilter.cpp:
(TestWebKitAPI::generateRandomDigests):
(TestWebKitAPI::TEST):</pre>
<h3>Modified Paths</h3>
<ul>
<li><a href="#trunkSourceWTFChangeLog">trunk/Source/WTF/ChangeLog</a></li>
<li><a href="#trunkSourceWTFwtfBloomFilterh">trunk/Source/WTF/wtf/BloomFilter.h</a></li>
<li><a href="#trunkSourceWebKit2ChangeLog">trunk/Source/WebKit2/ChangeLog</a></li>
<li><a href="#trunkSourceWebKit2NetworkProcesscacheNetworkCacheKeyh">trunk/Source/WebKit2/NetworkProcess/cache/NetworkCacheKey.h</a></li>
<li><a href="#trunkSourceWebKit2NetworkProcesscacheNetworkCacheStoragecpp">trunk/Source/WebKit2/NetworkProcess/cache/NetworkCacheStorage.cpp</a></li>
<li><a href="#trunkSourceWebKit2NetworkProcesscacheNetworkCacheStorageh">trunk/Source/WebKit2/NetworkProcess/cache/NetworkCacheStorage.h</a></li>
<li><a href="#trunkToolsChangeLog">trunk/Tools/ChangeLog</a></li>
<li><a href="#trunkToolsTestWebKitAPITestsWTFBloomFiltercpp">trunk/Tools/TestWebKitAPI/Tests/WTF/BloomFilter.cpp</a></li>
</ul>
</div>
<div id="patch">
<h3>Diff</h3>
<a id="trunkSourceWTFChangeLog"></a>
<div class="modfile"><h4>Modified: trunk/Source/WTF/ChangeLog (182362 => 182363)</h4>
<pre class="diff"><span>
<span class="info">--- trunk/Source/WTF/ChangeLog 2015-04-05 18:39:07 UTC (rev 182362)
+++ trunk/Source/WTF/ChangeLog 2015-04-05 18:45:38 UTC (rev 182363)
</span><span class="lines">@@ -1,3 +1,19 @@
</span><ins>+2015-04-05 Antti Koivisto <antti@apple.com>
+
+ Bloom filter should support longer hashes
+ https://bugs.webkit.org/show_bug.cgi?id=143419
+
+ Reviewed by Dan Bernstein.
+
+ It currently supports 'unsigned' hash only which is inconvenient and doesn't have enough independent bits for larger filters.
+
+ Support arbitrarily sized hashes of type std::array<uint8_t, hashSize> (like SHA1::Digest and MD5::Digest).
+
+ * wtf/BloomFilter.h:
+ (WTF::BloomFilter<keyBits>::keysFromHash):
+ (WTF::BloomFilter<keyBits>::mayContain):
+ (WTF::BloomFilter<keyBits>::add):
+
</ins><span class="cx"> 2015-04-03 Antti Koivisto <antti@apple.com>
</span><span class="cx">
</span><span class="cx"> Add non-counting Bloom filter implementation
</span></span></pre></div>
<a id="trunkSourceWTFwtfBloomFilterh"></a>
<div class="modfile"><h4>Modified: trunk/Source/WTF/wtf/BloomFilter.h (182362 => 182363)</h4>
<pre class="diff"><span>
<span class="info">--- trunk/Source/WTF/wtf/BloomFilter.h 2015-04-05 18:39:07 UTC (rev 182362)
+++ trunk/Source/WTF/wtf/BloomFilter.h 2015-04-05 18:45:38 UTC (rev 182363)
</span><span class="lines">@@ -44,11 +44,15 @@
</span><span class="cx"> BloomFilter();
</span><span class="cx">
</span><span class="cx"> void add(unsigned hash);
</span><ins>+ // For example SHA1::Digest.
+ template <size_t hashSize> void add(const std::array<uint8_t, hashSize>&);
+
</ins><span class="cx"> void add(const BloomFilter&);
</span><span class="cx">
</span><span class="cx"> // The filter may give false positives (claim it may contain a key it doesn't)
</span><span class="cx"> // but never false negatives (claim it doesn't contain a key it does).
</span><span class="cx"> bool mayContain(unsigned hash) const;
</span><ins>+ template <size_t hashSize> bool mayContain(const std::array<uint8_t, hashSize>&) const;
</ins><span class="cx">
</span><span class="cx"> void clear();
</span><span class="cx">
</span><span class="lines">@@ -62,6 +66,7 @@
</span><span class="cx"> static const unsigned keyMask = (1 << keyBits) - 1;
</span><span class="cx"> static unsigned arrayIndex(unsigned key) { return key / bitsPerPosition; }
</span><span class="cx"> static unsigned bitMask(unsigned key) { return 1 << (key % bitsPerPosition); }
</span><ins>+ template <size_t hashSize> static std::pair<unsigned, unsigned> keysFromHash(const std::array<uint8_t, hashSize>&);
</ins><span class="cx">
</span><span class="cx"> bool isBitSet(unsigned key) const;
</span><span class="cx"> void setBit(unsigned key);
</span><span class="lines">@@ -91,6 +96,35 @@
</span><span class="cx"> }
</span><span class="cx">
</span><span class="cx"> template <unsigned keyBits>
</span><ins>+template <size_t hashSize>
+inline std::pair<unsigned, unsigned> BloomFilter<keyBits>::keysFromHash(const std::array<uint8_t, hashSize>& hash)
+{
+ // We could use larger k value than 2 for long hashes.
+ static_assert(hashSize >= 2 * sizeof(unsigned), "Hash array too short");
+ return {
+ *reinterpret_cast<const unsigned*>(hash.data()),
+ *reinterpret_cast<const unsigned*>(hash.data() + sizeof(unsigned))
+ };
+}
+
+template <unsigned keyBits>
+template <size_t hashSize>
+inline bool BloomFilter<keyBits>::mayContain(const std::array<uint8_t, hashSize>& hash) const
+{
+ auto keys = keysFromHash(hash);
+ return isBitSet(keys.first) && isBitSet(keys.second);
+}
+
+template <unsigned keyBits>
+template <size_t hashSize>
+inline void BloomFilter<keyBits>::add(const std::array<uint8_t, hashSize>& hash)
+{
+ auto keys = keysFromHash(hash);
+ setBit(keys.first);
+ setBit(keys.second);
+}
+
+template <unsigned keyBits>
</ins><span class="cx"> inline void BloomFilter<keyBits>::add(const BloomFilter& other)
</span><span class="cx"> {
</span><span class="cx"> for (size_t i = 0; i < m_bitArray.size(); ++i)
</span></span></pre></div>
<a id="trunkSourceWebKit2ChangeLog"></a>
<div class="modfile"><h4>Modified: trunk/Source/WebKit2/ChangeLog (182362 => 182363)</h4>
<pre class="diff"><span>
<span class="info">--- trunk/Source/WebKit2/ChangeLog 2015-04-05 18:39:07 UTC (rev 182362)
+++ trunk/Source/WebKit2/ChangeLog 2015-04-05 18:45:38 UTC (rev 182363)
</span><span class="lines">@@ -1,5 +1,27 @@
</span><span class="cx"> 2015-04-05 Antti Koivisto <antti@apple.com>
</span><span class="cx">
</span><ins>+ Bloom filter should support longer hashes
+ https://bugs.webkit.org/show_bug.cgi?id=143419
+
+ Reviewed by Dan Bernstein.
+
+ Use the hash digest directly in the contents filter instead of going via shortHash.
+
+ * NetworkProcess/cache/NetworkCacheKey.h:
+ (WebKit::NetworkCache::Key::hash):
+ (WebKit::NetworkCache::Key::shortHash): Deleted.
+ (WebKit::NetworkCache::Key::toShortHash): Deleted.
+
+ No longer needed.
+
+ * NetworkProcess/cache/NetworkCacheStorage.cpp:
+ (WebKit::NetworkCache::Storage::synchronize):
+ (WebKit::NetworkCache::Storage::addToContentsFilter):
+ (WebKit::NetworkCache::Storage::mayContain):
+ * NetworkProcess/cache/NetworkCacheStorage.h:
+
+2015-04-05 Antti Koivisto <antti@apple.com>
+
</ins><span class="cx"> Network cache Bloom filter is too big
</span><span class="cx"> https://bugs.webkit.org/show_bug.cgi?id=143400
</span><span class="cx">
</span></span></pre></div>
<a id="trunkSourceWebKit2NetworkProcesscacheNetworkCacheKeyh"></a>
<div class="modfile"><h4>Modified: trunk/Source/WebKit2/NetworkProcess/cache/NetworkCacheKey.h (182362 => 182363)</h4>
<pre class="diff"><span>
<span class="info">--- trunk/Source/WebKit2/NetworkProcess/cache/NetworkCacheKey.h 2015-04-05 18:39:07 UTC (rev 182362)
+++ trunk/Source/WebKit2/NetworkProcess/cache/NetworkCacheKey.h 2015-04-05 18:45:38 UTC (rev 182363)
</span><span class="lines">@@ -55,9 +55,7 @@
</span><span class="cx"> const String& identifier() const { return m_identifier; }
</span><span class="cx">
</span><span class="cx"> HashType hash() const { return m_hash; }
</span><del>- unsigned shortHash() const { return toShortHash(m_hash); }
</del><span class="cx">
</span><del>- static unsigned toShortHash(const HashType& hash) { return *reinterpret_cast<const unsigned*>(hash.data()); }
</del><span class="cx"> static bool stringToHash(const String&, HashType&);
</span><span class="cx">
</span><span class="cx"> static size_t hashStringLength() { return 2 * sizeof(m_hash); }
</span></span></pre></div>
<a id="trunkSourceWebKit2NetworkProcesscacheNetworkCacheStoragecpp"></a>
<div class="modfile"><h4>Modified: trunk/Source/WebKit2/NetworkProcess/cache/NetworkCacheStorage.cpp (182362 => 182363)</h4>
<pre class="diff"><span>
<span class="info">--- trunk/Source/WebKit2/NetworkProcess/cache/NetworkCacheStorage.cpp 2015-04-05 18:39:07 UTC (rev 182362)
+++ trunk/Source/WebKit2/NetworkProcess/cache/NetworkCacheStorage.cpp 2015-04-05 18:45:38 UTC (rev 182363)
</span><span class="lines">@@ -99,7 +99,7 @@
</span><span class="cx"> WebCore::getFileSize(filePath, fileSize);
</span><span class="cx"> if (!fileSize)
</span><span class="cx"> return;
</span><del>- filter->add(Key::toShortHash(hash));
</del><ins>+ filter->add(hash);
</ins><span class="cx"> size += fileSize;
</span><span class="cx"> ++count;
</span><span class="cx"> });
</span><span class="lines">@@ -126,17 +126,17 @@
</span><span class="cx"> ASSERT(RunLoop::isMain());
</span><span class="cx">
</span><span class="cx"> if (m_contentsFilter)
</span><del>- m_contentsFilter->add(key.shortHash());
</del><ins>+ m_contentsFilter->add(key.hash());
</ins><span class="cx">
</span><span class="cx"> // If we get new entries during filter synchronization take care to add them to the new filter as well.
</span><span class="cx"> if (m_synchronizationInProgress)
</span><del>- m_contentsFilterHashesAddedDuringSynchronization.append(key.shortHash());
</del><ins>+ m_contentsFilterHashesAddedDuringSynchronization.append(key.hash());
</ins><span class="cx"> }
</span><span class="cx">
</span><span class="cx"> bool Storage::mayContain(const Key& key) const
</span><span class="cx"> {
</span><span class="cx"> ASSERT(RunLoop::isMain());
</span><del>- return !m_contentsFilter || m_contentsFilter->mayContain(key.shortHash());
</del><ins>+ return !m_contentsFilter || m_contentsFilter->mayContain(key.hash());
</ins><span class="cx"> }
</span><span class="cx">
</span><span class="cx"> static String directoryPathForKey(const Key& key, const String& cachePath)
</span></span></pre></div>
<a id="trunkSourceWebKit2NetworkProcesscacheNetworkCacheStorageh"></a>
<div class="modfile"><h4>Modified: trunk/Source/WebKit2/NetworkProcess/cache/NetworkCacheStorage.h (182362 => 182363)</h4>
<pre class="diff"><span>
<span class="info">--- trunk/Source/WebKit2/NetworkProcess/cache/NetworkCacheStorage.h 2015-04-05 18:39:07 UTC (rev 182362)
+++ trunk/Source/WebKit2/NetworkProcess/cache/NetworkCacheStorage.h 2015-04-05 18:45:38 UTC (rev 182363)
</span><span class="lines">@@ -129,7 +129,7 @@
</span><span class="cx"> bool m_synchronizationInProgress { false };
</span><span class="cx"> bool m_shrinkInProgress { false };
</span><span class="cx">
</span><del>- Vector<unsigned> m_contentsFilterHashesAddedDuringSynchronization;
</del><ins>+ Vector<Key::HashType> m_contentsFilterHashesAddedDuringSynchronization;
</ins><span class="cx">
</span><span class="cx"> static const int maximumRetrievePriority = 4;
</span><span class="cx"> Deque<std::unique_ptr<const ReadOperation>> m_pendingReadOperationsByPriority[maximumRetrievePriority + 1];
</span></span></pre></div>
<a id="trunkToolsChangeLog"></a>
<div class="modfile"><h4>Modified: trunk/Tools/ChangeLog (182362 => 182363)</h4>
<pre class="diff"><span>
<span class="info">--- trunk/Tools/ChangeLog 2015-04-05 18:39:07 UTC (rev 182362)
+++ trunk/Tools/ChangeLog 2015-04-05 18:45:38 UTC (rev 182363)
</span><span class="lines">@@ -1,3 +1,14 @@
</span><ins>+2015-04-05 Antti Koivisto <antti@apple.com>
+
+ Bloom filter should support longer hashes
+ https://bugs.webkit.org/show_bug.cgi?id=143419
+
+ Reviewed by Dan Bernstein.
+
+ * TestWebKitAPI/Tests/WTF/BloomFilter.cpp:
+ (TestWebKitAPI::generateRandomDigests):
+ (TestWebKitAPI::TEST):
+
</ins><span class="cx"> 2015-04-03 Csaba Osztrogonác <ossy@webkit.org>
</span><span class="cx">
</span><span class="cx"> run-jsc-stress-tests --remote should add libllvmForJSC.so to the bundle on Linux
</span></span></pre></div>
<a id="trunkToolsTestWebKitAPITestsWTFBloomFiltercpp"></a>
<div class="modfile"><h4>Modified: trunk/Tools/TestWebKitAPI/Tests/WTF/BloomFilter.cpp (182362 => 182363)</h4>
<pre class="diff"><span>
<span class="info">--- trunk/Tools/TestWebKitAPI/Tests/WTF/BloomFilter.cpp 2015-04-05 18:39:07 UTC (rev 182362)
+++ trunk/Tools/TestWebKitAPI/Tests/WTF/BloomFilter.cpp 2015-04-05 18:45:38 UTC (rev 182363)
</span><span class="lines">@@ -27,6 +27,7 @@
</span><span class="cx">
</span><span class="cx"> #include <wtf/BloomFilter.h>
</span><span class="cx"> #include <wtf/RandomNumber.h>
</span><ins>+#include <wtf/SHA1.h>
</ins><span class="cx">
</span><span class="cx"> namespace TestWebKitAPI {
</span><span class="cx">
</span><span class="lines">@@ -38,6 +39,20 @@
</span><span class="cx"> return hashes;
</span><span class="cx"> }
</span><span class="cx">
</span><ins>+static Vector<SHA1::Digest> generateRandomDigests(size_t hashCount)
+{
+ Vector<SHA1::Digest> hashes;
+ SHA1 sha1;
+ for (unsigned i = 0; i < hashCount; ++i) {
+ double random = randomNumber();
+ sha1.addBytes(reinterpret_cast<uint8_t*>(&random), sizeof(double));
+ SHA1::Digest digest;
+ sha1.computeHash(digest);
+ hashes.append(digest);
+ }
+ return hashes;
+}
+
</ins><span class="cx"> TEST(WTF_BloomFilter, Basic)
</span><span class="cx"> {
</span><span class="cx"> const unsigned hashCount = 1000;
</span><span class="lines">@@ -66,6 +81,34 @@
</span><span class="cx"> EXPECT_TRUE(filter.mayContain(hash));
</span><span class="cx"> }
</span><span class="cx">
</span><ins>+TEST(WTF_BloomFilter, BasicDigest)
+{
+ const unsigned hashCount = 1000;
+ auto hashes = generateRandomDigests(hashCount);
+
+ BloomFilter<20> filter;
+ for (auto hash : hashes)
+ filter.add(hash);
+
+ for (auto hash : hashes)
+ EXPECT_TRUE(filter.mayContain(hash));
+
+ auto moreHashes = generateRandomDigests(hashCount);
+ unsigned mayContainCount = 0;
+ for (auto hash : moreHashes)
+ mayContainCount += filter.mayContain(hash) ? 1 : 0;
+ // False positive rate is ~0.000004% so this should always be true.
+ EXPECT_TRUE(mayContainCount < hashCount / 10);
+
+ for (auto hash : moreHashes)
+ filter.add(hash);
+
+ for (auto hash : hashes)
+ EXPECT_TRUE(filter.mayContain(hash));
+ for (auto hash : moreHashes)
+ EXPECT_TRUE(filter.mayContain(hash));
+}
+
</ins><span class="cx"> TEST(WTF_BloomFilter, BasicCounting)
</span><span class="cx"> {
</span><span class="cx"> const unsigned hashCount = 1000;
</span></span></pre>
</div>
</div>
</body>
</html>