working client

.gitignore

@@ -1 +1,3 @@
 target/
+*.gz
+*.dat

client/css/reset.css

@@ -0,0 +1,43 @@
+html, body, div, span, applet, object, iframe,
+h1, h2, h3, h4, h5, h6, p, blockquote, pre,
+a, abbr, acronym, address, big, cite, code,
+del, dfn, em, img, ins, kbd, q, s, samp,
+small, strike, strong, sub, sup, tt, var,
+b, u, i, center,
+dl, dt, dd, ol, ul, li,
+fieldset, form, label, legend,
+table, caption, tbody, tfoot, thead, tr, th, td,
+article, aside, canvas, details, embed, 
+figure, figcaption, footer, header, hgroup, 
+menu, nav, output, ruby, section, summary,
+time, mark, audio, video {
+	margin: 0;
+	padding: 0;
+	border: 0;
+	font-size: 100%;
+	font: inherit;
+	vertical-align: baseline;
+}
+/* HTML5 display-role reset for older browsers */
+article, aside, details, figcaption, figure, 
+footer, header, hgroup, menu, nav, section {
+	display: block;
+}
+body {
+	line-height: 1;
+}
+ol, ul {
+	list-style: none;
+}
+blockquote, q {
+	quotes: none;
+}
+blockquote:before, blockquote:after,
+q:before, q:after {
+	content: '';
+	content: none;
+}
+table {
+	border-collapse: collapse;
+	border-spacing: 0;
+}

client/css/style.css

@@ -0,0 +1,465 @@
+
+/*
+The MIT License (MIT)
+Copyright (c) 2020 Tobias Ahlin
+Permission is hereby granted, free of charge, to any person obtaining a copy of
+this software and associated documentation files (the "Software"), to deal in
+the Software without restriction, including without limitation the rights to
+use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
+the Software, and to permit persons to whom the Software is furnished to do so,
+subject to the following conditions:
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
+FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/
+
+* {
+  box-sizing: border-box;
+}
+
+.articles article a.broken {
+  color: #9c3535;
+  cursor: not-allowed;
+  text-decoration: none;
+  pointer-events: none;
+}
+
+.fixedpanel {
+  width: 100%;
+  height: 50px;
+  position: fixed;
+  background-color: #FFF;
+  padding: 16px;
+  top: 0;
+  z-index: 1;
+}
+
+.searchbox {
+  width: 250px;
+  font-size: 20px;
+  line-height: 26px;
+  color: #202020;
+  height: 30px;
+  -webkit-appearance: none;
+  border: 1px solid #EEE;
+}
+
+.query {
+  width: 30px;
+  height: 30px;
+  -webkit-appearance: none;
+  border: none;
+  color: #909090;
+  background-color: #EEE;
+  margin-left: 4px;
+  cursor: pointer;
+}
+
+.content {
+  position: relative;
+  top: 40px;
+  display: flex;
+  flex-wrap: wrap;
+  padding: 16px;
+}
+
+.articles {
+  width: 800px;
+  margin-left: 20px;
+  flex-grow: 1;
+}
+
+.articles:empty {
+  display: none;
+}
+
+.sidebar {
+  max-width: 600px;
+  padding: 8px;
+  font-family: Arial;
+  background-color: #EEE;
+  position: relative;
+  min-height: 48px;
+  flex-grow: 1;
+}
+
+.sidebar.collapsed {
+  height: 0;
+  overflow-y: hidden;
+}
+
+.sidebar-header {
+  display: flex;
+  justify-content: space-between;
+  height: 48px;
+}
+
+.sidebar-title {
+  line-height: 36px;
+  font-weight: bold;
+  font-size: 24px;
+}
+
+.sidebar-content {
+  padding: 0 16px;
+  margin-top: 16px;
+}
+
+.sidebar-collapse-btn {
+  position: absolute;
+  top: 8px;
+  right: 8px;
+  width: 32px;
+  height: 32px;
+  display: flex;
+  align-items: center;
+  justify-content: center;
+}
+
+.sidebar-collapse-btn::before {
+	border-style: solid;
+	border-width: 3px 3px 0 0;
+	content: '';
+	display: inline-block;
+  height: 8px;
+  position: relative;
+  top: -3px;
+	transform: rotate(45deg) scaleY(-1);
+	vertical-align: top;
+	width: 8px;
+}
+
+.sidebar.collapsed .sidebar-collapse-btn {
+  transform: scaleY(-1);
+}
+
+@media (max-width: 1440px) {
+  .sidebar {
+    order: -1;
+    max-width: 900px;
+    width: 100%;
+  }
+}
+
+@media (max-width: 768px) {
+  .fixedpanel {
+    padding: 4px;
+  }
+  .content {
+    padding: 4px;
+  }
+  body {
+    margin: 4px;
+  }
+}
+
+.articles article {
+  font-family: sans-serif;
+  font-size: 14px;
+  color: rgb(32, 33, 34);
+  line-height: 22.4px;
+}
+
+.articles article a { 
+  color:rgb(6, 69, 173);
+  cursor: pointer;
+  direction: ltr;
+  display: inline;
+  font-family: sans-serif;
+  font-size: 14px;
+  line-height: 22.4px;
+  text-decoration-color: rgb(6, 69, 173);
+  text-decoration-line: none;
+  text-decoration-style: solid;
+}
+
+.articles article h1, .articles article h2, .articles article h3, .articles article h4, .articles article h5 {
+  margin-bottom: 0.25em;
+  padding: 0;
+  font-family: 'Linux Libertine','Georgia','Times',serif;
+  border-bottom-color: rgb(162, 169, 177);
+  border-bottom-style: solid;
+  border-bottom-width: 1px;
+  font-weight: 400;
+  color: rgb(0,0,0);
+}
+
+.articles article h2.title {
+  line-height: 37.44px;
+  font-size: 28.8px;
+}
+
+.articles article h1 {
+  line-height: 27.3px;
+  font-size: 21px;
+}
+
+.articles article h2:not(.title) {
+  line-height: 26.88px;
+  font-size: 16.8px;
+  border: none;
+  font-weight: 700;
+  font-family: sans-serif;
+}
+
+.articles article h3 {
+  line-height: 22.4px;
+  font-size: 14px;
+  border: none;
+  font-weight: 700;
+  font-family: sans-serif;
+}
+
+.articles article a:hover, .articles article a:focus { 
+  text-decoration: underline;
+}
+
+
+
+.sidebar li {
+  margin-bottom: 8px;
+}
+
+.sidebar h2 {
+  font-size: 16px;
+  font-weight: bold;
+  margin-top: 36px;
+}
+
+.sidebar h2:first-child {
+  margin-top: 0;
+}
+
+.sidebar {
+  color: #444;
+}
+
+.sidebar figure {
+  margin: 0;
+}
+
+.sidebar table {
+  width: 100%;
+  font-size: 13px;
+}
+
+.sidebar table thead {
+  text-align: left;
+}
+
+.suggestions {
+  font-family: sans-serif;
+  font-size: 13px;
+  margin-top: 4px;
+  margin-bottom: 0;
+  display: block;
+  padding: 4px;
+  background-color: #FFF;
+}
+
+.suggestions > div {
+  height: 19px;
+  padding: 0.01em 0.25em;
+  cursor: pointer;
+  align-items: center;
+}
+
+.suggestions > div:hover {
+  background-color: #666;
+  color: #FFF;
+}
+
+.suggestions .highlight {
+  font-weight: bold;
+}
+
+.nolistmarker {
+  list-style: none;
+}
+
+.diagram {
+  margin-top: 10px;
+  display: flex;
+}
+
+@keyframes clientProcessingBottom {
+  0% {
+    transform: translateX(0) scaleX(0);
+  }
+  25% {
+    transform: translateX(0) scaleX(1);
+  }
+  75% {
+    transform: translateX(0) scaleX(1);
+  }
+  100% {
+    transform: translateX(32px) scaleX(0);
+  }
+}
+
+@keyframes clientProcessingMiddle {
+  0% {
+    transform: translateX(0) scaleX(0);
+  }
+  25% {
+    transform: translateX(0) scaleX(0);
+  }
+  50% {
+    transform: translateX(0) scaleX(1);
+  }
+  75% {
+    transform: translateX(0) scaleX(1);
+  }
+  100% {
+    transform: translateX(32px) scaleX(0);
+  }
+}
+
+@keyframes clientProcessingTop {
+  0% {
+    transform: translateX(0) scaleX(0);
+  }
+  50% {
+    transform: translateX(0) scaleX(0);
+  }
+  75% {
+    transform: translateX(0) scaleX(1);
+  }
+  100% {
+    transform: translateX(32px) scaleX(0);
+  }
+}
+
+.client {
+  border: 4px solid #666;
+  height: 48px;
+  width: 48px;
+}
+
+.client .box {
+  height: 8px;
+  width: 32px;
+  margin: 4px;
+  background-color: #666;
+  transform-origin: 0% 50%;
+}
+
+.client .box.top {
+  animation: clientProcessingBottom 3s ease 0s infinite normal none;
+}
+
+.client .box.middle {
+  animation: clientProcessingMiddle 3s ease 0s infinite normal none;
+}
+
+.client .box.bottom {
+  animation: clientProcessingTop 3s ease 0s infinite normal none;
+}
+
+.box.complete {
+  animation: none !important;
+}
+
+@keyframes expandRightLine {
+  0% {
+    transform: scaleX(0);
+  }
+  100% {
+    transform: scaleX(1);
+  }
+}
+
+@keyframes expandRightArrowhead {
+  0% {
+    transform: translateX(0);
+  }
+  100% {
+    transform: translateX(200px);
+  }
+}
+
+.upload .line {
+  transform-origin: 0% 50%;
+  /* animation: expandRightLine 3s ease 0s infinite normal none; */
+
+  margin: 14px 4px 0 0;
+  width: 200px;
+  height: 4px;
+  background-color: #666;
+  position:relative;
+}
+
+.arrow {
+  position: relative;
+}
+
+.arrowhead {
+  /* animation: expandRightArrowhead 3s ease 0s infinite normal none; */
+  content:"";
+  position:absolute;
+  height:0;
+  width:0;
+  top:-8px;
+  border:10px solid transparent;
+  border-left: 10px solid #666;
+}
+
+.server {
+  border: 4px solid #666;
+  height: 48px;
+  width: 48px;
+  margin-left: 200px;
+}
+
+
+
+
+.hidden {
+  display: none;
+}
+
+.loadingbar {
+  width: 100px;
+  height: 14px;
+  background-color: #EEE;
+  padding: 2px;
+}
+
+.complete {
+  width: 100%;
+  height: 100%;
+  background-color: #CDC;
+  transform-origin: left;
+  /* transition: transform ease-in 0.3s; */
+}
+
+.bar {
+  display: flex;
+  width: 200px;
+  margin-bottom: 4px;
+}
+
+.bar .label {
+  font-family: sans-serif;
+  margin-left: 4px;
+  font-size: 10px;
+  display: flex;
+  justify-content: center;
+  align-content: center;
+  color: #666;
+}
+
+.searchsuperbox {
+  display: flex;
+}
+
+.loadingbars {
+  margin-left: 10px;
+  width: 100px;
+  display: inline-block;
+}

client/index.html

@@ -1,14 +1,23 @@
 <html>
   <head>
     <meta content="text/html;charset=utf-8" http-equiv="Content-Type"/>
+    <link href="css/reset.css" rel="stylesheet" />
+    <link href="css/style.css" rel="stylesheet" />
   </head>
   <body>
     <h1>Spiral Demo</h1>
-    <input id="query_idx" type="text" />
-    <input id="make_query" type="button" value="&gt;" />
 
-    <pre id="output"></pre>
+    <div class="searchsuperbox">
+      <div class="searchandsuggestions">
+        <input type="text" class="searchbox" placeholder="Article title" />
+      </div>
+      <input id="make_query" type="button" value="&gt;" />
+    </div>
 
+    <div id="output" class="articles"></div>
+
+    <!-- <script type="module" src="js/bz2.js"></script>
+    <script type="module" src="js/wtf_wikipedia-client.min.js"></script> -->
     <script type="module" src="main.js"></script>
   </body>
 </html>

client/js/bz2.js

@@ -0,0 +1,343 @@
+/*! bz2 (C) 2019-present SheetJS LLC */
+
+'use strict';
+
+(function bz2() {
+// https://www.ncbi.nlm.nih.gov/IEB/ToolBox/CPP_DOC/lxr/source/src/util/compress/bzip2/crctable.c
+  const crc32Table = [
+    0x00000000, 0x04c11db7, 0x09823b6e, 0x0d4326d9, 0x130476dc, 0x17c56b6b, 0x1a864db2, 0x1e475005,
+    0x2608edb8, 0x22c9f00f, 0x2f8ad6d6, 0x2b4bcb61, 0x350c9b64, 0x31cd86d3, 0x3c8ea00a, 0x384fbdbd,
+    0x4c11db70, 0x48d0c6c7, 0x4593e01e, 0x4152fda9, 0x5f15adac, 0x5bd4b01b, 0x569796c2, 0x52568b75,
+    0x6a1936c8, 0x6ed82b7f, 0x639b0da6, 0x675a1011, 0x791d4014, 0x7ddc5da3, 0x709f7b7a, 0x745e66cd,
+    0x9823b6e0, 0x9ce2ab57, 0x91a18d8e, 0x95609039, 0x8b27c03c, 0x8fe6dd8b, 0x82a5fb52, 0x8664e6e5,
+    0xbe2b5b58, 0xbaea46ef, 0xb7a96036, 0xb3687d81, 0xad2f2d84, 0xa9ee3033, 0xa4ad16ea, 0xa06c0b5d,
+    0xd4326d90, 0xd0f37027, 0xddb056fe, 0xd9714b49, 0xc7361b4c, 0xc3f706fb, 0xceb42022, 0xca753d95,
+    0xf23a8028, 0xf6fb9d9f, 0xfbb8bb46, 0xff79a6f1, 0xe13ef6f4, 0xe5ffeb43, 0xe8bccd9a, 0xec7dd02d,
+    0x34867077, 0x30476dc0, 0x3d044b19, 0x39c556ae, 0x278206ab, 0x23431b1c, 0x2e003dc5, 0x2ac12072,
+    0x128e9dcf, 0x164f8078, 0x1b0ca6a1, 0x1fcdbb16, 0x018aeb13, 0x054bf6a4, 0x0808d07d, 0x0cc9cdca,
+    0x7897ab07, 0x7c56b6b0, 0x71159069, 0x75d48dde, 0x6b93dddb, 0x6f52c06c, 0x6211e6b5, 0x66d0fb02,
+    0x5e9f46bf, 0x5a5e5b08, 0x571d7dd1, 0x53dc6066, 0x4d9b3063, 0x495a2dd4, 0x44190b0d, 0x40d816ba,
+    0xaca5c697, 0xa864db20, 0xa527fdf9, 0xa1e6e04e, 0xbfa1b04b, 0xbb60adfc, 0xb6238b25, 0xb2e29692,
+    0x8aad2b2f, 0x8e6c3698, 0x832f1041, 0x87ee0df6, 0x99a95df3, 0x9d684044, 0x902b669d, 0x94ea7b2a,
+    0xe0b41de7, 0xe4750050, 0xe9362689, 0xedf73b3e, 0xf3b06b3b, 0xf771768c, 0xfa325055, 0xfef34de2,
+    0xc6bcf05f, 0xc27dede8, 0xcf3ecb31, 0xcbffd686, 0xd5b88683, 0xd1799b34, 0xdc3abded, 0xd8fba05a,
+    0x690ce0ee, 0x6dcdfd59, 0x608edb80, 0x644fc637, 0x7a089632, 0x7ec98b85, 0x738aad5c, 0x774bb0eb,
+    0x4f040d56, 0x4bc510e1, 0x46863638, 0x42472b8f, 0x5c007b8a, 0x58c1663d, 0x558240e4, 0x51435d53,
+    0x251d3b9e, 0x21dc2629, 0x2c9f00f0, 0x285e1d47, 0x36194d42, 0x32d850f5, 0x3f9b762c, 0x3b5a6b9b,
+    0x0315d626, 0x07d4cb91, 0x0a97ed48, 0x0e56f0ff, 0x1011a0fa, 0x14d0bd4d, 0x19939b94, 0x1d528623,
+    0xf12f560e, 0xf5ee4bb9, 0xf8ad6d60, 0xfc6c70d7, 0xe22b20d2, 0xe6ea3d65, 0xeba91bbc, 0xef68060b,
+    0xd727bbb6, 0xd3e6a601, 0xdea580d8, 0xda649d6f, 0xc423cd6a, 0xc0e2d0dd, 0xcda1f604, 0xc960ebb3,
+    0xbd3e8d7e, 0xb9ff90c9, 0xb4bcb610, 0xb07daba7, 0xae3afba2, 0xaafbe615, 0xa7b8c0cc, 0xa379dd7b,
+    0x9b3660c6, 0x9ff77d71, 0x92b45ba8, 0x9675461f, 0x8832161a, 0x8cf30bad, 0x81b02d74, 0x857130c3,
+    0x5d8a9099, 0x594b8d2e, 0x5408abf7, 0x50c9b640, 0x4e8ee645, 0x4a4ffbf2, 0x470cdd2b, 0x43cdc09c,
+    0x7b827d21, 0x7f436096, 0x7200464f, 0x76c15bf8, 0x68860bfd, 0x6c47164a, 0x61043093, 0x65c52d24,
+    0x119b4be9, 0x155a565e, 0x18197087, 0x1cd86d30, 0x029f3d35, 0x065e2082, 0x0b1d065b, 0x0fdc1bec,
+    0x3793a651, 0x3352bbe6, 0x3e119d3f, 0x3ad08088, 0x2497d08d, 0x2056cd3a, 0x2d15ebe3, 0x29d4f654,
+    0xc5a92679, 0xc1683bce, 0xcc2b1d17, 0xc8ea00a0, 0xd6ad50a5, 0xd26c4d12, 0xdf2f6bcb, 0xdbee767c,
+    0xe3a1cbc1, 0xe760d676, 0xea23f0af, 0xeee2ed18, 0xf0a5bd1d, 0xf464a0aa, 0xf9278673, 0xfde69bc4,
+    0x89b8fd09, 0x8d79e0be, 0x803ac667, 0x84fbdbd0, 0x9abc8bd5, 0x9e7d9662, 0x933eb0bb, 0x97ffad0c,
+    0xafb010b1, 0xab710d06, 0xa6322bdf, 0xa2f33668, 0xbcb4666d, 0xb8757bda, 0xb5365d03, 0xb1f740b4,
+  ];
+
+  // generated from 1 << i, except for 32
+  const masks = [
+    0x00000000, 0x00000001, 0x00000003, 0x00000007,
+    0x0000000f, 0x0000001f, 0x0000003f, 0x0000007f,
+    0x000000ff, 0x000001ff, 0x000003ff, 0x000007ff,
+    0x00000fff, 0x00001fff, 0x00003fff, 0x00007fff,
+    0x0000ffff, 0x0001ffff, 0x0003ffff, 0x0007ffff,
+    0x000fffff, 0x001fffff, 0x003fffff, 0x007fffff,
+    0x00ffffff, 0x01ffffff, 0x03ffffff, 0x07ffffff,
+    0x0fffffff, 0x1fffffff, 0x3fffffff, -0x80000000,
+  ];
+
+  function createOrderedHuffmanTable(lengths) {
+    const z = [];
+    for (let i = 0; i < lengths.length; i += 1) {
+      z.push([i, lengths[i]]);
+    }
+    z.push([lengths.length, -1]);
+    const table = [];
+    let start = z[0][0];
+    let bits = z[0][1];
+    for (let i = 0; i < z.length; i += 1) {
+      const finish = z[i][0];
+      const endbits = z[i][1];
+      if (bits) {
+        for (let code = start; code < finish; code += 1) {
+          table.push({ code, bits, symbol: undefined });
+        }
+      }
+      start = finish;
+      bits = endbits;
+      if (endbits === -1) {
+        break;
+      }
+    }
+    table.sort((a, b) => ((a.bits - b.bits) || (a.code - b.code)));
+    let tempBits = 0;
+    let symbol = -1;
+    const fastAccess = [];
+    let current;
+    for (let i = 0; i < table.length; i += 1) {
+      const t = table[i];
+      symbol += 1;
+      if (t.bits !== tempBits) {
+        symbol <<= t.bits - tempBits;
+        tempBits = t.bits;
+        current = fastAccess[tempBits] = {};
+      }
+      t.symbol = symbol;
+      current[symbol] = t;
+    }
+    return {
+      table,
+      fastAccess,
+    };
+  }
+
+  function bwtReverse(src, primary) {
+    if (primary < 0 || primary >= src.length) {
+      throw RangeError('Out of bound');
+    }
+    const unsorted = src.slice();
+    src.sort((a, b) => a - b);
+    const start = {};
+    for (let i = src.length - 1; i >= 0; i -= 1) {
+      start[src[i]] = i;
+    }
+    const links = [];
+    for (let i = 0; i < src.length; i += 1) {
+      links.push(start[unsorted[i]]++); // eslint-disable-line no-plusplus
+    }
+    let i;
+    const first = src[i = primary];
+    const ret = [];
+    for (let j = 1; j < src.length; j += 1) {
+      const x = src[i = links[i]];
+      if (x === undefined) {
+        ret.push(255);
+      } else {
+        ret.push(x);
+      }
+    }
+    ret.push(first);
+    ret.reverse();
+    return ret;
+  }
+
+  function decompress(bytes, checkCRC = false) {
+    let index = 0;
+    let bitfield = 0;
+    let bits = 0;
+    const read = (n) => {
+      if (n >= 32) {
+        const nd = n >> 1;
+        return read(nd) * (1 << nd) + read(n - nd);
+      }
+      while (bits < n) {
+        bitfield = (bitfield << 8) + bytes[index];
+        index += 1;
+        bits += 8;
+      }
+      const m = masks[n];
+      const r = (bitfield >> (bits - n)) & m;
+      bits -= n;
+      bitfield &= ~(m << bits);
+      return r;
+    };
+
+    const magic = read(16);
+    if (magic !== 0x425A) { // 'BZ'
+      throw new Error('Invalid magic');
+    }
+    const method = read(8);
+    if (method !== 0x68) { // h for huffman
+      throw new Error('Invalid method');
+    }
+
+    let blocksize = read(8);
+    if (blocksize >= 49 && blocksize <= 57) { // 1..9
+      blocksize -= 48;
+    } else {
+      throw new Error('Invalid blocksize');
+    }
+
+    let out = new Uint8Array(bytes.length * 1.5);
+    let outIndex = 0;
+    let newCRC = -1;
+    while (true) {
+      const blocktype = read(48);
+      const crc = read(32) | 0;
+      if (blocktype === 0x314159265359) {
+        if (read(1)) {
+          throw new Error('do not support randomised');
+        }
+        const pointer = read(24);
+        const used = [];
+        const usedGroups = read(16);
+        for (let i = 1 << 15; i > 0; i >>= 1) {
+          if (!(usedGroups & i)) {
+            for (let j = 0; j < 16; j += 1) {
+              used.push(false);
+            }
+            continue; // eslint-disable-line no-continue
+          }
+          const usedChars = read(16);
+          for (let j = 1 << 15; j > 0; j >>= 1) {
+            used.push(!!(usedChars & j));
+          }
+        }
+        const groups = read(3);
+        if (groups < 2 || groups > 6) {
+          throw new Error('Invalid number of huffman groups');
+        }
+        const selectorsUsed = read(15);
+        const selectors = [];
+        const mtf = Array.from({ length: groups }, (_, i) => i);
+        for (let i = 0; i < selectorsUsed; i += 1) {
+          let c = 0;
+          while (read(1)) {
+            c += 1;
+            if (c >= groups) {
+              throw new Error('MTF table out of range');
+            }
+          }
+          const v = mtf[c];
+          for (let j = c; j > 0; mtf[j] = mtf[--j]) { // eslint-disable-line no-plusplus
+          // nothing
+          }
+          selectors.push(v);
+          mtf[0] = v;
+        }
+        const symbolsInUse = used.reduce((a, b) => a + b, 0) + 2;
+        const tables = [];
+        for (let i = 0; i < groups; i += 1) {
+          let length = read(5);
+          const lengths = [];
+          for (let j = 0; j < symbolsInUse; j += 1) {
+            if (length < 0 || length > 20) {
+              throw new Error('Huffman group length outside range');
+            }
+            while (read(1)) {
+              length -= (read(1) * 2) - 1;
+            }
+            lengths.push(length);
+          }
+          tables.push(createOrderedHuffmanTable(lengths));
+        }
+        const favourites = [];
+        for (let i = 0; i < used.length - 1; i += 1) {
+          if (used[i]) {
+            favourites.push(i);
+          }
+        }
+        let decoded = 0;
+        let selectorPointer = 0;
+        let t;
+        let r;
+        let repeat = 0;
+        let repeatPower = 0;
+        const buffer = [];
+        while (true) {
+          decoded -= 1;
+          if (decoded <= 0) {
+            decoded = 50;
+            if (selectorPointer <= selectors.length) {
+              t = tables[selectors[selectorPointer]];
+              selectorPointer += 1;
+            }
+          }
+          for (const b in t.fastAccess) {
+            if (!Object.prototype.hasOwnProperty.call(t.fastAccess, b)) {
+              continue; // eslint-disable-line no-continue
+            }
+            if (bits < b) {
+              bitfield = (bitfield << 8) + bytes[index];
+              index += 1;
+              bits += 8;
+            }
+            r = t.fastAccess[b][bitfield >> (bits - b)];
+            if (r) {
+              bitfield &= masks[bits -= b];
+              r = r.code;
+              break;
+            }
+          }
+          if (r >= 0 && r <= 1) {
+            if (repeat === 0) {
+              repeatPower = 1;
+            }
+            repeat += repeatPower << r;
+            repeatPower <<= 1;
+            continue; // eslint-disable-line no-continue
+          } else {
+            const v = favourites[0];
+            for (; repeat > 0; repeat -= 1) {
+              buffer.push(v);
+            }
+          }
+          if (r === symbolsInUse - 1) {
+            break;
+          } else {
+            const v = favourites[r - 1];
+            // eslint-disable-next-line no-plusplus
+            for (let j = r - 1; j > 0; favourites[j] = favourites[--j]) {
+            // nothing
+            }
+            favourites[0] = v;
+            buffer.push(v);
+          }
+        }
+        const nt = bwtReverse(buffer, pointer);
+        let i = 0;
+        while (i < nt.length) {
+          const c = nt[i];
+          let count = 1;
+          if ((i < nt.length - 4)
+            && nt[i + 1] === c
+            && nt[i + 2] === c
+            && nt[i + 3] === c) {
+            count = nt[i + 4] + 4;
+            i += 5;
+          } else {
+            i += 1;
+          }
+          if (outIndex + count >= out.length) {
+            const old = out;
+            out = new Uint8Array(old.length * 2);
+            out.set(old);
+          }
+          for (let j = 0; j < count; j += 1) {
+            if (checkCRC) {
+              newCRC = (newCRC << 8) ^ crc32Table[((newCRC >> 24) ^ c) & 0xff];
+            }
+            out[outIndex] = c;
+            outIndex += 1;
+          }
+        }
+        if (checkCRC) {
+          const calculatedCRC = newCRC ^ -1;
+          if (calculatedCRC !== crc) {
+            throw new Error(`CRC mismatch: ${calculatedCRC} !== ${crc}`);
+          }
+          newCRC = -1;
+        }
+      } else if (blocktype === 0x177245385090) {
+        read(bits & 0x07); // pad align
+        break;
+      } else {
+        throw new Error('Invalid bz2 blocktype');
+      }
+    }
+    return out.subarray(0, outIndex);
+  }
+
+  const exports = { decompress };
+
+  if (typeof window !== 'undefined') {
+    window.bz2 = exports; // eslint-disable-line no-undef
+  } else {
+    module.exports = exports;
+  }
+}());

client/main.js

@@ -1,10 +1,3 @@
-// Use ES module import syntax to import functionality from the module
-// that we have compiled.
-//
-// Note that the `default` import is an initialization function which
-// will "boot" the module and make it ready to use. Currently browsers
-// don't support natively imported WebAssembly as an ES module, but
-// eventually the manual initialization won't be required!
 import init, { 
     initialize,
     generate_public_parameters,
@@ -12,6 +5,11 @@ import init, {
     decode_response
 } from './pkg/client.js';
 
+import './js/bz2.js';
+import './js/wtf_wikipedia.js';
+import './js/wtf-plugin-html.js';
+wtf.extend(wtfHtml);
+
 const API_URL = "https://spiralwiki.com:8088";
 const SETUP_URL = "/setup";
 const QUERY_URL = "/query";
@@ -33,89 +31,244 @@ async function postData(url = '', data = {}, json = false) {
         let data = await response.arrayBuffer();
         return new Uint8Array(data);
     }
-  }
+}
+
+async function getData(url = '', json = false) {
+    const response = await fetch(url, {
+      method: 'GET',
+      cache: 'no-cache',
+      credentials: 'omit',
+      redirect: 'follow',
+      referrerPolicy: 'no-referrer'
+    });
+    if (json) {
+        return response.json();
+    } else {
+        let data = await response.arrayBuffer();
+        return new Uint8Array(data);
+    }
+}
 
 const api = {
     setup: async (data) => postData(API_URL + SETUP_URL, data, true),
     query: async (data) => postData(API_URL + QUERY_URL, data, false)
 }
-async function run() {
-    // First up we need to actually load the wasm file, so we use the
-    // default export to inform it where the wasm file is located on the
-    // server, and then we wait on the returned promise to wait for the
-    // wasm to be loaded.
-    //
-    // It may look like this: `await init('./pkg/without_a_bundler_bg.wasm');`,
-    // but there is also a handy default inside `init` function, which uses
-    // `import.meta` to locate the wasm file relatively to js file.
-    //
-    // Note that instead of a string you can also pass in any of the
-    // following things:
-    //
-    // * `WebAssembly.Module`
-    //
-    // * `ArrayBuffer`
-    //
-    // * `Response`
-    //
-    // * `Promise` which returns any of the above, e.g. `fetch("./path/to/wasm")`
-    //
-    // This gives you complete control over how the module is loaded
-    // and compiled.
-    //
-    // Also note that the promise, when resolved, yields the wasm module's
-    // exports which is the same as importing the `*_bg` module in other
-    // modes
-    await init();
 
-    let make_query_btn = document.getElementById("make_query");
-    let output_area = document.getElementById("output");
-
-    let has_set_up = false;
-    let id = "";
-    let client = null;
-
-    make_query_btn.onclick = async () => {
-        make_query_btn.disabled = true;
-
-        if (!has_set_up) {
-            console.log("Initializing...");
-            client = initialize();
-            console.log("done");
-            console.log("Generating public parameters...");
-            let publicParameters = generate_public_parameters(client);
-            console.log(`done (${publicParameters.length} bytes)`);
-            console.log("Sending public parameters...");
-            let setup_resp = await api.setup(publicParameters);
-            console.log("sent.");
-            console.log(setup_resp);
-            id = setup_resp["id"];
-            has_set_up = true;
+function preprocessWikiText(wikiText, targetTitle) {
+    targetTitle = targetTitle.toLowerCase();
+
+    wikiText = wikiText
+        // .replace(/<title>(.*?)<\/title><text>/gi, "<text>\n\n<h1>$1</h1>\n\n")
+        .replace(/&lt;ref&gt;[\s\S]*?&lt;\/ref&gt;/gi, "")
+        .replace(/&lt;ref[\s\S]*?&lt;\/ref&gt;/gi, "")
+        .replace(/&lt;ref[\s\S]*?\/&gt;/gi, "")
+        .replace(/&lt;![\s\S]*?--&gt;/gi, "");
+    
+    let articles = wikiText.split("<title>")
+        .filter(d => d.length > 10)
+        .filter(d => {
+            var title = "";
+            var endTitleTagIdx = d.indexOf("</title>");
+            if (endTitleTagIdx != -1) {
+                title = d.slice(0, endTitleTagIdx);
+            }
+            return title.toLowerCase() == targetTitle;
+        });
+
+    if (articles.length === 0) {
+        console.log("error decoding...");
+        return "";
+    }
+
+    let d = articles[0];
+    let articlePageMatch = d.match(/<text>/);
+    if (!articlePageMatch) {
+        console.log("error decoding...");
+        return "";
+    }
+    let startPageContentIdx = articlePageMatch.index + articlePageMatch[0].length;
+    let endPageContentIdx = d.slice(startPageContentIdx).indexOf("</text>")
+    d = d.slice(startPageContentIdx, endPageContentIdx);
+    return d;
+}
+function postProcessWikiHTML(wikiHTML, title) {
+    wikiHTML = wikiHTML.replace(/<img.*?\/>/g, "");
+    wikiHTML = "<h2 class=\"title\">"+title+"</h2>" + wikiHTML
+    return wikiHTML;
+}
+
+function resultToHtml(result, title) {
+    let decompressedData = bz2.decompress(result);
+    let wikiText = new TextDecoder("utf-8").decode(decompressedData);
+    wikiText = preprocessWikiText(wikiText, title);
+    console.log(wikiText);
+    let wikiHTML = wtf(wikiText).html();
+    wikiHTML = postProcessWikiHTML(wikiHTML, title);
+    return "<article>" + wikiHTML + "</article>";    
+}
+window.resultToHtml = resultToHtml;
+
+function addBold(suggestion, query) {
+    return '<span class="highlight">'
+        + suggestion.slice(0,query.length) 
+        + "</span>"
+        + suggestion.slice(query.length);
+}
+
+function showSuggestionsBox(suggestions, query) {
+    var htmlSuggestions = '<div class="suggestions">' 
+        + suggestions.map(m => "<div>"+addBold(m, query)+"</div>").join('') 
+        + "</div>";
+    document.querySelector('.searchbox').insertAdjacentHTML('afterend', htmlSuggestions);
+    document.querySelectorAll('.suggestions > div').forEach((el) => {
+        el.onclick = (e) => {
+            document.querySelector(".searchbox").value = el.innerHTML
+                .replace('<span class="highlight">', '')
+                .replace('</span>', '');
+            clearExistingSuggestionsBox();
+            document.querySelector('#make_query').click();
         }
+    });
+}
 
-        let targetIdx = parseInt(document.getElementById("query_idx").value, 10);
-        if (targetIdx === NaN) targetIdx = 7;
+function clearExistingSuggestionsBox() {
+    var existing = document.querySelector('.suggestions');
+    if (existing) {
+        existing.remove();
+    }
+}
+
+function hasTitle(title) {
+    return window.title_index.hasOwnProperty(title) && window.title_index[title] < window.numArticles;
+}
+
+function followRedirects(title) {
+    if (hasTitle(title)) {
+        return title;
+    } else if (window.redirects.hasOwnProperty(title) && hasTitle(window.redirects[title])) {
+        return window.redirects[title];
+    } else {
+        return null;
+    }
+}
 
-        console.log("Generating query...");
-        let query = generate_query(client, id, targetIdx);
-        console.log(`done (${query.length} bytes)`);
+function queryTitleOnClick(title) {
+    return async () => {
+        queryTitle(title);
+        return false;
+    }
+}
+
+function enableLinks(element) {
+    element.querySelectorAll('a').forEach((el) => {
+        var linkTitle = el.getAttribute("href").slice(2).replace(/_/g, " ").toLowerCase();
+        if (hasTitle(linkTitle)) {
+            el.onclick = queryTitleOnClick(linkTitle);
+        } else {
+            var redirected = followRedirects(linkTitle);
+            if (redirected !== null && hasTitle(redirected)) {
+                el.onclick = queryTitleOnClick(redirected);
+            } else {
+                el.classList.add("broken")
+            }
+        }
+    })
+}
 
-        console.log("Sending query...");
-        let response = await api.query(query);
+async function query(targetIdx, title) {    
+    if (!window.hasSetUp) {
+        console.log("Initializing...");
+        window.client = initialize();
+        console.log("done");
+        console.log("Generating public parameters...");
+        let publicParameters = generate_public_parameters(window.client);
+        console.log(`done (${publicParameters.length} bytes)`);
+        console.log("Sending public parameters...");
+        let setup_resp = await api.setup(publicParameters);
         console.log("sent.");
+        console.log(setup_resp);
+        window.id = setup_resp["id"];
+        window.hasSetUp = true;
+    }
+
+    console.log("Generating query... ("+targetIdx+")");
+    let query = generate_query(window.client, window.id, targetIdx);
+    console.log(`done (${query.length} bytes)`);
+
+    console.log("Sending query...");
+    let response = await api.query(query);
+    console.log("sent.");
 
-        console.log(`done, got (${response.length} bytes)`);
-        console.log(response);
+    console.log(`done, got (${response.length} bytes)`);
+    console.log(response);
 
-        console.log("Decoding result...");
-        let result = decode_response(client, response)
-        console.log("done.")
-        console.log("Final result:")
-        console.log(result);
+    console.log("Decoding result...");
+    let result = decode_response(window.client, response)
+    console.log("done.")
+    console.log("Final result:")
+    console.log(result);
+
+    let resultHtml = resultToHtml(result, title);
+
+    let outputArea = document.getElementById("output");
+    outputArea.innerHTML = resultHtml;
+
+    enableLinks(outputArea);
+}
+
+async function queryTitle(targetTitle) {
+    let redirectedTitle = followRedirects(targetTitle);
+    let articleIndex = window.title_index[redirectedTitle];
+    return await query(articleIndex, targetTitle);
+}
+
+async function run() {
+    await init();
+
+    window.numArticles = 65536;
+    window.articleSize = 100000;
+
+    let makeQueryBtn = document.querySelector('#make_query');
+    let searchBox = document.querySelector(".searchbox");
+
+    window.sample_data = await getData("sample.dat");
+    window.title_index = await getData("enwiki-20220320-index.json", true);
+    let keys = Object.keys(window.title_index);
+    for (var i = 0; i < keys.length; i++) {
+        let key = keys[i];
+        window.title_index[key] /= window.articleSize; 
+        window.title_index[key.toLowerCase()] = window.title_index[key];
+    }
+    let redirect_backlinks = await getData("redirects-old.json", true);
+    keys = Object.keys(redirect_backlinks);
+    window.redirects = {}
+    for (var i = 0; i < keys.length; i++) {
+        let redirect_dest = keys[i];
+        let redirect_srcs = redirect_backlinks[redirect_dest];
+        for (var j = 0; j < redirect_srcs.length; j++) {
+            window.redirects[redirect_srcs[j].toLowerCase()] = redirect_dest;
+        }
+    }
 
-        output_area.innerHTML = result.map (b => b.toString(10)).join(" ");
+    searchBox.addEventListener('input', (e) => {
+        clearExistingSuggestionsBox();
+    
+        let search = e.target.value;
+        if (search.length < 1) return;
+    
+        var matching = Object.keys(window.title_index).filter((v) => v.startsWith(search));
+        if (matching.length == 0) return;
+    
+        matching.sort();
+        if (matching.length > 10) matching = matching.slice(0, 10);
+    
+        showSuggestionsBox(matching, search);
+    })
 
-        make_query_btn.disabled = false;
+    makeQueryBtn.onclick = async () => {
+        makeQueryBtn.disabled = true;
+        await queryTitle(searchBox.value);
+        makeQueryBtn.disabled = false;
     }
 }
 run();

client/src/lib.rs

@@ -50,17 +50,18 @@ pub fn initialize(json_params: Option<String>) -> WrappedClient {
     dg_seems_okay();
     // spiral_rs::ntt::test::ntt_correct();
     let cfg = r#"
-        {'kinda_direct_upload': 1,
-        'n': 5,
-        'nu_1': 11,
-        'nu_2': 3,
-        'p': 65536,
-        'q_prime_bits': 27,
-        's_e': 57.793748020122216,
-        't_GSW': 3,
-        't_conv': 56,
-        't_exp': 56,
-        't_exp_right': 56}
+        {'n': 2,
+        'nu_1': 10,
+        'nu_2': 6,
+        'p': 512,
+        'q_prime_bits': 21,
+        's_e': 85.83255142749422,
+        't_GSW': 10,
+        't_conv': 4,
+        't_exp': 16,
+        't_exp_right': 56,
+        'instances': 11,
+        'db_item_size': 100000 }
     "#;
     let mut cfg = cfg.replace("'", "\"");
     if json_params.is_some() {

spiral-rs/src/client.rs

@@ -132,6 +132,8 @@ fn params_with_moduli(params: &Params, moduli: &Vec<u64>) -> Params {
         params.expand_queries,
         params.db_dim_1,
         params.db_dim_2,
+        params.instances,
+        params.db_item_size,
     )
 }
 
@@ -410,51 +412,60 @@ impl<'a> Client<'a> {
         let mut sk_gsw_q2_ntt = PolyMatrixNTT::zero(&q2_params, params.n, 1);
         to_ntt(&mut sk_gsw_q2_ntt, &sk_gsw_q2);
 
-        // this must be done during decoding
-        let mut first_row = PolyMatrixRaw::zero(&q2_params, 1, params.n);
-        let mut rest_rows = PolyMatrixRaw::zero(&params, params.n, params.n);
+        let mut result = PolyMatrixRaw::zero(&params, params.instances * params.n, params.n);
+
         let mut bit_offs = 0;
-        for i in 0..params.n * params.poly_len {
-            first_row.data[i] = read_arbitrary_bits(data, bit_offs, q2_bits);
-            bit_offs += q2_bits;
-        }
-        for i in 0..params.n * params.n * params.poly_len {
-            rest_rows.data[i] = read_arbitrary_bits(data, bit_offs, q1_bits);
-            bit_offs += q1_bits;
-        }
+        for instance in 0..params.instances {
+            // this must be done during decoding
+            let mut first_row = PolyMatrixRaw::zero(&q2_params, 1, params.n);
+            let mut rest_rows = PolyMatrixRaw::zero(&params, params.n, params.n);
+            for i in 0..params.n * params.poly_len {
+                first_row.data[i] = read_arbitrary_bits(data, bit_offs, q2_bits);
+                bit_offs += q2_bits;
+            }
+            for i in 0..params.n * params.n * params.poly_len {
+                rest_rows.data[i] = read_arbitrary_bits(data, bit_offs, q1_bits);
+                bit_offs += q1_bits;
+            }
 
-        let mut first_row_q2 = PolyMatrixNTT::zero(&q2_params, 1, params.n);
-        to_ntt(&mut first_row_q2, &first_row);
+            let mut first_row_q2 = PolyMatrixNTT::zero(&q2_params, 1, params.n);
+            to_ntt(&mut first_row_q2, &first_row);
 
-        let sk_prod = (&sk_gsw_q2_ntt * &first_row_q2).raw();
+            let sk_prod = (&sk_gsw_q2_ntt * &first_row_q2).raw();
 
-        let q1_i64 = q1 as i64;
-        let q2_i64 = q2 as i64;
-        let p_i128 = p as i128;
-        let mut result = PolyMatrixRaw::zero(&params, params.n, params.n);
-        for i in 0..result.rows * result.cols * params.poly_len {
-            let mut val_first = sk_prod.data[i] as i64;
-            if val_first >= q2_i64/2 {
-                val_first -= q2_i64;
-            }
-            let mut val_rest = rest_rows.data[i] as i64;
-            if val_rest >= q1_i64/2 {
-                val_rest -= q1_i64;
-            }
+            let q1_i64 = q1 as i64;
+            let q2_i64 = q2 as i64;
+            let p_i128 = p as i128;
+            for i in 0..params.n * params.n * params.poly_len {
+                let mut val_first = sk_prod.data[i] as i64;
+                if val_first >= q2_i64/2 {
+                    val_first -= q2_i64;
+                }
+                let mut val_rest = rest_rows.data[i] as i64;
+                if val_rest >= q1_i64/2 {
+                    val_rest -= q1_i64;
+                }
 
-            let denom = (q2 * (q1 / p)) as i64;
+                let denom = (q2 * (q1 / p)) as i64;
 
-            let mut r = val_first * q1_i64;
-            r += val_rest * q2_i64;
+                let mut r = val_first * q1_i64;
+                r += val_rest * q2_i64;
 
-            // divide r by q2, rounding
-            let sign: i64 = if r >= 0 { 1 } else { -1 };
-            let mut res = ((r + sign*(denom/2)) as i128) / (denom as i128);
-            res = (res + (denom as i128/p_i128)*(p_i128) + 2*(p_i128)) % (p_i128);
-            result.data[i] = res as u64;
+                // divide r by q2, rounding
+                let sign: i64 = if r >= 0 { 1 } else { -1 };
+                let mut res = ((r + sign*(denom/2)) as i128) / (denom as i128);
+                res = (res + (denom as i128/p_i128)*(p_i128) + 2*(p_i128)) % (p_i128);
+                let idx = instance * params.n * params.n * params.poly_len + i;
+                result.data[idx] = res as u64;
+            }
         }
-        println!("{:?}", result.data);
-        
-        result.to_vec(p_bits as usize)
+
+        // println!("{:?}", result.data);
+        let trials = params.n * params.n;
+        let chunks = params.instances * trials;
+        let bytes_per_chunk = f64::ceil(params.db_item_size as f64 / chunks as f64) as usize;
+        let logp = log2(params.pt_modulus);
+        let modp_words_per_chunk = f64::ceil((bytes_per_chunk * 8) as f64 / logp as f64) as usize;
+        result.to_vec(p_bits as usize, modp_words_per_chunk)
     }
 }

spiral-rs/src/main.rs

@@ -32,15 +32,17 @@ fn send_api_req_vec(path: &str, data: Vec<u8>) -> Option<Vec<u8>> {
 fn main() {
     let cfg_expand = r#"
         {'n': 2,
-        'nu_1': 9,
+        'nu_1': 10,
         'nu_2': 6,
-        'p': 256,
-        'q_prime_bits': 20,
-        's_e': 87.62938774292914,
-        't_GSW': 8,
+        'p': 512,
+        'q_prime_bits': 21,
+        's_e': 85.83255142749422,
+        't_GSW': 10,
         't_conv': 4,
-        't_exp': 8,
-        't_exp_right': 56}
+        't_exp': 16,
+        't_exp_right': 56,
+        'instances': 11,
+        'db_item_size': 100000 }
     "#;
     let cfg_direct = r#"
         {'kinda_direct_upload': 1,
@@ -55,7 +57,7 @@ fn main() {
         't_exp': 56,
         't_exp_right': 56}
     "#;
-    let cfg = cfg_direct;
+    let cfg = cfg_expand;
     let cfg = cfg.replace("'", "\"");
     let params = params_from_json(&cfg);
 
@@ -84,7 +86,8 @@ fn main() {
     let duration = now.elapsed().as_millis();
     println!("duration of query processing is {} ms", duration);
     println!("query_resp len {}", query_resp.len());
+    // println!("query_resp {:x?}", query_resp);
 
-    let _result = c.decode_response(query_resp.as_slice());
-    // println!("{:x?}", result);
+    let result = c.decode_response(query_resp.as_slice());
+    println!("{:x?}", result);
 }

spiral-rs/src/params.rs

@@ -29,6 +29,8 @@ pub struct Params {
     pub expand_queries: bool,
     pub db_dim_1: usize,
     pub db_dim_2: usize,
+    pub instances: usize,
+    pub db_item_size: usize,
 }
 
 impl Params {
@@ -95,6 +97,8 @@ impl Params {
         expand_queries: bool,
         db_dim_1: usize,
         db_dim_2: usize,
+        instances: usize,
+        db_item_size: usize,
     ) -> Self {
         let poly_len_log2 = log2(poly_len as u64) as usize;
         let crt_count = moduli.len();
@@ -125,6 +129,8 @@ impl Params {
             expand_queries,
             db_dim_1,
             db_dim_2,
+            instances,
+            db_item_size,
         }
     }
 }

spiral-rs/src/poly.rs

@@ -1,3 +1,4 @@
+use core::num;
 #[cfg(target_feature = "avx2")]
 use std::arch::x86_64::*;
 
@@ -146,16 +147,21 @@ impl<'a> PolyMatrixRaw<'a> {
         to_ntt_alloc(&self)
     }
 
-    pub fn to_vec(&self, modulus_bits: usize) -> Vec<u8> {
-        let params = self.params;
-        let sz_bits = self.rows * self.cols * params.poly_len * modulus_bits;
-        let sz_bytes = f64::ceil((sz_bits as f64) / 8f64) as usize;
+    pub fn to_vec(&self, modulus_bits: usize, num_coeffs: usize) -> Vec<u8> {
+        let sz_bits = self.rows * self.cols * num_coeffs * modulus_bits;
+        let sz_bytes = f64::ceil((sz_bits as f64) / 8f64) as usize + 32;
         let sz_bytes_roundup_16 = ((sz_bytes + 15) / 16) * 16;
         let mut data = vec![0u8; sz_bytes_roundup_16];
         let mut bit_offs = 0;
-        for i in 0..self.rows * self.cols * params.poly_len {
-            write_arbitrary_bits(data.as_mut_slice(), self.data[i], bit_offs, modulus_bits);
-            bit_offs += modulus_bits;
+        for r in 0..self.rows {
+            for c in 0..self.cols {
+                for z in 0..num_coeffs {
+                    write_arbitrary_bits(data.as_mut_slice(), self.get_poly(r,c)[z], bit_offs, modulus_bits);
+                    bit_offs += modulus_bits;
+                }
+                // round bit_offs down to nearest byte boundary
+                bit_offs = (bit_offs / 8) * 8
+            }
         }
         data
     }
@@ -558,4 +564,24 @@ mod test {
         let m3 = from_ntt_alloc(&m3_ntt);
         assert_eq!(m3.get_poly(0, 0)[2], 700);
     }
+
+    #[test]
+    fn to_vec_correctness() {
+        let params = get_params();
+        let mut m1 = PolyMatrixRaw::zero(&params, 1, 1);
+        for i in 0..params.poly_len {
+            m1.data[i] = 1;
+        }
+        let modulus_bits = 9;
+        let v = m1.to_vec(modulus_bits, params.poly_len);
+        for i in 0..v.len() {
+            println!("{:?}", v[i]);
+        }
+        let mut bit_offs = 0;
+        for i in 0..params.poly_len {
+            let val = read_arbitrary_bits(v.as_slice(), bit_offs, modulus_bits);
+            assert_eq!(m1.data[i], val);
+            bit_offs += modulus_bits;
+        }
+    }
 }

spiral-rs/src/util.rs

@@ -26,6 +26,8 @@ pub fn get_test_params() -> Params {
         true,
         9,
         6,
+        1,
+        2048
     )
 }
 
@@ -49,7 +51,9 @@ pub const fn get_empty_params() -> Params {
         t_gsw: 0, 
         expand_queries: false, 
         db_dim_1: 0, 
-        db_dim_2: 0 
+        db_dim_2: 0,
+        instances: 0,
+        db_item_size: 0,
     }
 }
 
@@ -58,6 +62,8 @@ pub fn params_from_json(cfg: &str) -> Params {
     let n = v["n"].as_u64().unwrap() as usize;
     let db_dim_1 = v["nu_1"].as_u64().unwrap() as usize;
     let db_dim_2 = v["nu_2"].as_u64().unwrap() as usize;
+    let instances = v["instances"].as_u64().unwrap_or(1) as usize;
+    let db_item_size = v["db_item_size"].as_u64().unwrap_or(1) as usize;
     let p = v["p"].as_u64().unwrap();
     let q2_bits = v["q_prime_bits"].as_u64().unwrap();
     let t_gsw = v["t_GSW"].as_u64().unwrap() as usize;
@@ -79,6 +85,8 @@ pub fn params_from_json(cfg: &str) -> Params {
         do_expansion,
         db_dim_1,
         db_dim_2,
+        instances,
+        db_item_size,
     )
 }
 
@@ -132,7 +140,9 @@ mod test {
             't_GSW': 8,
             't_conv': 4,
             't_exp': 8,
-            't_exp_right': 56}
+            't_exp_right': 56,
+            'instances': 1,
+            'db_item_size': 2048 }
         "#;
         let cfg = cfg.replace("'", "\"");
         let b = params_from_json(&cfg);
@@ -150,23 +160,28 @@ mod test {
             true,
             9,
             6,
+            1,
+            2048
         );
         assert_eq!(b, c);
     }
 
     #[test]
     fn test_read_write_arbitrary_bits() {
-        let mut data = vec![0u8; 1024];
-        let num_bits = 5;
+        let len = 4096;
+        let num_bits = 9;
+        let mut data = vec![0u8; len];
+        let scaled_len = len * 8 / num_bits - 64;
         let mut bit_offs = 0;
-        for i in 0..1000 {
-            write_arbitrary_bits(data.as_mut_slice(), i % 32, bit_offs, num_bits);
+        let get_from = |i: usize| -> u64 { ((i * 7 + 13) % (1 << num_bits)) as u64 };
+        for i in 0..scaled_len {
+            write_arbitrary_bits(data.as_mut_slice(), get_from(i), bit_offs, num_bits);
             bit_offs += num_bits;
         }
         bit_offs = 0;
-        for i in 0..1000 {
+        for i in 0..scaled_len {
             let val = read_arbitrary_bits(data.as_slice(), bit_offs, num_bits);
-            assert_eq!(val, i % 32);
+            assert_eq!(val, get_from(i));
             bit_offs += num_bits;
         }
     }