update client, support single sever executable

client/.gitignore

@@ -4,3 +4,4 @@ Cargo.lock
 bin/
 pkg/
 wasm-pack.log
+static/data

client/css/style.css → client/static/css/style.css

@@ -463,3 +463,11 @@ CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
   width: 100px;
   display: inline-block;
 }
+
+b, strong {
+  font-weight: bold;
+}
+
+i, em {
+  font-style: italic;
+}

client/index.html → client/static/index.html

@@ -18,6 +18,6 @@
 
     <!-- <script type="module" src="js/bz2.js"></script>
     <script type="module" src="js/wtf_wikipedia-client.min.js"></script> -->
-    <script type="module" src="main.js"></script>
+    <script type="module" src="js/main.js"></script>
   </body>
 </html>

client/main.js → client/static/js/main.js

@@ -3,14 +3,14 @@ import init, {
     generate_public_parameters,
     generate_query,
     decode_response
-} from './pkg/client.js';
+} from '../pkg/client.js';
 
-import './js/bz2.js';
-import './js/wtf_wikipedia.js';
-import './js/wtf-plugin-html.js';
+import './bz2.js';
+import './wtf_wikipedia.js';
+import './wtf-plugin-html.js';
 wtf.extend(wtfHtml);
 
-const API_URL = "https://spiralwiki.com:8088";
+const API_URL = "";
 const SETUP_URL = "/setup";
 const QUERY_URL = "/query";
 
@@ -18,9 +18,12 @@ async function postData(url = '', data = {}, json = false) {
     const response = await fetch(url, {
       method: 'POST',
       mode: 'cors',
-      cache: 'no-cache',
+      cache: 'no-store',
       credentials: 'omit',
-      headers: { 'Content-Type': 'application/octet-stream' },
+      headers: { 
+          'Content-Type': 'application/octet-stream',
+          'Content-Length': data.length
+      },
       redirect: 'follow',
       referrerPolicy: 'no-referrer',
       body: data
@@ -36,7 +39,7 @@ async function postData(url = '', data = {}, json = false) {
 async function getData(url = '', json = false) {
     const response = await fetch(url, {
       method: 'GET',
-      cache: 'no-cache',
+      cache: 'default',
       credentials: 'omit',
       redirect: 'follow',
       referrerPolicy: 'no-referrer'
@@ -54,25 +57,22 @@ const api = {
     query: async (data) => postData(API_URL + QUERY_URL, data, false)
 }
 
+function extractTitle(article) {
+    var title = "";
+    var endTitleTagIdx = article.indexOf("</title>");
+    if (endTitleTagIdx != -1) {
+        title = article.slice(0, endTitleTagIdx);
+    }
+    return title;
+}
+
 function preprocessWikiText(wikiText, targetTitle) {
     targetTitle = targetTitle.toLowerCase();
-
-    wikiText = wikiText
-        // .replace(/<title>(.*?)<\/title><text>/gi, "<text>\n\n<h1>$1</h1>\n\n")
-        .replace(/&lt;ref&gt;[\s\S]*?&lt;\/ref&gt;/gi, "")
-        .replace(/&lt;ref[\s\S]*?&lt;\/ref&gt;/gi, "")
-        .replace(/&lt;ref[\s\S]*?\/&gt;/gi, "")
-        .replace(/&lt;![\s\S]*?--&gt;/gi, "");
     
     let articles = wikiText.split("<title>")
         .filter(d => d.length > 10)
         .filter(d => {
-            var title = "";
-            var endTitleTagIdx = d.indexOf("</title>");
-            if (endTitleTagIdx != -1) {
-                title = d.slice(0, endTitleTagIdx);
-            }
-            return title.toLowerCase() == targetTitle;
+            return extractTitle(d).toLowerCase() == targetTitle;
         });
 
     if (articles.length === 0) {
@@ -81,6 +81,7 @@ function preprocessWikiText(wikiText, targetTitle) {
     }
 
     let d = articles[0];
+    let title = extractTitle(d);
     let articlePageMatch = d.match(/<text>/);
     if (!articlePageMatch) {
         console.log("error decoding...");
@@ -89,7 +90,17 @@ function preprocessWikiText(wikiText, targetTitle) {
     let startPageContentIdx = articlePageMatch.index + articlePageMatch[0].length;
     let endPageContentIdx = d.slice(startPageContentIdx).indexOf("</text>")
     d = d.slice(startPageContentIdx, endPageContentIdx);
-    return d;
+
+    d = d
+        .replace(/&lt;ref[\s\S]{0,500}?&lt;\/ref&gt;/gi, "")
+        .replace(/&lt;ref[\s\S]{0,500}?\/&gt;/gi, "")
+        .replace(/&lt;ref&gt;[\s\S]{0,500}?&lt;\/ref&gt;/gi, "")
+        .replace(/&lt;![\s\S]{0,500}?--&gt;/gi, "");
+
+    return {
+        "wikiText": d,
+        "title": title
+    };
 }
 function postProcessWikiHTML(wikiHTML, title) {
     wikiHTML = wikiHTML.replace(/<img.*?\/>/g, "");
@@ -100,10 +111,9 @@ function postProcessWikiHTML(wikiHTML, title) {
 function resultToHtml(result, title) {
     let decompressedData = bz2.decompress(result);
     let wikiText = new TextDecoder("utf-8").decode(decompressedData);
-    wikiText = preprocessWikiText(wikiText, title);
-    console.log(wikiText);
-    let wikiHTML = wtf(wikiText).html();
-    wikiHTML = postProcessWikiHTML(wikiHTML, title);
+    let processedData = preprocessWikiText(wikiText, title);
+    let wikiHTML = wtf(processedData.wikiText).html();
+    wikiHTML = postProcessWikiHTML(wikiHTML, processedData.title);
     return "<article>" + wikiHTML + "</article>";    
 }
 window.resultToHtml = resultToHtml;
@@ -139,7 +149,7 @@ function clearExistingSuggestionsBox() {
 }
 
 function hasTitle(title) {
-    return window.title_index.hasOwnProperty(title) && window.title_index[title] < window.numArticles;
+    return title && window.title_index.hasOwnProperty(title) && window.title_index[title] < window.numArticles;
 }
 
 function followRedirects(title) {
@@ -153,7 +163,8 @@ function followRedirects(title) {
 }
 
 function queryTitleOnClick(title) {
-    return async () => {
+    return async (e) => {
+        e.preventDefault();
         queryTitle(title);
         return false;
     }
@@ -184,7 +195,7 @@ async function query(targetIdx, title) {
         let publicParameters = generate_public_parameters(window.client);
         console.log(`done (${publicParameters.length} bytes)`);
         console.log("Sending public parameters...");
-        let setup_resp = await api.setup(publicParameters);
+        let setup_resp = await api.setup(new Blob([publicParameters.buffer]));
         console.log("sent.");
         console.log(setup_resp);
         window.id = setup_resp["id"];
@@ -231,15 +242,17 @@ async function run() {
     let makeQueryBtn = document.querySelector('#make_query');
     let searchBox = document.querySelector(".searchbox");
 
-    window.sample_data = await getData("sample.dat");
-    window.title_index = await getData("enwiki-20220320-index.json", true);
+    let title_index_p = getData("data/enwiki-20220320-index.json", true);
+    let redirect_backlinks_p = getData("data/redirects-old.json", true);
+
+    window.title_index = await title_index_p;
     let keys = Object.keys(window.title_index);
     for (var i = 0; i < keys.length; i++) {
         let key = keys[i];
         window.title_index[key] /= window.articleSize; 
         window.title_index[key.toLowerCase()] = window.title_index[key];
     }
-    let redirect_backlinks = await getData("redirects-old.json", true);
+    let redirect_backlinks = await redirect_backlinks_p;
     keys = Object.keys(redirect_backlinks);
     window.redirects = {}
     for (var i = 0; i < keys.length; i++) {

spiral-rs/Cargo.lock

@@ -19,6 +19,44 @@ dependencies = [
  "tokio-util 0.7.1",
 ]
 
+[[package]]
+name = "actix-cors"
+version = "0.6.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "414360eed71ba2d5435b185ba43ecbe281dfab5df3898286d6b7be8074372c92"
+dependencies = [
+ "actix-utils",
+ "actix-web",
+ "derive_more",
+ "futures-util",
+ "log",
+ "once_cell",
+ "smallvec",
+]
+
+[[package]]
+name = "actix-files"
+version = "0.6.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d81bde9a79336aa51ebed236e91fc1a0528ff67cfdf4f68ca4c61ede9fd26fb5"
+dependencies = [
+ "actix-http",
+ "actix-service",
+ "actix-utils",
+ "actix-web",
+ "askama_escape",
+ "bitflags",
+ "bytes",
+ "derive_more",
+ "futures-core",
+ "http-range",
+ "log",
+ "mime",
+ "mime_guess",
+ "percent-encoding",
+ "pin-project-lite",
+]
+
 [[package]]
 name = "actix-http"
 version = "3.0.4"
@@ -260,6 +298,12 @@ dependencies = [
  "nodrop",
 ]
 
+[[package]]
+name = "askama_escape"
+version = "0.10.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "619743e34b5ba4e9703bba34deac3427c72507c7159f5fd030aea8cac0cfe341"
+
 [[package]]
 name = "atty"
 version = "0.2.14"
@@ -865,6 +909,12 @@ dependencies = [
  "pin-project-lite",
 ]
 
+[[package]]
+name = "http-range"
+version = "0.1.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "21dec9db110f5f872ed9699c3ecf50cf16f423502706ba5c72462e28d3157573"
+
 [[package]]
 name = "httparse"
 version = "1.6.0"
@@ -1098,6 +1148,16 @@ version = "0.3.16"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "2a60c7ce501c71e03a9c9c0d35b861413ae925bd979cc7a4e30d060069aaac8d"
 
+[[package]]
+name = "mime_guess"
+version = "2.0.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4192263c238a5f0d0c6bfd21f336a313a4ce1c450542449ca191bb657b4642ef"
+dependencies = [
+ "mime",
+ "unicase",
+]
+
 [[package]]
 name = "miniz_oxide"
 version = "0.4.4"
@@ -1747,6 +1807,11 @@ dependencies = [
 name = "spiral-rs"
 version = "0.1.0"
 dependencies = [
+ "actix-cors",
+ "actix-files",
+ "actix-http",
+ "actix-server",
+ "actix-service",
  "actix-web",
  "criterion",
  "futures",
@@ -2012,6 +2077,15 @@ version = "1.15.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "dcf81ac59edc17cc8697ff311e8f5ef2d99fcbd9817b34cec66f90b6c3dfd987"
 
+[[package]]
+name = "unicase"
+version = "2.6.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "50f37be617794602aabbeee0be4f259dc1778fabe05e2d67ee8f79326d5cb4f6"
+dependencies = [
+ "version_check",
+]
+
 [[package]]
 name = "unicode-bidi"
 version = "0.3.7"

spiral-rs/Cargo.toml

@@ -4,7 +4,7 @@ version = "0.1.0"
 edition = "2021"
 
 [features]
-server = ["actix-web", "openssl", "futures", "uuid"]
+server = ["actix-web", "actix-cors", "actix-files", "actix-server", "actix-http", "actix-service", "openssl", "futures", "uuid"]
 
 [[bin]]
 name = "server"
@@ -17,6 +17,11 @@ reqwest = { version = "0.11", features = ["blocking"] }
 serde_json = "1.0"
 rayon = "1.5.2"
 actix-web = { version = "4.0.1", features = ["openssl"], optional = true }
+actix-cors = { version = "0.6.1", optional = true }
+actix-files = { version = "0.6.0", optional = true }
+actix-server = { version = "2.1.1", optional = true }
+actix-http = { version = "3.0.4", optional = true }
+actix-service = { version = "2.0.2", optional = true }
 openssl = { version = "0.10", features = ["v110"], optional = true }
 futures = { version = "0.3", optional = true }
 uuid = { version = "1.0.0", features = ["v4"], optional = true }

spiral-rs/src/bin/server.rs

@@ -1,4 +1,3 @@
-use actix_web::error::PayloadError;
 use futures::StreamExt;
 use spiral_rs::aligned_memory::*;
 use spiral_rs::client::*;
@@ -10,7 +9,13 @@ use std::env;
 use std::fs::File;
 use std::sync::Mutex;
 
-use actix_web::{get, http, post, web, App, HttpServer};
+use actix_cors::Cors;
+use actix_files as fs;
+use actix_http::HttpServiceBuilder;
+use actix_server::{Server, ServerBuilder};
+use actix_service::map_config;
+use actix_web::error::PayloadError;
+use actix_web::{get, http, middleware, post, web, App, HttpServer};
 use openssl::ssl::{SslAcceptor, SslFiletype, SslMethod};
 
 const CERT_FNAME: &str = "/etc/letsencrypt/live/spiralwiki.com/fullchain.pem";
@@ -28,7 +33,8 @@ async fn get_request_bytes(
 ) -> Result<Vec<u8>, http::Error> {
     let mut bytes = web::BytesMut::new();
     while let Some(item) = body.next().await {
-        bytes.extend_from_slice(&item?);
+        let item_ref = &item?;
+        bytes.extend_from_slice(item_ref);
         if bytes.len() > sz_bytes {
             println!("too big! {}", sz_bytes);
             return Err(PayloadError::Overflow.into());
@@ -56,14 +62,13 @@ async fn index<'a>(data: web::Data<ServerState<'a>>) -> String {
 
 #[post("/setup")]
 async fn setup<'a>(
-    body: web::Payload,
+    body: web::Bytes,
     data: web::Data<ServerState<'a>>,
 ) -> Result<String, http::Error> {
     println!("/setup");
 
     // Parse the request
-    let request_bytes = get_request_bytes(body, data.params.setup_bytes()).await?;
-    let pub_params = PublicParameters::deserialize(data.params, request_bytes.as_slice());
+    let pub_params = PublicParameters::deserialize(data.params, &body);
 
     // Generate a UUID and store it
     let uuid = uuid::Uuid::new_v4();
@@ -130,12 +135,6 @@ async fn main() -> std::io::Result<()> {
     let mut file = File::open(db_preprocessed_path).unwrap();
     let db = load_preprocessed_db_from_file(params, &mut file);
 
-    let mut builder = SslAcceptor::mozilla_intermediate(SslMethod::tls()).unwrap();
-    builder
-        .set_private_key_file(KEY_FNAME, SslFiletype::PEM)
-        .unwrap();
-    builder.set_certificate_chain_file(CERT_FNAME).unwrap();
-
     let server_state = ServerState {
         params: params,
         db: db,
@@ -143,17 +142,42 @@ async fn main() -> std::io::Result<()> {
     };
     let state = web::Data::new(server_state);
 
-    let res = HttpServer::new(move || {
+    let app_builder = move || {
+        let cors = Cors::default()
+            .allow_any_origin()
+            .allowed_headers([
+                http::header::ORIGIN,
+                http::header::CONTENT_TYPE,
+                http::header::ACCEPT,
+            ])
+            .allow_any_method()
+            .max_age(3600);
+
         App::new()
+            .wrap(middleware::Compress::default())
+            .wrap(cors)
             .app_data(state.clone())
-            .service(index)
+            .app_data(web::PayloadConfig::new(1 << 25))
             .service(setup)
             .service(query)
-    })
-    .bind_openssl("0.0.0.0:8088", builder)?
-    // .bind_openssl("127.0.0.1:7777", builder)?
-    .run()
-    .await;
+            .service(fs::Files::new("/", "../client/static").index_file("index.html"))
+    };
 
-    res
+    Server::build()
+        .bind("http/1", "0.0.0.0:8088", move || {
+            let mut builder = SslAcceptor::mozilla_intermediate(SslMethod::tls()).unwrap();
+            builder
+                .set_private_key_file(KEY_FNAME, SslFiletype::PEM)
+                .unwrap();
+            builder.set_certificate_chain_file(CERT_FNAME).unwrap();
+            builder.set_alpn_protos(b"\x08http/1.1").unwrap();
+
+            HttpServiceBuilder::default()
+                .h1(map_config(app_builder(), |_| {
+                    actix_web::dev::AppConfig::default()
+                }))
+                .openssl(builder.build())
+        })?
+        .run()
+        .await
 }

spiral-rs/src/util.rs

@@ -131,23 +131,20 @@ pub fn get_no_expansion_testing_params() -> Params {
     params_from_json(&cfg.replace("'", "\""))
 }
 
-pub fn get_seed() -> [u8; 32] {
-    thread_rng().gen::<[u8; 32]>()
+pub fn get_seed() -> u64 {
+    thread_rng().gen::<u64>()
 }
 
 pub fn get_seeded_rng() -> SmallRng {
-    SmallRng::from_seed(get_seed())
+    SmallRng::seed_from_u64(get_seed())
 }
 
-pub fn get_static_seed() -> [u8; 32] {
-    [
-        1, 2, 3, 4, 5, 6, 7, 8, 1, 2, 3, 4, 5, 6, 7, 8, 1, 2, 3, 4, 5, 6, 7, 8, 1, 2, 3, 4, 5, 6,
-        7, 8,
-    ]
+pub fn get_static_seed() -> u64 {
+    0x123456789
 }
 
 pub fn get_static_seeded_rng() -> SmallRng {
-    SmallRng::from_seed(get_static_seed())
+    SmallRng::seed_from_u64(get_static_seed())
 }
 
 pub const fn get_empty_params() -> Params {