import init, {
initialize,
generate_public_parameters,
generate_query,
decode_response
} from './pkg/client.js';
import './js/bz2.js';
import './js/wtf_wikipedia.js';
import './js/wtf-plugin-html.js';
wtf.extend(wtfHtml);
const API_URL = "https://spiralwiki.com:8088";
const SETUP_URL = "/setup";
const QUERY_URL = "/query";
async function postData(url = '', data = {}, json = false) {
const response = await fetch(url, {
method: 'POST',
mode: 'cors',
cache: 'no-cache',
credentials: 'omit',
headers: { 'Content-Type': 'application/octet-stream' },
redirect: 'follow',
referrerPolicy: 'no-referrer',
body: data
});
if (json) {
return response.json();
} else {
let data = await response.arrayBuffer();
return new Uint8Array(data);
}
}
async function getData(url = '', json = false) {
const response = await fetch(url, {
method: 'GET',
cache: 'no-cache',
credentials: 'omit',
redirect: 'follow',
referrerPolicy: 'no-referrer'
});
if (json) {
return response.json();
} else {
let data = await response.arrayBuffer();
return new Uint8Array(data);
}
}
const api = {
setup: async (data) => postData(API_URL + SETUP_URL, data, true),
query: async (data) => postData(API_URL + QUERY_URL, data, false)
}
function preprocessWikiText(wikiText, targetTitle) {
targetTitle = targetTitle.toLowerCase();
wikiText = wikiText
// .replace(/
(.*?)<\/title>/gi, "\n\n$1
\n\n")
.replace(/<ref>[\s\S]*?<\/ref>/gi, "")
.replace(/<ref[\s\S]*?<\/ref>/gi, "")
.replace(/<ref[\s\S]*?\/>/gi, "")
.replace(/<![\s\S]*?-->/gi, "");
let articles = wikiText.split("")
.filter(d => d.length > 10)
.filter(d => {
var title = "";
var endTitleTagIdx = d.indexOf("");
if (endTitleTagIdx != -1) {
title = d.slice(0, endTitleTagIdx);
}
return title.toLowerCase() == targetTitle;
});
if (articles.length === 0) {
console.log("error decoding...");
return "";
}
let d = articles[0];
let articlePageMatch = d.match(//);
if (!articlePageMatch) {
console.log("error decoding...");
return "";
}
let startPageContentIdx = articlePageMatch.index + articlePageMatch[0].length;
let endPageContentIdx = d.slice(startPageContentIdx).indexOf("")
d = d.slice(startPageContentIdx, endPageContentIdx);
return d;
}
function postProcessWikiHTML(wikiHTML, title) {
wikiHTML = wikiHTML.replace(//g, "");
wikiHTML = ""+title+"
" + wikiHTML
return wikiHTML;
}
function resultToHtml(result, title) {
let decompressedData = bz2.decompress(result);
let wikiText = new TextDecoder("utf-8").decode(decompressedData);
wikiText = preprocessWikiText(wikiText, title);
console.log(wikiText);
let wikiHTML = wtf(wikiText).html();
wikiHTML = postProcessWikiHTML(wikiHTML, title);
return "" + wikiHTML + "";
}
window.resultToHtml = resultToHtml;
function addBold(suggestion, query) {
return ''
+ suggestion.slice(0,query.length)
+ ""
+ suggestion.slice(query.length);
}
function showSuggestionsBox(suggestions, query) {
var htmlSuggestions = ''
+ suggestions.map(m => "
"+addBold(m, query)+"
").join('')
+ "
";
document.querySelector('.searchbox').insertAdjacentHTML('afterend', htmlSuggestions);
document.querySelectorAll('.suggestions > div').forEach((el) => {
el.onclick = (e) => {
document.querySelector(".searchbox").value = el.innerHTML
.replace('', '')
.replace('', '');
clearExistingSuggestionsBox();
document.querySelector('#make_query').click();
}
});
}
function clearExistingSuggestionsBox() {
var existing = document.querySelector('.suggestions');
if (existing) {
existing.remove();
}
}
function hasTitle(title) {
return window.title_index.hasOwnProperty(title) && window.title_index[title] < window.numArticles;
}
function followRedirects(title) {
if (hasTitle(title)) {
return title;
} else if (window.redirects.hasOwnProperty(title) && hasTitle(window.redirects[title])) {
return window.redirects[title];
} else {
return null;
}
}
function queryTitleOnClick(title) {
return async () => {
queryTitle(title);
return false;
}
}
function enableLinks(element) {
element.querySelectorAll('a').forEach((el) => {
var linkTitle = el.getAttribute("href").slice(2).replace(/_/g, " ").toLowerCase();
if (hasTitle(linkTitle)) {
el.onclick = queryTitleOnClick(linkTitle);
} else {
var redirected = followRedirects(linkTitle);
if (redirected !== null && hasTitle(redirected)) {
el.onclick = queryTitleOnClick(redirected);
} else {
el.classList.add("broken")
}
}
})
}
async function query(targetIdx, title) {
if (!window.hasSetUp) {
console.log("Initializing...");
window.client = initialize();
console.log("done");
console.log("Generating public parameters...");
let publicParameters = generate_public_parameters(window.client);
console.log(`done (${publicParameters.length} bytes)`);
console.log("Sending public parameters...");
let setup_resp = await api.setup(publicParameters);
console.log("sent.");
console.log(setup_resp);
window.id = setup_resp["id"];
window.hasSetUp = true;
}
console.log("Generating query... ("+targetIdx+")");
let query = generate_query(window.client, window.id, targetIdx);
console.log(`done (${query.length} bytes)`);
console.log("Sending query...");
let response = await api.query(query);
console.log("sent.");
console.log(`done, got (${response.length} bytes)`);
console.log(response);
console.log("Decoding result...");
let result = decode_response(window.client, response)
console.log("done.")
console.log("Final result:")
console.log(result);
let resultHtml = resultToHtml(result, title);
let outputArea = document.getElementById("output");
outputArea.innerHTML = resultHtml;
enableLinks(outputArea);
}
async function queryTitle(targetTitle) {
let redirectedTitle = followRedirects(targetTitle);
let articleIndex = window.title_index[redirectedTitle];
return await query(articleIndex, targetTitle);
}
async function run() {
await init();
window.numArticles = 65536;
window.articleSize = 100000;
let makeQueryBtn = document.querySelector('#make_query');
let searchBox = document.querySelector(".searchbox");
window.sample_data = await getData("sample.dat");
window.title_index = await getData("enwiki-20220320-index.json", true);
let keys = Object.keys(window.title_index);
for (var i = 0; i < keys.length; i++) {
let key = keys[i];
window.title_index[key] /= window.articleSize;
window.title_index[key.toLowerCase()] = window.title_index[key];
}
let redirect_backlinks = await getData("redirects-old.json", true);
keys = Object.keys(redirect_backlinks);
window.redirects = {}
for (var i = 0; i < keys.length; i++) {
let redirect_dest = keys[i];
let redirect_srcs = redirect_backlinks[redirect_dest];
for (var j = 0; j < redirect_srcs.length; j++) {
window.redirects[redirect_srcs[j].toLowerCase()] = redirect_dest;
}
}
searchBox.addEventListener('input', (e) => {
clearExistingSuggestionsBox();
let search = e.target.value;
if (search.length < 1) return;
var matching = Object.keys(window.title_index).filter((v) => v.startsWith(search));
if (matching.length == 0) return;
matching.sort();
if (matching.length > 10) matching = matching.slice(0, 10);
showSuggestionsBox(matching, search);
})
makeQueryBtn.onclick = async () => {
makeQueryBtn.disabled = true;
await queryTitle(searchBox.value);
makeQueryBtn.disabled = false;
}
}
run();