123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274 |
- import init, {
- initialize,
- generate_public_parameters,
- generate_query,
- decode_response
- } from './pkg/client.js';
- import './js/bz2.js';
- import './js/wtf_wikipedia.js';
- import './js/wtf-plugin-html.js';
- wtf.extend(wtfHtml);
- const API_URL = "https://spiralwiki.com:8088";
- const SETUP_URL = "/setup";
- const QUERY_URL = "/query";
- async function postData(url = '', data = {}, json = false) {
- const response = await fetch(url, {
- method: 'POST',
- mode: 'cors',
- cache: 'no-cache',
- credentials: 'omit',
- headers: { 'Content-Type': 'application/octet-stream' },
- redirect: 'follow',
- referrerPolicy: 'no-referrer',
- body: data
- });
- if (json) {
- return response.json();
- } else {
- let data = await response.arrayBuffer();
- return new Uint8Array(data);
- }
- }
- async function getData(url = '', json = false) {
- const response = await fetch(url, {
- method: 'GET',
- cache: 'no-cache',
- credentials: 'omit',
- redirect: 'follow',
- referrerPolicy: 'no-referrer'
- });
- if (json) {
- return response.json();
- } else {
- let data = await response.arrayBuffer();
- return new Uint8Array(data);
- }
- }
- const api = {
- setup: async (data) => postData(API_URL + SETUP_URL, data, true),
- query: async (data) => postData(API_URL + QUERY_URL, data, false)
- }
- function preprocessWikiText(wikiText, targetTitle) {
- targetTitle = targetTitle.toLowerCase();
- wikiText = wikiText
- // .replace(/<title>(.*?)<\/title><text>/gi, "<text>\n\n<h1>$1</h1>\n\n")
- .replace(/<ref>[\s\S]*?<\/ref>/gi, "")
- .replace(/<ref[\s\S]*?<\/ref>/gi, "")
- .replace(/<ref[\s\S]*?\/>/gi, "")
- .replace(/<![\s\S]*?-->/gi, "");
-
- let articles = wikiText.split("<title>")
- .filter(d => d.length > 10)
- .filter(d => {
- var title = "";
- var endTitleTagIdx = d.indexOf("</title>");
- if (endTitleTagIdx != -1) {
- title = d.slice(0, endTitleTagIdx);
- }
- return title.toLowerCase() == targetTitle;
- });
- if (articles.length === 0) {
- console.log("error decoding...");
- return "";
- }
- let d = articles[0];
- let articlePageMatch = d.match(/<text>/);
- if (!articlePageMatch) {
- console.log("error decoding...");
- return "";
- }
- let startPageContentIdx = articlePageMatch.index + articlePageMatch[0].length;
- let endPageContentIdx = d.slice(startPageContentIdx).indexOf("</text>")
- d = d.slice(startPageContentIdx, endPageContentIdx);
- return d;
- }
- function postProcessWikiHTML(wikiHTML, title) {
- wikiHTML = wikiHTML.replace(/<img.*?\/>/g, "");
- wikiHTML = "<h2 class=\"title\">"+title+"</h2>" + wikiHTML
- return wikiHTML;
- }
- function resultToHtml(result, title) {
- let decompressedData = bz2.decompress(result);
- let wikiText = new TextDecoder("utf-8").decode(decompressedData);
- wikiText = preprocessWikiText(wikiText, title);
- console.log(wikiText);
- let wikiHTML = wtf(wikiText).html();
- wikiHTML = postProcessWikiHTML(wikiHTML, title);
- return "<article>" + wikiHTML + "</article>";
- }
- window.resultToHtml = resultToHtml;
- function addBold(suggestion, query) {
- return '<span class="highlight">'
- + suggestion.slice(0,query.length)
- + "</span>"
- + suggestion.slice(query.length);
- }
- function showSuggestionsBox(suggestions, query) {
- var htmlSuggestions = '<div class="suggestions">'
- + suggestions.map(m => "<div>"+addBold(m, query)+"</div>").join('')
- + "</div>";
- document.querySelector('.searchbox').insertAdjacentHTML('afterend', htmlSuggestions);
- document.querySelectorAll('.suggestions > div').forEach((el) => {
- el.onclick = (e) => {
- document.querySelector(".searchbox").value = el.innerHTML
- .replace('<span class="highlight">', '')
- .replace('</span>', '');
- clearExistingSuggestionsBox();
- document.querySelector('#make_query').click();
- }
- });
- }
- function clearExistingSuggestionsBox() {
- var existing = document.querySelector('.suggestions');
- if (existing) {
- existing.remove();
- }
- }
- function hasTitle(title) {
- return window.title_index.hasOwnProperty(title) && window.title_index[title] < window.numArticles;
- }
- function followRedirects(title) {
- if (hasTitle(title)) {
- return title;
- } else if (window.redirects.hasOwnProperty(title) && hasTitle(window.redirects[title])) {
- return window.redirects[title];
- } else {
- return null;
- }
- }
- function queryTitleOnClick(title) {
- return async () => {
- queryTitle(title);
- return false;
- }
- }
- function enableLinks(element) {
- element.querySelectorAll('a').forEach((el) => {
- var linkTitle = el.getAttribute("href").slice(2).replace(/_/g, " ").toLowerCase();
- if (hasTitle(linkTitle)) {
- el.onclick = queryTitleOnClick(linkTitle);
- } else {
- var redirected = followRedirects(linkTitle);
- if (redirected !== null && hasTitle(redirected)) {
- el.onclick = queryTitleOnClick(redirected);
- } else {
- el.classList.add("broken")
- }
- }
- })
- }
- async function query(targetIdx, title) {
- if (!window.hasSetUp) {
- console.log("Initializing...");
- window.client = initialize();
- console.log("done");
- console.log("Generating public parameters...");
- let publicParameters = generate_public_parameters(window.client);
- console.log(`done (${publicParameters.length} bytes)`);
- console.log("Sending public parameters...");
- let setup_resp = await api.setup(publicParameters);
- console.log("sent.");
- console.log(setup_resp);
- window.id = setup_resp["id"];
- window.hasSetUp = true;
- }
- console.log("Generating query... ("+targetIdx+")");
- let query = generate_query(window.client, window.id, targetIdx);
- console.log(`done (${query.length} bytes)`);
- console.log("Sending query...");
- let response = await api.query(query);
- console.log("sent.");
- console.log(`done, got (${response.length} bytes)`);
- console.log(response);
- console.log("Decoding result...");
- let result = decode_response(window.client, response)
- console.log("done.")
- console.log("Final result:")
- console.log(result);
- let resultHtml = resultToHtml(result, title);
- let outputArea = document.getElementById("output");
- outputArea.innerHTML = resultHtml;
- enableLinks(outputArea);
- }
- async function queryTitle(targetTitle) {
- let redirectedTitle = followRedirects(targetTitle);
- let articleIndex = window.title_index[redirectedTitle];
- return await query(articleIndex, targetTitle);
- }
- async function run() {
- await init();
- window.numArticles = 65536;
- window.articleSize = 100000;
- let makeQueryBtn = document.querySelector('#make_query');
- let searchBox = document.querySelector(".searchbox");
- window.sample_data = await getData("sample.dat");
- window.title_index = await getData("enwiki-20220320-index.json", true);
- let keys = Object.keys(window.title_index);
- for (var i = 0; i < keys.length; i++) {
- let key = keys[i];
- window.title_index[key] /= window.articleSize;
- window.title_index[key.toLowerCase()] = window.title_index[key];
- }
- let redirect_backlinks = await getData("redirects-old.json", true);
- keys = Object.keys(redirect_backlinks);
- window.redirects = {}
- for (var i = 0; i < keys.length; i++) {
- let redirect_dest = keys[i];
- let redirect_srcs = redirect_backlinks[redirect_dest];
- for (var j = 0; j < redirect_srcs.length; j++) {
- window.redirects[redirect_srcs[j].toLowerCase()] = redirect_dest;
- }
- }
- searchBox.addEventListener('input', (e) => {
- clearExistingSuggestionsBox();
-
- let search = e.target.value;
- if (search.length < 1) return;
-
- var matching = Object.keys(window.title_index).filter((v) => v.startsWith(search));
- if (matching.length == 0) return;
-
- matching.sort();
- if (matching.length > 10) matching = matching.slice(0, 10);
-
- showSuggestionsBox(matching, search);
- })
- makeQueryBtn.onclick = async () => {
- makeQueryBtn.disabled = true;
- await queryTitle(searchBox.value);
- makeQueryBtn.disabled = false;
- }
- }
- run();
|