PIRReplyGeneratorNFL_internal.cpp 32 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921
  1. /* Copyright (C) 2014 Carlos Aguilar Melchor, Joris Barrier, Marc-Olivier Killijian
  2. * This file is part of XPIR.
  3. *
  4. * XPIR is free software: you can redistribute it and/or modify
  5. * it under the terms of the GNU General Public License as published by
  6. * the Free Software Foundation, either version 3 of the License, or
  7. * (at your option) any later version.
  8. *
  9. * XPIR is distributed in the hope that it will be useful,
  10. * but WITHOUT ANY WARRANTY; without even the implied warranty of
  11. * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  12. * GNU General Public License for more details.
  13. *
  14. * You should have received a copy of the GNU General Public License
  15. * along with XPIR. If not, see <http://www.gnu.org/licenses/>.
  16. */
  17. #include "PIRReplyGeneratorNFL_internal.hpp"
  18. #include "sys/time.h"
  19. //#define SIMULATE_PRE_NFL_DATA //Use this to simulate imported data is in NFL form
  20. //#define TEST_NFL_PERF_ITERATIONS //Loop to simulate very large databases
  21. //#define SNIFFER_CHUNK_BYTESIZE 12800
  22. //#define SNIFFER_CHUNK_BYTESIZE 2560
  23. //#define SNIFFER_CHUNK_BYTESIZE 1500
  24. //#define SNIFFER_CHUNK_BYTESIZE 600
  25. //#define SNIFFER //Use this to activate a sniffer like behavior
  26. PIRReplyGeneratorNFL_internal::PIRReplyGeneratorNFL_internal():
  27. lwe(false),
  28. currentMaxNbPolys(0),
  29. queriesBuf(NULL),
  30. current_query_index(0),
  31. current_dim_index(0),
  32. input_data(NULL),
  33. cryptoMethod(NULL)
  34. {
  35. }
  36. /**
  37. * Constructor of the class.
  38. * Params :
  39. * - vector <File*>& database : reference of a File pointer vector.
  40. * - PIRParameters& param : reference to a PIRParameters object.
  41. **/
  42. PIRReplyGeneratorNFL_internal::PIRReplyGeneratorNFL_internal( PIRParameters& param, DBHandler* db):
  43. lwe(false),
  44. currentMaxNbPolys(0),
  45. GenericPIRReplyGenerator(param,db),
  46. queriesBuf(NULL),
  47. current_query_index(0),
  48. current_dim_index(0),
  49. input_data(NULL),
  50. cryptoMethod(NULL)
  51. {
  52. // cryptoMethod will be set later by setCryptoMethod
  53. }
  54. void PIRReplyGeneratorNFL_internal::importFakeData(uint64_t plaintext_nbr)
  55. {
  56. uint64_t files_nbr = 1;
  57. for (unsigned int i = 0 ; i < pirParam.d ; i++) files_nbr *= pirParam.n[i];
  58. uint64_t plain_bytesize = cryptoMethod->getnflInstance().getpolyDegree()*cryptoMethod->getnflInstance().getnbModuli()*8;
  59. dbhandler = new DBGenerator(files_nbr, plaintext_nbr*plain_bytesize, true);
  60. currentMaxNbPolys = plaintext_nbr;
  61. importDataNFL(0, plaintext_nbr*plain_bytesize);
  62. }
  63. /**
  64. * Convert raw data from file in usable NFL data.
  65. **/
  66. void PIRReplyGeneratorNFL_internal::importDataNFL(uint64_t offset, uint64_t bytes_per_file)
  67. {
  68. uint64_t fileByteSize = min(bytes_per_file, dbhandler->getmaxFileBytesize()-offset);
  69. uint64_t theoretical_files_nbr = 1;
  70. uint64_t nbFiles = dbhandler->getNbStream();
  71. for (unsigned int i = 0 ; i < pirParam.d ; i++) theoretical_files_nbr *= pirParam.n[i];
  72. input_data = (lwe_in_data *) malloc(sizeof(lwe_in_data)*theoretical_files_nbr);
  73. char *rawBits = (char*)calloc(fileByteSize*pirParam.alpha, sizeof(char));
  74. currentMaxNbPolys=0;
  75. #ifdef PERF_TIMERS
  76. double vtstart = omp_get_wtime();
  77. bool wasVerbose = false;
  78. uint64_t lastindex = 0;
  79. #endif
  80. // For global time measurement
  81. double start = omp_get_wtime();double now,delta;
  82. int nbruns=ceil((double)nbFiles/pirParam.alpha);
  83. // WARNING this section should not be multithreade as rawbits is shared and readAggregatedStream
  84. // is not threadsafe
  85. for (int i=0; i < nbruns; i++)
  86. {
  87. dbhandler->readAggregatedStream(i, pirParam.alpha, offset, bytes_per_file, rawBits);
  88. #ifdef SIMULATE_PRE_NFL_DATA
  89. uint64_t abssize = cryptoMethod->getPublicParameters().getAbsorptionBitsize();
  90. uint64_t polysize = cryptoMethod->getpolyDegree() * cryptoMethod->getnbModuli()*sizeof(uint64_t);
  91. uint64_t nbpolys = ceil((double)fileByteSize * pirParam.alpha * 8 / abssize);
  92. input_data[i].p = (poly64*) malloc(nbpolys*sizeof(poly64*));
  93. input_data[i].p[0] = (poly64) malloc(nbpolys*polysize);
  94. for (unsigned j = 0; j < nbpolys ; j++)
  95. {
  96. input_data[i].p[j] = input_data[i].p[0]+j*polysize/8;
  97. memcpy(input_data[i].p[j], rawBits, min(fileByteSize, polysize));
  98. }
  99. input_data[i].nbPolys = nbpolys;
  100. #else
  101. input_data[i].p = cryptoMethod->deserializeDataNFL((unsigned char**)&rawBits, (uint64_t) 1, fileByteSize*pirParam.alpha*GlobalConstant::kBitsPerByte, input_data[i].nbPolys);
  102. #endif
  103. #ifdef PERF_TIMERS
  104. // Give some feedback if it takes too long
  105. double vtstop = omp_get_wtime();
  106. if (vtstop - vtstart > 1)
  107. {
  108. vtstart = vtstop;
  109. std::cout <<"PIRReplyGeneratorNFL_internal: Element " << i+1 << "/" << nbruns << " imported\r" << std::flush;
  110. wasVerbose = true;
  111. lastindex = i+1;
  112. }
  113. #endif
  114. }
  115. for (int i=0; i < nbruns; i++)
  116. if (input_data[i].nbPolys>currentMaxNbPolys) currentMaxNbPolys=input_data[i].nbPolys;
  117. #ifdef PERF_TIMERS
  118. // If feedback was given say we finished
  119. if (wasVerbose && lastindex != nbFiles) std::cout <<"PIRReplyGeneratorNFL_internal: Element " << nbruns << "/" << nbFiles/pirParam.alpha << " imported" << std::endl;
  120. #endif
  121. /** FILE PADDING **/
  122. for (uint64_t i = ceil((double)nbFiles/pirParam.alpha) ; i < theoretical_files_nbr ; i++)
  123. {
  124. input_data[i].p = (poly64 *) malloc(currentMaxNbPolys*sizeof(poly64));
  125. input_data[i].p[0] = (poly64) calloc(cryptoMethod->getpolyDegree()*cryptoMethod->getnbModuli()*currentMaxNbPolys,sizeof(uint64_t));
  126. for (uint64_t j = 1 ; j < currentMaxNbPolys ; j++) input_data[i].p[j] = input_data[i].p[0]+cryptoMethod->getpolyDegree()*cryptoMethod->getnbModuli()*j;
  127. input_data[i].nbPolys = currentMaxNbPolys;
  128. }
  129. free(rawBits);
  130. std::cout<<"PIRReplyGeneratorNFL_internal: Finished importing the database in " << omp_get_wtime() - start << " seconds" << std::endl;
  131. }
  132. #ifdef SNIFFER
  133. imported_database_t PIRReplyGeneratorNFL_internal::generateReplyGeneric(bool keep_imported_data)
  134. {
  135. imported_database_t database_wrapper;
  136. boost::mutex::scoped_lock l(mutex);
  137. const uint64_t chunkBytesize = SNIFFER_CHUNK_BYTESIZE;
  138. const uint64_t iterations = dbhandler->getmaxFileBytesize()/(chunkBytesize+1);
  139. // std::ifstream *is = dbhandler->openStream(0,0);
  140. const uint64_t nbFiles = dbhandler->getNbStream();
  141. const unsigned int polysize = cryptoMethod->getpolyDegree()*cryptoMethod->getnbModuli();
  142. const unsigned int jumpcipher = 2*polysize / sizeof(uint64_t);
  143. currentMaxNbPolys=0;
  144. lwe_in_data *input = new lwe_in_data[iterations];
  145. lwe_cipher *resul = new lwe_cipher[iterations];
  146. uint64_t index;
  147. lwe_query **queries;
  148. char **rawBits = (char**) malloc(iterations*sizeof(char*));
  149. cryptoMethod->setandgetAbsBitPerCiphertext(1);
  150. queries = queriesBuf[0];
  151. const uint64_t mask = (1<<((int)log2(nbFiles)))-1;
  152. for (uint64_t it = 0 ; it < iterations ; it++)
  153. {
  154. resul[it].a = (uint64_t *) malloc(2*polysize*sizeof(uint64_t));
  155. resul[it].b = (uint64_t *) resul[it].a + polysize;
  156. rawBits[it] = (char*)malloc(((chunkBytesize)*sizeof(char)+sizeof(int)));
  157. }
  158. double start = omp_get_wtime();
  159. #pragma omp parallel for firstprivate(input,queries,resul)
  160. for (uint64_t it = 0 ; it < iterations ; it++)
  161. {
  162. dbhandler->readStream(0, rawBits[it], chunkBytesize+sizeof(int));
  163. input[it].p = cryptoMethod->deserializeDataNFL((unsigned char**)&(rawBits[it]), (uint64_t) 1, chunkBytesize*GlobalConstant::kBitsPerByte, input[it].nbPolys);
  164. index = *(int *)(rawBits[it]+chunkBytesize) & mask;
  165. cryptoMethod->mul(resul[it], input[it], queries[0][index],queries[1][index], 0, 0);
  166. }
  167. double end = omp_get_wtime();
  168. std::cout<<"PIRReplyGeneratorNFL_internal: Finished processing the sniffed data in " << end - start << " seconds" << std::endl;
  169. std::cout<<"PIRReplyGeneratorNFL_internal: Processing throughput " << (double)chunkBytesize*8*iterations / ((end - start)*1000000000ULL) << " Gbps" << std::endl;
  170. return database_wrapper;
  171. }
  172. #else
  173. imported_database_t PIRReplyGeneratorNFL_internal::generateReplyGeneric(bool keep_imported_data)
  174. {
  175. imported_database_t database_wrapper;
  176. uint64_t usable_memory, database_size, max_memory_per_file, max_readable_size, nbr_of_iterations;
  177. double start, end;
  178. // Init database_wrapper to NULL values so that we are able to know if it has been initialized
  179. database_wrapper.imported_database_ptr = NULL;
  180. database_wrapper.nbElements = 0;
  181. database_wrapper.polysPerElement = 0;
  182. database_wrapper.beforeImportElementBytesize = 0;
  183. // Don't use more than half of the computer's memory
  184. usable_memory = getTotalSystemMemory()/2;
  185. database_size = dbhandler->getmaxFileBytesize() * dbhandler->getNbStream();
  186. #ifndef TEST_NFL_PERF_ITERATIONS
  187. // This is the maximum amount of data per file we can get in memory
  188. max_memory_per_file = usable_memory / dbhandler->getNbStream();
  189. // Given the expansion factor of importation we get the max we can read per file
  190. max_readable_size = max_memory_per_file / 4 ;
  191. // Reduce it so that we have full absorption in all the ciphertexts sent
  192. max_readable_size = (max_readable_size * GlobalConstant::kBitsPerByte / cryptoMethod->getPublicParameters().getAbsorptionBitsize(0)) * cryptoMethod->getPublicParameters().getAbsorptionBitsize(0)/GlobalConstant::kBitsPerByte;
  193. #else
  194. // For our tests we will need to have databases of an integer amount of gigabits
  195. max_readable_size = 1280000000UL/dbhandler->getNbStream();
  196. #endif
  197. // If we reduced it too much set it at least to a ciphertext
  198. if (max_readable_size == 0) max_readable_size = cryptoMethod->getPublicParameters().getAbsorptionBitsize(0);
  199. // Ensure it is not larger than maxfilebytesize
  200. max_readable_size = min(max_readable_size, dbhandler->getmaxFileBytesize());
  201. // Given readable size we get how many iterations we need
  202. nbr_of_iterations = ceil((double)dbhandler->getmaxFileBytesize()/max_readable_size);
  203. #ifndef TEST_NFL_PERF_ITERATIONS
  204. // If aggregation is used we cannot iterate
  205. if ((pirParam.alpha != 1 || pirParam.d > 1) && nbr_of_iterations > 1)
  206. {
  207. std::cout << "PIRReplyGeneratorNFL_internal: Cannot handle aggregation or dimensions on databases requiring multiple iterations" << std::endl;
  208. std::cout << "PIRReplyGeneratorNFL_internal: Handling the database on a single iteration, this can cause memory issues ..." << std::endl;
  209. nbr_of_iterations = 1;
  210. max_readable_size = dbhandler->getmaxFileBytesize();
  211. }
  212. // If we cannot read the whole database we cannot store it precomputed
  213. if (nbr_of_iterations > 1) keep_imported_data = false;
  214. #endif
  215. // If we need to do more than an iteration say it
  216. if (nbr_of_iterations > 1)
  217. {
  218. std::cout << "PIRReplyGeneratorNFL_internal: Database is considered too large, processing it in "
  219. << nbr_of_iterations << " iterations" << std::endl;
  220. }
  221. start = omp_get_wtime();
  222. // #pragma omp parallel for
  223. for (unsigned iteration = 0; iteration < nbr_of_iterations; iteration++)
  224. {
  225. if (nbr_of_iterations > 1) cout << "PIRReplyGeneratorNFL_internal: Iteration " << iteration << endl;
  226. repliesIndex = computeReplySizeInChunks(iteration*max_readable_size);
  227. // Import a chunk of max_readable_size bytes per file with an adapted offset
  228. importDataNFL(iteration*max_readable_size, max_readable_size);
  229. if(keep_imported_data && iteration == nbr_of_iterations - 1) // && added for Perf test but is no harmful
  230. {
  231. database_wrapper.polysPerElement = currentMaxNbPolys;
  232. }
  233. boost::mutex::scoped_lock l(mutex);
  234. repliesAmount = computeReplySizeInChunks(dbhandler->getmaxFileBytesize());
  235. generateReply();
  236. end = omp_get_wtime();
  237. if(keep_imported_data && iteration == nbr_of_iterations - 1) // && added for Perf test but is no harmful
  238. {
  239. database_wrapper.imported_database_ptr = (void*)input_data;
  240. database_wrapper.beforeImportElementBytesize = dbhandler->getmaxFileBytesize();
  241. database_wrapper.nbElements = dbhandler->getNbStream();
  242. }
  243. else
  244. {
  245. freeInputData();
  246. }
  247. }
  248. std::cout<<"PIRReplyGeneratorNFL_internal: Total process time " << end - start << " seconds" << std::endl;
  249. std::cout<<"PIRReplyGeneratorNFL_internal: DB processing throughput " << 8*database_size/(end - start) << "bps" << std::endl;
  250. std::cout<<"PIRReplyGeneratorNFL_internal: Client cleartext reception throughput " << 8*dbhandler->getmaxFileBytesize()/(end - start) << "bps" << std::endl;
  251. freeQueries();
  252. return database_wrapper;
  253. }
  254. #endif
  255. // Function used to generate a PIR reply if:
  256. // - database is small enough to be kept in memory
  257. // - it has already been imported to it
  258. void PIRReplyGeneratorNFL_internal::generateReplyGenericFromData(const imported_database_t database)
  259. {
  260. #ifndef TEST_NFL_PERF_ITERATIONS
  261. input_data = (lwe_in_data*) database.imported_database_ptr;
  262. currentMaxNbPolys = database.polysPerElement;
  263. boost::mutex::scoped_lock l(mutex);
  264. double start = omp_get_wtime();
  265. repliesAmount = computeReplySizeInChunks(database.beforeImportElementBytesize);
  266. generateReply();
  267. #else
  268. uint64_t max_readable_size, database_size, nbr_of_iterations;
  269. database_size = database.beforeImportElementBytesize * database.nbElements;
  270. max_readable_size = 1280000000UL/database.nbElements;
  271. // Ensure it is not larger than maxfilebytesize
  272. max_readable_size = min(max_readable_size, database.beforeImportElementBytesize);
  273. // Given readable size we get how many iterations we need
  274. nbr_of_iterations = ceil((double)database.beforeImportElementBytesize/max_readable_size);
  275. boost::mutex::scoped_lock l(mutex);
  276. double start = omp_get_wtime();
  277. for (unsigned iteration = 0; iteration < nbr_of_iterations; iteration++)
  278. {
  279. input_data = (lwe_in_data*) database.imported_database_ptr;
  280. currentMaxNbPolys = database.polysPerElement;
  281. repliesAmount = computeReplySizeInChunks(database.beforeImportElementBytesize);
  282. generateReply();
  283. }
  284. freeInputData();
  285. #endif
  286. double end = omp_get_wtime();
  287. std::cout<<"PIRReplyGeneratorNFL_internal: Total process time " << end - start << " seconds" << std::endl;
  288. std::cout<<"PIRReplyGeneratorNFL_internal: DB processing throughput " << 8*dbhandler->getmaxFileBytesize()*dbhandler->getNbStream()/(end - start) << "bps" << std::endl;
  289. std::cout<<"PIRReplyGeneratorNFL_internal: Client cleartext reception throughput " << 8*dbhandler->getmaxFileBytesize()/(end - start) << "bps" << std::endl;
  290. freeQueries();
  291. }
  292. /**
  293. * Prepare reply and start absoptions.
  294. **/
  295. void PIRReplyGeneratorNFL_internal::generateReply()
  296. {
  297. lwe_in_data *in_data = input_data;
  298. lwe_cipher **inter_reply;
  299. #ifdef SHOUP
  300. lwe_query **queries;
  301. #else
  302. lwe_query *queries;
  303. #endif
  304. uint64_t old_reply_elt_nbr = 0;
  305. uint64_t reply_elt_nbr = 1;
  306. uint64_t old_poly_nbr = 1;
  307. // Allocate memory for the reply array
  308. if (repliesArray != NULL) freeResult();
  309. repliesArray = (char**)calloc(repliesAmount,sizeof(char*));
  310. // Start global timers
  311. double start = omp_get_wtime();
  312. #ifdef PERF_TIMERS
  313. double vtstart = start;
  314. bool wasVerbose = false;
  315. #endif
  316. for (unsigned int i = 0 ; i < pirParam.d ; i++) // For each recursion level
  317. {
  318. old_reply_elt_nbr = reply_elt_nbr;
  319. reply_elt_nbr = 1;
  320. for (unsigned int j = i + 1 ; j < pirParam.d ; j++ ) reply_elt_nbr *= pirParam.n[j];
  321. #ifdef DEBUG
  322. cout << "PIRReplyGeneratorNFL_internal: currentMaxNbPolys = " << currentMaxNbPolys << endl;
  323. #endif
  324. inter_reply = new lwe_cipher*[reply_elt_nbr]();
  325. queries = queriesBuf[i];
  326. for (uint64_t j = 0 ; j < reply_elt_nbr ; j++) // Boucle de reply_elt_nbr PIR
  327. {
  328. inter_reply[j] = new lwe_cipher[currentMaxNbPolys];
  329. // Warning of the trick in case SHOUP is defined : we cast quesries to a (lwe_query*) and will have to uncast it
  330. generateReply((lwe_query*)queries , in_data + (pirParam.n[i] * j ), i, inter_reply[j]);
  331. #ifdef DEBUG_WITH_FILE_OUTPUT
  332. if (i ==0 && j==1) {
  333. std::ofstream file(std::string("output_level_"+ std::to_string(i)).c_str(), std::ios::out| std::ios::binary);
  334. for (int k = 0 ; k < currentMaxNbPolys ; k++)
  335. {
  336. file.write((char*)inter_reply[j][k].a,1024*2*8);
  337. }
  338. file.close();
  339. }
  340. #endif
  341. #ifdef PERF_TIMERS
  342. // Give some feedback if it takes too long
  343. double vtstop = omp_get_wtime();
  344. if (vtstop - vtstart > 1)
  345. {
  346. vtstart = vtstop;
  347. std::cout <<"PIRReplyGeneratorNFL_internal: Reply " << j+1 << "/" << reply_elt_nbr << " generated\r" << std::flush;
  348. wasVerbose = true;
  349. }
  350. #endif
  351. }
  352. /*****************/
  353. /*MEMORY CLEANING*/
  354. /*****************/
  355. #ifdef DEBUG
  356. if ( i > 0)
  357. {
  358. cout << "PIRReplyGeneratorNFL_internal: reply_elt_nbr_OLD: " << old_reply_elt_nbr << endl;
  359. }
  360. #endif
  361. if (i > 0)
  362. {
  363. for (int j = 0 ; j < old_reply_elt_nbr ; j++)
  364. {
  365. free(in_data[j].p[0]);
  366. free(in_data[j].p);
  367. }
  368. delete[] in_data;
  369. }
  370. if (i < pirParam.d - 1) {
  371. old_poly_nbr = currentMaxNbPolys;
  372. in_data = fromResulttoInData(inter_reply, reply_elt_nbr, i);
  373. }
  374. for (uint64_t j = 0 ; j < reply_elt_nbr ; j++) {
  375. for (uint64_t k = 0 ; (k < old_poly_nbr) && (i < pirParam.d - 1); k++){
  376. free(inter_reply[j][k].a);
  377. inter_reply[j][k].a = NULL;
  378. }
  379. delete[] inter_reply[j];
  380. inter_reply[j] = NULL;
  381. }
  382. delete[] inter_reply; // allocated with a 'new' above.
  383. inter_reply = NULL;
  384. }
  385. // Compute execution time
  386. printf( "PIRReplyGeneratorNFL_internal: Global reply generation took %f (omp)seconds\n", omp_get_wtime() - start);
  387. }
  388. double PIRReplyGeneratorNFL_internal::generateReplySimulation(const PIRParameters& pir_params, uint64_t plaintext_nbr)
  389. {
  390. setPirParams((PIRParameters&)pir_params);
  391. pushFakeQuery();
  392. importFakeData(plaintext_nbr);
  393. repliesAmount = computeReplySizeInChunks(plaintext_nbr*cryptoMethod->getPublicParameters().getCiphertextBitsize() / CHAR_BIT);
  394. repliesIndex = 0;
  395. double start = omp_get_wtime();
  396. generateReply();
  397. double result = omp_get_wtime() - start;
  398. freeQueries();
  399. freeInputData();
  400. freeResult();
  401. delete dbhandler;
  402. return result;
  403. }
  404. double PIRReplyGeneratorNFL_internal::precomputationSimulation(const PIRParameters& pir_params, uint64_t plaintext_nbr)
  405. {
  406. NFLlib *nflptr = &(cryptoMethod->getnflInstance());
  407. setPirParams((PIRParameters&)pir_params);
  408. pushFakeQuery();
  409. importFakeData(plaintext_nbr);
  410. uint64_t files_nbr = 1;
  411. for (unsigned int i = 0 ; i < pir_params.d ; i++) files_nbr *= pir_params.n[i];
  412. double start = omp_get_wtime();
  413. for (unsigned int i = 0 ; i < files_nbr ; i++)
  414. {
  415. poly64 *tmp;
  416. tmp = cryptoMethod->deserializeDataNFL((unsigned char**)(input_data[i].p), (uint64_t) plaintext_nbr, cryptoMethod->getPublicParameters().getCiphertextBitsize()/2 , input_data[i].nbPolys);
  417. free(tmp[0]);
  418. tmp = NULL;
  419. }
  420. double result = omp_get_wtime() - start;
  421. std::cout << "PIRReplyGeneratorNFL_internal: Deserialize took " << result << " (omp)seconds" << std::endl;
  422. freeQueries();
  423. freeInputData();
  424. freeResult();
  425. delete dbhandler;
  426. return result;
  427. }
  428. /**
  429. * Multiply each query parts by each files and sum the result.
  430. * Params :
  431. * - lwe_queries* : the query ;
  432. * - lwe_in_data* : data to be processed.
  433. * - int begin_data : index where begins the data absorption
  434. * - int lvl : recursion level ;
  435. * - lwe_cipher* result : Array to store the result.
  436. **/
  437. void PIRReplyGeneratorNFL_internal::generateReply( lwe_query *queries_,
  438. lwe_in_data* data,
  439. unsigned int lvl,
  440. lwe_cipher* result)
  441. {
  442. #ifdef SHOUP
  443. lwe_query **queries=(lwe_query**)queries_;
  444. #else
  445. lwe_query *queries=queries_;
  446. #endif
  447. unsigned int query_size = pirParam.n[lvl];
  448. #ifdef PERF_TIMERS
  449. bool wasVerbose = false;
  450. double vtstart = omp_get_wtime();
  451. #endif
  452. // In order to parallelize we must ensure replies are somehow ordered
  453. // (see comment at the end of PIRReplyExtraction)
  454. //#pragma omp parallel for firstprivate(result,data, lvl, queries)
  455. #ifdef MULTI_THREAD
  456. # pragma omp parallel for
  457. #endif
  458. for (unsigned int current_poly=0 ; current_poly < currentMaxNbPolys ; current_poly++)
  459. {
  460. posix_memalign((void**) &(result[current_poly].a), 32,
  461. 2*cryptoMethod->getpolyDegree()*cryptoMethod->getnbModuli()*sizeof(uint64_t));
  462. memset(result[current_poly].a,0,
  463. 2*cryptoMethod->getpolyDegree()*cryptoMethod->getnbModuli()*sizeof(uint64_t));
  464. result[current_poly].b = (uint64_t *) result[current_poly].a +
  465. cryptoMethod->getpolyDegree()*cryptoMethod->getnbModuli();
  466. for (unsigned int offset = 0; offset < query_size; offset += 200)
  467. {
  468. for (unsigned int query_index = offset, ggg=0; query_index < query_size && ggg < 200 ;
  469. query_index++, ggg++)
  470. {
  471. #ifdef SHOUP
  472. #ifdef CRYPTO_DEBUG
  473. if(current_poly==0)
  474. {
  475. std::cout<<"Query poped.a ";NFLTools::print_poly64hex(queries[0][query_index].a,4);
  476. if (lwe)
  477. {
  478. std::cout<<"Query poped.b ";NFLTools::print_poly64hex(queries[0][query_index].b,4);
  479. }
  480. std::cout<<"Query poped.a' ";NFLTools::print_poly64hex(queries[1][query_index].a,4);
  481. if (lwe)
  482. {
  483. std::cout<<"Query poped.b' ";NFLTools::print_poly64hex(queries[1][query_index].b,4);
  484. }
  485. }
  486. #endif
  487. cryptoMethod->mulandadd(result[current_poly], data[query_index], queries[0][query_index],
  488. queries[1][query_index], current_poly, lvl);
  489. #else
  490. cryptoMethod->mulandadd(result[current_poly], data[query_index], queries[query_index],
  491. current_poly, lvl);
  492. #endif
  493. }
  494. if ( lvl == pirParam.d-1 && offset + 200 >= query_size)
  495. {
  496. // Watchout lwe_cipher.a and .b need to be allocated contiguously
  497. repliesArray[repliesIndex+current_poly] = (char*)result[current_poly].a;
  498. }
  499. #ifdef PERF_TIMERS
  500. // Give some feedback if it takes too long
  501. double vtstop = omp_get_wtime();
  502. if (vtstop - vtstart > 1)
  503. {
  504. vtstart = vtstop;
  505. if(currentMaxNbPolys != 1) std::cout <<"PIRReplyGeneratorNFL_internal: Dealt with chunk " <<
  506. current_poly+1 << "/" << currentMaxNbPolys << "\r" << std::flush;
  507. wasVerbose = true;
  508. }
  509. #endif
  510. }
  511. }
  512. #ifdef PERF_TIMERS
  513. if (wasVerbose) std::cout <<" \r" << std::flush;
  514. #endif
  515. }
  516. // New version using the multiple buffer serialize function
  517. lwe_in_data* PIRReplyGeneratorNFL_internal::fromResulttoInData(lwe_cipher** inter_reply, uint64_t reply_elt_nbr, unsigned int reply_rec_lvl)
  518. {
  519. uint64_t in_data2b_bytes = cryptoMethod->getPublicParameters().getAbsorptionBitsize()/8;
  520. uint64_t in_data2b_nbr_polys = ceil((double(currentMaxNbPolys * cryptoMethod->getPublicParameters().getCiphertextBitsize())/8.)/double(in_data2b_bytes));
  521. lwe_in_data *in_data2b = new lwe_in_data[reply_elt_nbr]();
  522. uint64_t **bufferOfBuffers = (uint64_t **) calloc(currentMaxNbPolys,sizeof(uint64_t*));
  523. //For each element in the reply
  524. for (uint64_t i = 0 ; i < reply_elt_nbr ; i++)
  525. {
  526. //Build the buffer of buffers
  527. for (uint64_t j = 0 ; j < currentMaxNbPolys ; j++)
  528. {
  529. bufferOfBuffers[j]=inter_reply[i][j].a;
  530. }
  531. // Ciphertexts can be serialized in a single block as a,b are allocatted contiguously
  532. in_data2b[i].p = cryptoMethod->deserializeDataNFL((unsigned char**)bufferOfBuffers,
  533. currentMaxNbPolys,
  534. cryptoMethod->getPublicParameters().getCiphertextBitsize(),
  535. in_data2b[i].nbPolys);
  536. }
  537. free(bufferOfBuffers);
  538. currentMaxNbPolys = in_data2b_nbr_polys;
  539. return in_data2b;
  540. }
  541. //// Original function
  542. //lwe_in_data* PIRReplyGeneratorNFL_internal::fromResulttoInData(lwe_cipher** inter_reply, uint64_t reply_elt_nbr, unsigned int reply_rec_lvl)
  543. //{
  544. // uint64_t in_data2b_bytes = cryptoMethod->getPublicParameters().getAbsorptionBitsize()/8;
  545. // uint64_t in_data2b_polys_per_reply_poly = ceil((double)(cryptoMethod->getPublicParameters().getCiphertextBitsize()/8)/in_data2b_bytes);
  546. // uint64_t in_data2b_nbr_polys = currentMaxNbPolys * in_data2b_polys_per_reply_poly;
  547. //
  548. // lwe_in_data *in_data2b = new lwe_in_data[reply_elt_nbr]();
  549. // lwe_in_data tmp_in_data;
  550. //
  551. // //For each element in the reply
  552. // for (uint64_t i = 0 ; i < reply_elt_nbr ; i++)
  553. // {
  554. // in_data2b[i].p = (poly64 *) malloc(in_data2b_nbr_polys*sizeof(poly64));
  555. // in_data2b[i].nbPolys = 0;
  556. //
  557. // // For each polynomial in a reply element
  558. // for (uint64_t j = 0 ; j < currentMaxNbPolys ; j++)
  559. // {
  560. // // Ciphertexts can be serialized in a single block as a,b are allocatted contiguously
  561. // tmp_in_data.p = cryptoMethod->deserializeDataNFL((unsigned char*)inter_reply[i][j].a, cryptoMethod->getPublicParameters().getCiphertextBitsize(), tmp_in_data.nbPolys);
  562. // for (uint64_t k = 0 ; k < in_data2b_polys_per_reply_poly; k++)
  563. // {
  564. // in_data2b[i].p[k + j * in_data2b_polys_per_reply_poly] = tmp_in_data.p[k];
  565. // }
  566. // in_data2b[i].nbPolys += in_data2b_polys_per_reply_poly;
  567. // }
  568. // delete[] inter_reply[i];
  569. // }
  570. // delete[] inter_reply;
  571. //
  572. // currentMaxNbPolys = in_data2b_nbr_polys;
  573. // return in_data2b;
  574. //}
  575. /**
  576. * Compute Reply Size une chunks.
  577. * WARNING blocking function.
  578. **/
  579. unsigned long PIRReplyGeneratorNFL_internal::computeReplySizeInChunks(unsigned long int maxFileBytesize)
  580. {
  581. using namespace GlobalConstant;
  582. unsigned int out = ceil((double)maxFileBytesize*kBitsPerByte*pirParam.alpha/cryptoMethod->getPublicParameters().getAbsorptionBitsize(0));
  583. for (unsigned int i = 1; i < pirParam.d; i++) {
  584. out = ceil(out * double(cryptoMethod->getPublicParameters().getCiphBitsizeFromRecLvl(i)/kBitsPerByte) / double(cryptoMethod->getPublicParameters().getAbsorptionBitsize(i) / kBitsPerByte));
  585. }
  586. return out;
  587. }
  588. /**
  589. * Overloaded fonction from GenericPIRReplyGenerator.
  590. * Initalise queriesBuf.
  591. **/
  592. void PIRReplyGeneratorNFL_internal::initQueriesBuffer() {
  593. const unsigned int nbQueriesBuf=pirParam.d;;
  594. #ifdef SHOUP
  595. queriesBuf = new lwe_query**[nbQueriesBuf]();
  596. for (unsigned int i = 0 ; i < nbQueriesBuf ; i++)
  597. {
  598. queriesBuf[i] = new lwe_query*[2];
  599. queriesBuf[i][0] = new lwe_query[pirParam.n[i]]();
  600. queriesBuf[i][1] = new lwe_query[pirParam.n[i]]();
  601. }
  602. #else
  603. queriesBuf = new lwe_query*[nbQueriesBuf]();
  604. for (unsigned int i = 0 ; i < nbQueriesBuf ; i++)
  605. {
  606. queriesBuf[i] = new lwe_query[pirParam.n[i]]();
  607. }
  608. #endif
  609. #ifdef DEBUG
  610. std::cout<<"Created a queriesBuf for "<<nbQueriesBuf<<" queries"<<std::endl;
  611. #endif
  612. }
  613. void PIRReplyGeneratorNFL_internal::pushFakeQuery()
  614. {
  615. char* query_element;
  616. for (unsigned int dim = 0 ; dim < pirParam.d ; dim++) {
  617. for(unsigned int j = 0 ; j < pirParam.n[dim] ; j++) {
  618. query_element = cryptoMethod->encrypt(0, 1);
  619. pushQuery(query_element, cryptoMethod->getPublicParameters().getCiphertextBitsize()/8, dim, j);
  620. }
  621. }
  622. }
  623. void PIRReplyGeneratorNFL_internal::pushQuery(char* rawQuery)
  624. {
  625. pushQuery(rawQuery, cryptoMethod->getPublicParameters().getCiphertextBitsize()/8, current_dim_index, current_query_index);
  626. current_query_index++;
  627. if (current_query_index >= pirParam.n[current_dim_index])
  628. {
  629. current_query_index = 0;
  630. current_dim_index++;
  631. }
  632. if (current_dim_index >= pirParam.d)
  633. {
  634. std::cout << "PIRReplyGeneratorNFL: Finished importing query (this message should appear only once)" << std::endl;
  635. }
  636. }
  637. void PIRReplyGeneratorNFL_internal::pushQuery(char* rawQuery, unsigned int size, int dim, int nbr)
  638. {
  639. unsigned int polyDegree = cryptoMethod->getpolyDegree();
  640. unsigned int nbModuli = cryptoMethod->getnbModuli();
  641. // Trick, we get both a and b at the same time, b needs to be set afterwards
  642. uint64_t *a,*b;
  643. // We push the query we do not copy it
  644. //a = (poly64) calloc(size, 1);
  645. //memcpy(a,rawQuery,size);
  646. a = (poly64) rawQuery;
  647. if (lwe) b = a+nbModuli*polyDegree;
  648. #ifdef CRYPTO_DEBUG
  649. std::cout<<"\nQuery received.a ";NFLTools::print_poly64(a,4);
  650. if (lwe) {std::cout<<"Query received.b ";NFLTools::print_poly64hex(b,4);}
  651. #endif
  652. #ifdef SHOUP
  653. uint64_t *ap,*bp;
  654. ap = (poly64) calloc(size, 1);
  655. if (lwe) bp = ap+nbModuli*polyDegree;
  656. for (unsigned int cm = 0 ; cm < nbModuli ; cm++)
  657. {
  658. for (unsigned i = 0 ; i < polyDegree ;i++)
  659. {
  660. ap[i+cm*polyDegree] = ((uint128_t) a[i+cm*polyDegree] << 64) / cryptoMethod->getmoduli()[cm];
  661. if (lwe) bp[i+cm*polyDegree] = ((uint128_t) b[i+cm*polyDegree] << 64) / cryptoMethod->getmoduli()[cm];
  662. }
  663. }
  664. queriesBuf[dim][0][nbr].a = a;
  665. queriesBuf[dim][0][nbr].b = b;
  666. queriesBuf[dim][1][nbr].a = ap;
  667. queriesBuf[dim][1][nbr].b = bp;
  668. #ifdef CRYPTO_DEBUG
  669. std::cout << "Query NFL pushed.a' "; NFLTools::print_poly64hex(queriesBuf[dim][1][nbr].a,4);
  670. if (lwe) { std::cout << "Query NFL pushed.b' "; NFLTools::print_poly64hex(queriesBuf[dim][1][nbr].b,4);}
  671. #endif
  672. #else
  673. queriesBuf[dim][nbr].a = a;
  674. queriesBuf[dim][nbr].b = b;
  675. #endif
  676. }
  677. size_t PIRReplyGeneratorNFL_internal::getTotalSystemMemory()
  678. {
  679. #ifdef __APPLE__
  680. int m[2];
  681. m[0] = CTL_HW;
  682. m[1] = HW_MEMSIZE;
  683. int64_t size = 0;
  684. size_t len = sizeof( size );
  685. sysctl( m, 2, &size, &len, NULL, 0 );
  686. return (size_t)size;
  687. #else
  688. long pages = /*get_phys_pages();*/sysconf(_SC_PHYS_PAGES);
  689. long page_size = /*getpagesize();*/sysconf(_SC_PAGE_SIZE);
  690. return pages * page_size;
  691. #endif
  692. }
  693. void PIRReplyGeneratorNFL_internal::setPirParams(PIRParameters& param)
  694. {
  695. freeQueries();
  696. freeQueriesBuffer();
  697. pirParam = param;
  698. cryptoMethod->setandgetAbsBitPerCiphertext(pirParam.n[0]);
  699. initQueriesBuffer();
  700. }
  701. void PIRReplyGeneratorNFL_internal::setCryptoMethod(CryptographicSystem* cm)
  702. {
  703. //cryptoMethod = (NFLLWE*) cm;
  704. cryptoMethod = (LatticesBasedCryptosystem*) cm;
  705. lwe = (cryptoMethod->toString() == "LWE") ? true : false;
  706. }
  707. void PIRReplyGeneratorNFL_internal::freeInputData()
  708. {
  709. #ifdef DEBUG
  710. std:cout << "PIRReplyGeneratorNFL_internal: freeing input_data" << std::endl;
  711. #endif
  712. uint64_t theoretical_files_nbr = 1;
  713. for (unsigned int i = 0 ; i < pirParam.d ; i++) theoretical_files_nbr *= pirParam.n[i];
  714. if (input_data != NULL){
  715. for (unsigned int i = 0 ; i < theoretical_files_nbr ; i++){
  716. if (input_data[i].p != NULL){
  717. if (input_data[i].p[0] != NULL){
  718. free(input_data[i].p[0]);
  719. input_data[i].p[0] = NULL;
  720. }
  721. free(input_data[i].p);
  722. input_data[i].p = NULL;
  723. }
  724. }
  725. delete[] input_data;
  726. input_data = NULL;
  727. }
  728. #ifdef DEBUG
  729. printf( "PIRReplyGeneratorNFL_internal: input_data freed\n");
  730. #endif
  731. }
  732. void PIRReplyGeneratorNFL_internal::freeQueries()
  733. {
  734. for (unsigned int i = 0; i < pirParam.d; i++)
  735. {
  736. for (unsigned int j = 0 ; j < pirParam.n[i] ; j++)
  737. {
  738. if (queriesBuf != NULL && queriesBuf[i] != NULL && queriesBuf[i][0][j].a != NULL)
  739. {
  740. free(queriesBuf[i][0][j].a); //only free a because a and b and contingus, see pushQuery
  741. queriesBuf[i][0][j].a = NULL;
  742. }
  743. if (queriesBuf != NULL && queriesBuf[i] != NULL && queriesBuf[i][1][j].a != NULL)
  744. {
  745. free(queriesBuf[i][1][j].a); //only free a because a and b and contingus, see pushQuery
  746. queriesBuf[i][1][j].a = NULL;
  747. }
  748. }
  749. }
  750. current_query_index = 0;
  751. current_dim_index = 0;
  752. #ifdef DEBUG
  753. printf( "queriesBuf freed\n");
  754. #endif
  755. }
  756. void PIRReplyGeneratorNFL_internal::freeQueriesBuffer()
  757. {
  758. if (queriesBuf != NULL){
  759. for (unsigned int i = 0; i < pirParam.d; i++){
  760. if (queriesBuf[i] != NULL){
  761. if (queriesBuf[i][0] != NULL){
  762. delete[] queriesBuf[i][0]; //allocated in intQueriesBuf with new.
  763. queriesBuf[i][0] = NULL;
  764. }
  765. if (queriesBuf[i][1] != NULL){
  766. delete[] queriesBuf[i][1]; //allocated in intQueriesBuf with new.
  767. queriesBuf[i][1] = NULL;
  768. }
  769. delete[] queriesBuf[i]; //allocated in intQueriesBuf with new.
  770. queriesBuf[i] = NULL;
  771. }
  772. }
  773. delete[] queriesBuf; //allocated in intQueriesBuf with new.
  774. queriesBuf = NULL;
  775. }
  776. }
  777. void PIRReplyGeneratorNFL_internal::freeResult()
  778. {
  779. if(repliesArray!=NULL)
  780. {
  781. for(unsigned i=0 ; i < repliesAmount; i++)
  782. {
  783. if(repliesArray[i]!=NULL) free(repliesArray[i]);
  784. repliesArray[i] = NULL;
  785. }
  786. free(repliesArray);
  787. repliesArray=NULL;
  788. }
  789. }
  790. PIRReplyGeneratorNFL_internal::~PIRReplyGeneratorNFL_internal()
  791. {
  792. freeQueries();
  793. freeQueriesBuffer();
  794. freeResult();
  795. mutex.try_lock();
  796. mutex.unlock();
  797. }