PIRReplyGeneratorNFL_internal.cpp 32 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932
  1. /* Copyright (C) 2014 Carlos Aguilar Melchor, Joris Barrier, Marc-Olivier Killijian
  2. * This file is part of XPIR.
  3. *
  4. * XPIR is free software: you can redistribute it and/or modify
  5. * it under the terms of the GNU General Public License as published by
  6. * the Free Software Foundation, either version 3 of the License, or
  7. * (at your option) any later version.
  8. *
  9. * XPIR is distributed in the hope that it will be useful,
  10. * but WITHOUT ANY WARRANTY; without even the implied warranty of
  11. * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  12. * GNU General Public License for more details.
  13. *
  14. * You should have received a copy of the GNU General Public License
  15. * along with XPIR. If not, see <http://www.gnu.org/licenses/>.
  16. */
  17. #include "PIRReplyGeneratorNFL_internal.hpp"
  18. #include "sys/time.h"
  19. //#define SIMULATE_PRE_NFL_DATA //Use this to simulate imported data is in NFL form
  20. //#define TEST_NFL_PERF_ITERATIONS //Loop to simulate very large databases
  21. //#define SNIFFER_CHUNK_BYTESIZE 12800
  22. //#define SNIFFER_CHUNK_BYTESIZE 2560
  23. //#define SNIFFER_CHUNK_BYTESIZE 1500
  24. //#define SNIFFER_CHUNK_BYTESIZE 600
  25. //#define SNIFFER //Use this to activate a sniffer like behavior
  26. PIRReplyGeneratorNFL_internal::PIRReplyGeneratorNFL_internal():
  27. lwe(false),
  28. currentMaxNbPolys(0),
  29. queriesBuf(NULL),
  30. current_query_index(0),
  31. current_dim_index(0),
  32. input_data(NULL),
  33. cryptoMethod(NULL)
  34. {
  35. }
  36. /**
  37. * Constructor of the class.
  38. * Params :
  39. * - vector <File*>& database : reference of a File pointer vector.
  40. * - PIRParameters& param : reference to a PIRParameters object.
  41. **/
  42. PIRReplyGeneratorNFL_internal::PIRReplyGeneratorNFL_internal( PIRParameters& param, DBHandler* db):
  43. lwe(false),
  44. currentMaxNbPolys(0),
  45. GenericPIRReplyGenerator(param,db),
  46. queriesBuf(NULL),
  47. current_query_index(0),
  48. current_dim_index(0),
  49. input_data(NULL),
  50. cryptoMethod(NULL)
  51. {
  52. // cryptoMethod will be set later by setCryptoMethod
  53. }
  54. void PIRReplyGeneratorNFL_internal::importFakeData(uint64_t plaintext_nbr)
  55. {
  56. uint64_t files_nbr = 1;
  57. for (unsigned int i = 0 ; i < pirParam.d ; i++) files_nbr *= pirParam.n[i];
  58. uint64_t plain_bytesize = cryptoMethod->getnflInstance().getpolyDegree()*cryptoMethod->getnflInstance().getnbModuli()*8;
  59. dbhandler = new DBGenerator(files_nbr, plaintext_nbr*plain_bytesize, true);
  60. currentMaxNbPolys = plaintext_nbr;
  61. importDataNFL(0, plaintext_nbr*plain_bytesize);
  62. }
  63. /**
  64. * Convert raw data from file in usable NFL data.
  65. **/
  66. void PIRReplyGeneratorNFL_internal::importDataNFL(uint64_t offset, uint64_t bytes_per_file)
  67. {
  68. uint64_t fileByteSize = min(bytes_per_file, dbhandler->getmaxFileBytesize()-offset);
  69. uint64_t theoretical_files_nbr = 1;
  70. uint64_t nbFiles = dbhandler->getNbStream();
  71. for (unsigned int i = 0 ; i < pirParam.d ; i++) theoretical_files_nbr *= pirParam.n[i];
  72. input_data = (lwe_in_data *) malloc(sizeof(lwe_in_data)*theoretical_files_nbr);
  73. char *rawBits = (char*)calloc(fileByteSize*pirParam.alpha, sizeof(char));
  74. currentMaxNbPolys=0;
  75. #ifdef PERF_TIMERS
  76. double vtstart = omp_get_wtime();
  77. bool wasVerbose = false;
  78. uint64_t lastindex = 0;
  79. #endif
  80. // For global time measurement
  81. double start = omp_get_wtime();
  82. int nbruns=ceil((double)nbFiles/pirParam.alpha);
  83. // WARNING this section should not be multithreade as rawbits is shared and readAggregatedStream
  84. // is not threadsafe
  85. for (int i=0; i < nbruns; i++)
  86. {
  87. dbhandler->readAggregatedStream(i, pirParam.alpha, offset, bytes_per_file, rawBits);
  88. #ifdef SIMULATE_PRE_NFL_DATA
  89. uint64_t abssize = cryptoMethod->getPublicParameters().getAbsorptionBitsize();
  90. uint64_t polysize = cryptoMethod->getpolyDegree() * cryptoMethod->getnbModuli()*sizeof(uint64_t);
  91. uint64_t nbpolys = ceil((double)fileByteSize * pirParam.alpha * 8 / abssize);
  92. input_data[i].p = (poly64*) malloc(nbpolys*sizeof(poly64*));
  93. input_data[i].p[0] = (poly64) malloc(nbpolys*polysize);
  94. for (unsigned j = 0; j < nbpolys ; j++)
  95. {
  96. input_data[i].p[j] = input_data[i].p[0]+j*polysize/8;
  97. memcpy(input_data[i].p[j], rawBits, min(fileByteSize, polysize));
  98. }
  99. input_data[i].nbPolys = nbpolys;
  100. #else
  101. input_data[i].p = cryptoMethod->deserializeDataNFL((unsigned char**)&rawBits, (uint64_t) 1, fileByteSize*pirParam.alpha*GlobalConstant::kBitsPerByte, input_data[i].nbPolys);
  102. #endif
  103. #ifdef PERF_TIMERS
  104. // Give some feedback if it takes too long
  105. double vtstop = omp_get_wtime();
  106. if (vtstop - vtstart > 1)
  107. {
  108. vtstart = vtstop;
  109. std::cout <<"PIRReplyGeneratorNFL_internal: Element " << i+1 << "/" << nbruns << " imported\r" << std::flush;
  110. wasVerbose = true;
  111. lastindex = i+1;
  112. }
  113. #endif
  114. }
  115. for (int i=0; i < nbruns; i++)
  116. if (input_data[i].nbPolys>currentMaxNbPolys) currentMaxNbPolys=input_data[i].nbPolys;
  117. #ifdef PERF_TIMERS
  118. // If feedback was given say we finished
  119. if (wasVerbose && lastindex != nbFiles) std::cout <<"PIRReplyGeneratorNFL_internal: Element " << nbruns << "/" << nbFiles/pirParam.alpha << " imported" << std::endl;
  120. #endif
  121. /** FILE PADDING **/
  122. for (uint64_t i = ceil((double)nbFiles/pirParam.alpha) ; i < theoretical_files_nbr ; i++)
  123. {
  124. input_data[i].p = (poly64 *) malloc(currentMaxNbPolys*sizeof(poly64));
  125. input_data[i].p[0] = (poly64) calloc(cryptoMethod->getpolyDegree()*cryptoMethod->getnbModuli()*currentMaxNbPolys,sizeof(uint64_t));
  126. for (uint64_t j = 1 ; j < currentMaxNbPolys ; j++) input_data[i].p[j] = input_data[i].p[0]+cryptoMethod->getpolyDegree()*cryptoMethod->getnbModuli()*j;
  127. input_data[i].nbPolys = currentMaxNbPolys;
  128. }
  129. free(rawBits);
  130. std::cout<<"PIRReplyGeneratorNFL_internal: Finished importing the database in " << omp_get_wtime() - start << " seconds" << std::endl;
  131. }
  132. #ifdef SNIFFER
  133. imported_database_t PIRReplyGeneratorNFL_internal::generateReplyGeneric(bool keep_imported_data)
  134. {
  135. imported_database_t database_wrapper;
  136. boost::mutex::scoped_lock l(mutex);
  137. const uint64_t chunkBytesize = SNIFFER_CHUNK_BYTESIZE;
  138. const uint64_t iterations = dbhandler->getmaxFileBytesize()/(chunkBytesize+1);
  139. // std::ifstream *is = dbhandler->openStream(0,0);
  140. const uint64_t nbFiles = dbhandler->getNbStream();
  141. const unsigned int polysize = cryptoMethod->getpolyDegree()*cryptoMethod->getnbModuli();
  142. currentMaxNbPolys=0;
  143. lwe_in_data *input = new lwe_in_data[iterations];
  144. lwe_cipher *resul = new lwe_cipher[iterations];
  145. uint64_t index;
  146. lwe_query **queries;
  147. char **rawBits = (char**) malloc(iterations*sizeof(char*));
  148. cryptoMethod->setandgetAbsBitPerCiphertext(1);
  149. queries = queriesBuf[0];
  150. const uint64_t mask = (1<<((int)log2(nbFiles)))-1;
  151. for (uint64_t it = 0 ; it < iterations ; it++)
  152. {
  153. resul[it].a = (uint64_t *) malloc(2*polysize*sizeof(uint64_t));
  154. resul[it].b = (uint64_t *) resul[it].a + polysize;
  155. rawBits[it] = (char*)malloc(((chunkBytesize)*sizeof(char)+sizeof(int)));
  156. }
  157. double start = omp_get_wtime();
  158. #pragma omp parallel for firstprivate(input,queries,resul)
  159. for (uint64_t it = 0 ; it < iterations ; it++)
  160. {
  161. dbhandler->readStream(0, rawBits[it], chunkBytesize+sizeof(int));
  162. input[it].p = cryptoMethod->deserializeDataNFL((unsigned char**)&(rawBits[it]), (uint64_t) 1, chunkBytesize*GlobalConstant::kBitsPerByte, input[it].nbPolys);
  163. index = *(int *)(rawBits[it]+chunkBytesize) & mask;
  164. cryptoMethod->mul(resul[it], input[it], queries[0][index],queries[1][index], 0, 0);
  165. }
  166. double end = omp_get_wtime();
  167. std::cout<<"PIRReplyGeneratorNFL_internal: Finished processing the sniffed data in " << end - start << " seconds" << std::endl;
  168. std::cout<<"PIRReplyGeneratorNFL_internal: Processing throughput " << (double)chunkBytesize*8*iterations / ((end - start)*1000000000ULL) << " Gbps" << std::endl;
  169. return database_wrapper;
  170. }
  171. #else
  172. imported_database_t PIRReplyGeneratorNFL_internal::generateReplyGeneric(bool keep_imported_data)
  173. {
  174. imported_database_t database_wrapper;
  175. uint64_t usable_memory, database_size, max_memory_per_file, max_readable_size, nbr_of_iterations;
  176. double start, end;
  177. clock_t rg_start, rg_stop;
  178. // Init database_wrapper to NULL values so that we are able to know if it has been initialized
  179. database_wrapper.imported_database_ptr = NULL;
  180. database_wrapper.nbElements = 0;
  181. database_wrapper.polysPerElement = 0;
  182. database_wrapper.beforeImportElementBytesize = 0;
  183. // Don't use more than half of the computer's memory
  184. usable_memory = getTotalSystemMemory()/2;
  185. database_size = dbhandler->getmaxFileBytesize() * dbhandler->getNbStream();
  186. #ifndef TEST_NFL_PERF_ITERATIONS
  187. // This is the maximum amount of data per file we can get in memory
  188. max_memory_per_file = usable_memory / dbhandler->getNbStream();
  189. // Given the expansion factor of importation we get the max we can read per file
  190. max_readable_size = max_memory_per_file / 4 ;
  191. // Reduce it so that we have full absorption in all the ciphertexts sent
  192. max_readable_size = (max_readable_size * GlobalConstant::kBitsPerByte / cryptoMethod->getPublicParameters().getAbsorptionBitsize(0)) * cryptoMethod->getPublicParameters().getAbsorptionBitsize(0)/GlobalConstant::kBitsPerByte;
  193. #else
  194. // For our tests we will need to have databases of an integer amount of gigabits
  195. max_readable_size = 1280000000UL/dbhandler->getNbStream();
  196. #endif
  197. // If we reduced it too much set it at least to a ciphertext
  198. if (max_readable_size == 0) max_readable_size = cryptoMethod->getPublicParameters().getAbsorptionBitsize(0);
  199. // Ensure it is not larger than maxfilebytesize
  200. max_readable_size = min(max_readable_size, dbhandler->getmaxFileBytesize());
  201. // Given readable size we get how many iterations we need
  202. nbr_of_iterations = ceil((double)dbhandler->getmaxFileBytesize()/max_readable_size);
  203. #ifndef TEST_NFL_PERF_ITERATIONS
  204. // If aggregation is used we cannot iterate
  205. if ((pirParam.alpha != 1 || pirParam.d > 1) && nbr_of_iterations > 1)
  206. {
  207. std::cout << "PIRReplyGeneratorNFL_internal: Cannot handle aggregation or dimensions on databases requiring multiple iterations" << std::endl;
  208. std::cout << "PIRReplyGeneratorNFL_internal: Handling the database on a single iteration, this can cause memory issues ..." << std::endl;
  209. nbr_of_iterations = 1;
  210. max_readable_size = dbhandler->getmaxFileBytesize();
  211. }
  212. // If we cannot read the whole database we cannot store it precomputed
  213. if (nbr_of_iterations > 1) keep_imported_data = false;
  214. #endif
  215. // If we need to do more than an iteration say it
  216. if (nbr_of_iterations > 1)
  217. {
  218. std::cout << "PIRReplyGeneratorNFL_internal: Database is considered too large, processing it in "
  219. << nbr_of_iterations << " iterations" << std::endl;
  220. }
  221. start = omp_get_wtime();
  222. rg_start = clock();
  223. // #pragma omp parallel for
  224. for (unsigned iteration = 0; iteration < nbr_of_iterations; iteration++)
  225. {
  226. if (nbr_of_iterations > 1) cout << "PIRReplyGeneratorNFL_internal: Iteration " << iteration << endl;
  227. repliesIndex = computeReplySizeInChunks(iteration*max_readable_size);
  228. // Import a chunk of max_readable_size bytes per file with an adapted offset
  229. importDataNFL(iteration*max_readable_size, max_readable_size);
  230. if(keep_imported_data && iteration == nbr_of_iterations - 1) // && added for Perf test but is no harmful
  231. {
  232. database_wrapper.polysPerElement = currentMaxNbPolys;
  233. }
  234. boost::mutex::scoped_lock l(mutex);
  235. repliesAmount = computeReplySizeInChunks(dbhandler->getmaxFileBytesize());
  236. generateReply();
  237. end = omp_get_wtime();
  238. rg_stop = clock();
  239. if(keep_imported_data && iteration == nbr_of_iterations - 1) // && added for Perf test but is no harmful
  240. {
  241. database_wrapper.imported_database_ptr = (void*)input_data;
  242. database_wrapper.beforeImportElementBytesize = dbhandler->getmaxFileBytesize();
  243. database_wrapper.nbElements = dbhandler->getNbStream();
  244. }
  245. else
  246. {
  247. freeInputData();
  248. }
  249. }
  250. std::cout<<"nbr_of_iterations:"<<nbr_of_iterations<<std::endl;
  251. std::cout<<"measure_reply_size:"<<repliesAmount<<std::endl;
  252. std::cout<<"consensgx : ReplyGeneration time was " << (double) 1000 * double(rg_stop - rg_start) / double(CLOCKS_PER_SEC)<< " ms" <<endl<<std::flush;
  253. std::cout<<"PIRReplyGeneratorNFL_internal: Total process time " << end - start << " seconds" << std::endl;
  254. std::cout<<"PIRReplyGeneratorNFL_internal: DB processing throughput " << 8*database_size/(end - start) << "bps" << std::endl;
  255. std::cout<<"PIRReplyGeneratorNFL_internal: Client cleartext reception throughput " << 8*dbhandler->getmaxFileBytesize()/(end - start) << "bps" << std::endl;
  256. freeQueries();
  257. return database_wrapper;
  258. }
  259. #endif
  260. // Function used to generate a PIR reply if:
  261. // - database is small enough to be kept in memory
  262. // - it has already been imported to it
  263. void PIRReplyGeneratorNFL_internal::generateReplyGenericFromData(const imported_database_t database)
  264. {
  265. #ifndef TEST_NFL_PERF_ITERATIONS
  266. input_data = (lwe_in_data*) database.imported_database_ptr;
  267. currentMaxNbPolys = database.polysPerElement;
  268. boost::mutex::scoped_lock l(mutex);
  269. double start = omp_get_wtime();
  270. clock_t rg_start = clock();
  271. repliesAmount = computeReplySizeInChunks(database.beforeImportElementBytesize);
  272. generateReply();
  273. #else
  274. uint64_t max_readable_size, database_size, nbr_of_iterations;
  275. database_size = database.beforeImportElementBytesize * database.nbElements;
  276. max_readable_size = 1280000000UL/database.nbElements;
  277. // Ensure it is not larger than maxfilebytesize
  278. max_readable_size = min(max_readable_size, database.beforeImportElementBytesize);
  279. // Given readable size we get how many iterations we need
  280. nbr_of_iterations = ceil((double)database.beforeImportElementBytesize/max_readable_size);
  281. std::cout<<"nbr_of_iterations:"<<nbr_of_iterations<<std::endl;
  282. boost::mutex::scoped_lock l(mutex);
  283. double start = omp_get_wtime();
  284. clock_t rg_start = clock();
  285. for (unsigned iteration = 0; iteration < nbr_of_iterations; iteration++)
  286. {
  287. input_data = (lwe_in_data*) database.imported_database_ptr;
  288. currentMaxNbPolys = database.polysPerElement;
  289. repliesAmount = computeReplySizeInChunks(database.beforeImportElementBytesize);
  290. generateReply();
  291. }
  292. freeInputData();
  293. #endif
  294. clock_t rg_stop = clock();
  295. double end = omp_get_wtime();
  296. std::cout<<"measure_reply_size:"<<repliesAmount<<std::endl;
  297. std::cout<<"consensgx : ReplyGeneration time was " << (double) 1000 * double(rg_stop - rg_start) / double(CLOCKS_PER_SEC)<< " ms" <<endl;
  298. std::cout<<"PIRReplyGeneratorNFL_internal: Total process time " << end - start << " seconds" << std::endl << std::flush;
  299. std::cout<<"PIRReplyGeneratorNFL_internal: DB processing throughput " << 8*dbhandler->getmaxFileBytesize()*dbhandler->getNbStream()/(end - start) << "bps" << std::endl;
  300. std::cout<<"PIRReplyGeneratorNFL_internal: Client cleartext reception throughput " << 8*dbhandler->getmaxFileBytesize()/(end - start) << "bps" << std::endl;
  301. freeQueries();
  302. }
  303. /**
  304. * Prepare reply and start absoptions.
  305. **/
  306. void PIRReplyGeneratorNFL_internal::generateReply()
  307. {
  308. lwe_in_data *in_data = input_data;
  309. lwe_cipher **inter_reply;
  310. #ifdef SHOUP
  311. lwe_query **queries;
  312. #else
  313. lwe_query *queries;
  314. #endif
  315. uint64_t old_reply_elt_nbr = 0;
  316. uint64_t reply_elt_nbr = 1;
  317. uint64_t old_poly_nbr = 1;
  318. // Allocate memory for the reply array
  319. if (repliesArray != NULL) freeResult();
  320. repliesArray = (char**)calloc(repliesAmount,sizeof(char*));
  321. // Start global timers
  322. double start = omp_get_wtime();
  323. #ifdef PERF_TIMERS
  324. double vtstart = start;
  325. bool wasVerbose = false;
  326. #endif
  327. for (unsigned int i = 0 ; i < pirParam.d ; i++) // For each recursion level
  328. {
  329. old_reply_elt_nbr = reply_elt_nbr;
  330. reply_elt_nbr = 1;
  331. for (unsigned int j = i + 1 ; j < pirParam.d ; j++ ) reply_elt_nbr *= pirParam.n[j];
  332. #ifdef DEBUG
  333. cout << "PIRReplyGeneratorNFL_internal: currentMaxNbPolys = " << currentMaxNbPolys << endl;
  334. #endif
  335. inter_reply = new lwe_cipher*[reply_elt_nbr]();
  336. queries = queriesBuf[i];
  337. for (uint64_t j = 0 ; j < reply_elt_nbr ; j++) // Boucle de reply_elt_nbr PIR
  338. {
  339. inter_reply[j] = new lwe_cipher[currentMaxNbPolys];
  340. // Warning of the trick in case SHOUP is defined : we cast quesries to a (lwe_query*) and will have to uncast it
  341. generateReply((lwe_query*)queries , in_data + (pirParam.n[i] * j ), i, inter_reply[j]);
  342. #ifdef DEBUG_WITH_FILE_OUTPUT
  343. if (i ==0 && j==1) {
  344. std::ofstream file(std::string("output_level_"+ std::to_string(i)).c_str(), std::ios::out| std::ios::binary);
  345. for (int k = 0 ; k < currentMaxNbPolys ; k++)
  346. {
  347. file.write((char*)inter_reply[j][k].a,1024*2*8);
  348. }
  349. file.close();
  350. }
  351. #endif
  352. #ifdef PERF_TIMERS
  353. // Give some feedback if it takes too long
  354. double vtstop = omp_get_wtime();
  355. if (vtstop - vtstart > 1)
  356. {
  357. vtstart = vtstop;
  358. std::cout <<"PIRReplyGeneratorNFL_internal: Reply " << j+1 << "/" << reply_elt_nbr << " generated\r" << std::flush;
  359. wasVerbose = true;
  360. }
  361. #endif
  362. }
  363. /*****************/
  364. /*MEMORY CLEANING*/
  365. /*****************/
  366. #ifdef DEBUG
  367. if ( i > 0)
  368. {
  369. cout << "PIRReplyGeneratorNFL_internal: reply_elt_nbr_OLD: " << old_reply_elt_nbr << endl;
  370. }
  371. #endif
  372. if (i > 0)
  373. {
  374. for (int j = 0 ; j < old_reply_elt_nbr ; j++)
  375. {
  376. free(in_data[j].p[0]);
  377. free(in_data[j].p);
  378. }
  379. delete[] in_data;
  380. }
  381. if (i < pirParam.d - 1) {
  382. old_poly_nbr = currentMaxNbPolys;
  383. in_data = fromResulttoInData(inter_reply, reply_elt_nbr, i);
  384. }
  385. for (uint64_t j = 0 ; j < reply_elt_nbr ; j++) {
  386. for (uint64_t k = 0 ; (k < old_poly_nbr) && (i < pirParam.d - 1); k++){
  387. free(inter_reply[j][k].a);
  388. inter_reply[j][k].a = NULL;
  389. }
  390. delete[] inter_reply[j];
  391. inter_reply[j] = NULL;
  392. }
  393. delete[] inter_reply; // allocated with a 'new' above.
  394. inter_reply = NULL;
  395. }
  396. // Compute execution time
  397. printf( "PIRReplyGeneratorNFL_internal: Global reply generation took %f (omp)seconds\n", omp_get_wtime() - start);
  398. }
  399. double PIRReplyGeneratorNFL_internal::generateReplySimulation(const PIRParameters& pir_params, uint64_t plaintext_nbr)
  400. {
  401. setPirParams((PIRParameters&)pir_params);
  402. pushFakeQuery();
  403. importFakeData(plaintext_nbr);
  404. repliesAmount = computeReplySizeInChunks(plaintext_nbr*cryptoMethod->getPublicParameters().getCiphertextBitsize() / CHAR_BIT);
  405. repliesIndex = 0;
  406. double start = omp_get_wtime();
  407. generateReply();
  408. double result = omp_get_wtime() - start;
  409. freeQueries();
  410. freeInputData();
  411. freeResult();
  412. delete dbhandler;
  413. return result;
  414. }
  415. double PIRReplyGeneratorNFL_internal::precomputationSimulation(const PIRParameters& pir_params, uint64_t plaintext_nbr)
  416. {
  417. NFLlib *nflptr = &(cryptoMethod->getnflInstance());
  418. setPirParams((PIRParameters&)pir_params);
  419. pushFakeQuery();
  420. importFakeData(plaintext_nbr);
  421. uint64_t files_nbr = 1;
  422. for (unsigned int i = 0 ; i < pir_params.d ; i++) files_nbr *= pir_params.n[i];
  423. double start = omp_get_wtime();
  424. for (unsigned int i = 0 ; i < files_nbr ; i++)
  425. {
  426. poly64 *tmp;
  427. tmp = cryptoMethod->deserializeDataNFL((unsigned char**)(input_data[i].p), (uint64_t) plaintext_nbr, cryptoMethod->getPublicParameters().getCiphertextBitsize()/2 , input_data[i].nbPolys);
  428. free(tmp[0]);
  429. tmp = NULL;
  430. }
  431. double result = omp_get_wtime() - start;
  432. std::cout << "PIRReplyGeneratorNFL_internal: Deserialize took " << result << " (omp)seconds" << std::endl;
  433. freeQueries();
  434. freeInputData();
  435. freeResult();
  436. delete dbhandler;
  437. return result;
  438. }
  439. /**
  440. * Multiply each query parts by each files and sum the result.
  441. * Params :
  442. * - lwe_queries* : the query ;
  443. * - lwe_in_data* : data to be processed.
  444. * - int begin_data : index where begins the data absorption
  445. * - int lvl : recursion level ;
  446. * - lwe_cipher* result : Array to store the result.
  447. **/
  448. void PIRReplyGeneratorNFL_internal::generateReply( lwe_query *queries_,
  449. lwe_in_data* data,
  450. unsigned int lvl,
  451. lwe_cipher* result)
  452. {
  453. #ifdef SHOUP
  454. lwe_query **queries=(lwe_query**)queries_;
  455. #else
  456. lwe_query *queries=queries_;
  457. #endif
  458. unsigned int query_size = pirParam.n[lvl];
  459. #ifdef PERF_TIMERS
  460. bool wasVerbose = false;
  461. double vtstart = omp_get_wtime();
  462. #endif
  463. // In order to parallelize we must ensure replies are somehow ordered
  464. // (see comment at the end of PIRReplyExtraction)
  465. //#pragma omp parallel for firstprivate(result,data, lvl, queries)
  466. #ifdef MULTI_THREAD
  467. # pragma omp parallel for
  468. #endif
  469. for (unsigned int current_poly=0 ; current_poly < currentMaxNbPolys ; current_poly++)
  470. {
  471. posix_memalign((void**) &(result[current_poly].a), 32,
  472. 2*cryptoMethod->getpolyDegree()*cryptoMethod->getnbModuli()*sizeof(uint64_t));
  473. memset(result[current_poly].a,0,
  474. 2*cryptoMethod->getpolyDegree()*cryptoMethod->getnbModuli()*sizeof(uint64_t));
  475. result[current_poly].b = (uint64_t *) result[current_poly].a +
  476. cryptoMethod->getpolyDegree()*cryptoMethod->getnbModuli();
  477. for (unsigned int offset = 0; offset < query_size; offset += 200)
  478. {
  479. for (unsigned int query_index = offset, ggg=0; query_index < query_size && ggg < 200 ;
  480. query_index++, ggg++)
  481. {
  482. #ifdef SHOUP
  483. #ifdef CRYPTO_DEBUG
  484. if(current_poly==0)
  485. {
  486. std::cout<<"Query poped.a ";NFLTools::print_poly64hex(queries[0][query_index].a,4);
  487. if (lwe)
  488. {
  489. std::cout<<"Query poped.b ";NFLTools::print_poly64hex(queries[0][query_index].b,4);
  490. }
  491. std::cout<<"Query poped.a' ";NFLTools::print_poly64hex(queries[1][query_index].a,4);
  492. if (lwe)
  493. {
  494. std::cout<<"Query poped.b' ";NFLTools::print_poly64hex(queries[1][query_index].b,4);
  495. }
  496. }
  497. #endif
  498. cryptoMethod->mulandadd(result[current_poly], data[query_index], queries[0][query_index],
  499. queries[1][query_index], current_poly, lvl);
  500. #else
  501. cryptoMethod->mulandadd(result[current_poly], data[query_index], queries[query_index],
  502. current_poly, lvl);
  503. #endif
  504. }
  505. if ( lvl == pirParam.d-1 && offset + 200 >= query_size)
  506. {
  507. // Watchout lwe_cipher.a and .b need to be allocated contiguously
  508. repliesArray[repliesIndex+current_poly] = (char*)result[current_poly].a;
  509. }
  510. #ifdef PERF_TIMERS
  511. // Give some feedback if it takes too long
  512. double vtstop = omp_get_wtime();
  513. if (vtstop - vtstart > 1)
  514. {
  515. vtstart = vtstop;
  516. if(currentMaxNbPolys != 1) std::cout <<"PIRReplyGeneratorNFL_internal: Dealt with chunk " <<
  517. current_poly+1 << "/" << currentMaxNbPolys << "\r" << std::flush;
  518. wasVerbose = true;
  519. }
  520. #endif
  521. }
  522. }
  523. #ifdef PERF_TIMERS
  524. if (wasVerbose) std::cout <<" \r" << std::flush;
  525. #endif
  526. }
  527. // New version using the multiple buffer serialize function
  528. lwe_in_data* PIRReplyGeneratorNFL_internal::fromResulttoInData(lwe_cipher** inter_reply, uint64_t reply_elt_nbr, unsigned int reply_rec_lvl)
  529. {
  530. uint64_t in_data2b_bytes = cryptoMethod->getPublicParameters().getAbsorptionBitsize()/8;
  531. uint64_t in_data2b_nbr_polys = ceil((double(currentMaxNbPolys * cryptoMethod->getPublicParameters().getCiphertextBitsize())/8.)/double(in_data2b_bytes));
  532. lwe_in_data *in_data2b = new lwe_in_data[reply_elt_nbr]();
  533. uint64_t **bufferOfBuffers = (uint64_t **) calloc(currentMaxNbPolys,sizeof(uint64_t*));
  534. //For each element in the reply
  535. for (uint64_t i = 0 ; i < reply_elt_nbr ; i++)
  536. {
  537. //Build the buffer of buffers
  538. for (uint64_t j = 0 ; j < currentMaxNbPolys ; j++)
  539. {
  540. bufferOfBuffers[j]=inter_reply[i][j].a;
  541. }
  542. // Ciphertexts can be serialized in a single block as a,b are allocatted contiguously
  543. in_data2b[i].p = cryptoMethod->deserializeDataNFL((unsigned char**)bufferOfBuffers,
  544. currentMaxNbPolys,
  545. cryptoMethod->getPublicParameters().getCiphertextBitsize(),
  546. in_data2b[i].nbPolys);
  547. }
  548. free(bufferOfBuffers);
  549. currentMaxNbPolys = in_data2b_nbr_polys;
  550. return in_data2b;
  551. }
  552. //// Original function
  553. //lwe_in_data* PIRReplyGeneratorNFL_internal::fromResulttoInData(lwe_cipher** inter_reply, uint64_t reply_elt_nbr, unsigned int reply_rec_lvl)
  554. //{
  555. // uint64_t in_data2b_bytes = cryptoMethod->getPublicParameters().getAbsorptionBitsize()/8;
  556. // uint64_t in_data2b_polys_per_reply_poly = ceil((double)(cryptoMethod->getPublicParameters().getCiphertextBitsize()/8)/in_data2b_bytes);
  557. // uint64_t in_data2b_nbr_polys = currentMaxNbPolys * in_data2b_polys_per_reply_poly;
  558. //
  559. // lwe_in_data *in_data2b = new lwe_in_data[reply_elt_nbr]();
  560. // lwe_in_data tmp_in_data;
  561. //
  562. // //For each element in the reply
  563. // for (uint64_t i = 0 ; i < reply_elt_nbr ; i++)
  564. // {
  565. // in_data2b[i].p = (poly64 *) malloc(in_data2b_nbr_polys*sizeof(poly64));
  566. // in_data2b[i].nbPolys = 0;
  567. //
  568. // // For each polynomial in a reply element
  569. // for (uint64_t j = 0 ; j < currentMaxNbPolys ; j++)
  570. // {
  571. // // Ciphertexts can be serialized in a single block as a,b are allocatted contiguously
  572. // tmp_in_data.p = cryptoMethod->deserializeDataNFL((unsigned char*)inter_reply[i][j].a, cryptoMethod->getPublicParameters().getCiphertextBitsize(), tmp_in_data.nbPolys);
  573. // for (uint64_t k = 0 ; k < in_data2b_polys_per_reply_poly; k++)
  574. // {
  575. // in_data2b[i].p[k + j * in_data2b_polys_per_reply_poly] = tmp_in_data.p[k];
  576. // }
  577. // in_data2b[i].nbPolys += in_data2b_polys_per_reply_poly;
  578. // }
  579. // delete[] inter_reply[i];
  580. // }
  581. // delete[] inter_reply;
  582. //
  583. // currentMaxNbPolys = in_data2b_nbr_polys;
  584. // return in_data2b;
  585. //}
  586. /**
  587. * Compute Reply Size une chunks.
  588. * WARNING blocking function.
  589. **/
  590. unsigned long PIRReplyGeneratorNFL_internal::computeReplySizeInChunks(unsigned long int maxFileBytesize)
  591. {
  592. using namespace GlobalConstant;
  593. unsigned int out = ceil((double)maxFileBytesize*kBitsPerByte*pirParam.alpha/cryptoMethod->getPublicParameters().getAbsorptionBitsize(0));
  594. for (unsigned int i = 1; i < pirParam.d; i++) {
  595. out = ceil(out * double(cryptoMethod->getPublicParameters().getCiphBitsizeFromRecLvl(i)/kBitsPerByte) / double(cryptoMethod->getPublicParameters().getAbsorptionBitsize(i) / kBitsPerByte));
  596. }
  597. return out;
  598. }
  599. /**
  600. * Overloaded fonction from GenericPIRReplyGenerator.
  601. * Initalise queriesBuf.
  602. **/
  603. void PIRReplyGeneratorNFL_internal::initQueriesBuffer() {
  604. const unsigned int nbQueriesBuf=pirParam.d;;
  605. #ifdef SHOUP
  606. queriesBuf = new lwe_query**[nbQueriesBuf]();
  607. for (unsigned int i = 0 ; i < nbQueriesBuf ; i++)
  608. {
  609. queriesBuf[i] = new lwe_query*[2];
  610. queriesBuf[i][0] = new lwe_query[pirParam.n[i]]();
  611. queriesBuf[i][1] = new lwe_query[pirParam.n[i]]();
  612. }
  613. #else
  614. queriesBuf = new lwe_query*[nbQueriesBuf]();
  615. for (unsigned int i = 0 ; i < nbQueriesBuf ; i++)
  616. {
  617. queriesBuf[i] = new lwe_query[pirParam.n[i]]();
  618. }
  619. #endif
  620. #ifdef DEBUG
  621. std::cout<<"Created a queriesBuf for "<<nbQueriesBuf<<" queries"<<std::endl;
  622. #endif
  623. }
  624. void PIRReplyGeneratorNFL_internal::pushFakeQuery()
  625. {
  626. char* query_element;
  627. for (unsigned int dim = 0 ; dim < pirParam.d ; dim++) {
  628. for(unsigned int j = 0 ; j < pirParam.n[dim] ; j++) {
  629. query_element = cryptoMethod->encrypt(0, 1);
  630. pushQuery(query_element, cryptoMethod->getPublicParameters().getCiphertextBitsize()/8, dim, j);
  631. }
  632. }
  633. }
  634. void PIRReplyGeneratorNFL_internal::pushQuery(char* rawQuery)
  635. {
  636. pushQuery(rawQuery, cryptoMethod->getPublicParameters().getCiphertextBitsize()/8, current_dim_index, current_query_index);
  637. current_query_index++;
  638. if (current_query_index >= pirParam.n[current_dim_index])
  639. {
  640. current_query_index = 0;
  641. current_dim_index++;
  642. }
  643. if (current_dim_index >= pirParam.d)
  644. {
  645. std::cout << "PIRReplyGeneratorNFL: Finished importing query (this message should appear only once)" << std::endl;
  646. }
  647. }
  648. void PIRReplyGeneratorNFL_internal::pushQuery(char* rawQuery, unsigned int size, int dim, int nbr)
  649. {
  650. unsigned int polyDegree = cryptoMethod->getpolyDegree();
  651. unsigned int nbModuli = cryptoMethod->getnbModuli();
  652. // Trick, we get both a and b at the same time, b needs to be set afterwards
  653. uint64_t *a,*b;
  654. // We push the query we do not copy it
  655. //a = (poly64) calloc(size, 1);
  656. //memcpy(a,rawQuery,size);
  657. a = (poly64) rawQuery;
  658. if (lwe) b = a+nbModuli*polyDegree;
  659. #ifdef CRYPTO_DEBUG
  660. std::cout<<"\nQuery received.a ";NFLTools::print_poly64(a,4);
  661. if (lwe) {std::cout<<"Query received.b ";NFLTools::print_poly64hex(b,4);}
  662. #endif
  663. #ifdef SHOUP
  664. uint64_t *ap,*bp;
  665. ap = (poly64) calloc(size, 1);
  666. if (lwe) bp = ap+nbModuli*polyDegree;
  667. for (unsigned int cm = 0 ; cm < nbModuli ; cm++)
  668. {
  669. for (unsigned i = 0 ; i < polyDegree ;i++)
  670. {
  671. ap[i+cm*polyDegree] = ((uint128_t) a[i+cm*polyDegree] << 64) / cryptoMethod->getmoduli()[cm];
  672. if (lwe) bp[i+cm*polyDegree] = ((uint128_t) b[i+cm*polyDegree] << 64) / cryptoMethod->getmoduli()[cm];
  673. }
  674. }
  675. queriesBuf[dim][0][nbr].a = a;
  676. queriesBuf[dim][0][nbr].b = b;
  677. queriesBuf[dim][1][nbr].a = ap;
  678. queriesBuf[dim][1][nbr].b = bp;
  679. #ifdef CRYPTO_DEBUG
  680. std::cout << "Query NFL pushed.a' "; NFLTools::print_poly64hex(queriesBuf[dim][1][nbr].a,4);
  681. if (lwe) { std::cout << "Query NFL pushed.b' "; NFLTools::print_poly64hex(queriesBuf[dim][1][nbr].b,4);}
  682. #endif
  683. #else
  684. queriesBuf[dim][nbr].a = a;
  685. queriesBuf[dim][nbr].b = b;
  686. #endif
  687. }
  688. size_t PIRReplyGeneratorNFL_internal::getTotalSystemMemory()
  689. {
  690. #ifdef __APPLE__
  691. int m[2];
  692. m[0] = CTL_HW;
  693. m[1] = HW_MEMSIZE;
  694. int64_t size = 0;
  695. size_t len = sizeof( size );
  696. sysctl( m, 2, &size, &len, NULL, 0 );
  697. return (size_t)size;
  698. #else
  699. long pages = /*get_phys_pages();*/sysconf(_SC_PHYS_PAGES);
  700. long page_size = /*getpagesize();*/sysconf(_SC_PAGE_SIZE);
  701. return pages * page_size;
  702. #endif
  703. }
  704. void PIRReplyGeneratorNFL_internal::setPirParams(PIRParameters& param)
  705. {
  706. freeQueries();
  707. freeQueriesBuffer();
  708. pirParam = param;
  709. cryptoMethod->setandgetAbsBitPerCiphertext(pirParam.n[0]);
  710. initQueriesBuffer();
  711. }
  712. void PIRReplyGeneratorNFL_internal::setCryptoMethod(CryptographicSystem* cm)
  713. {
  714. //cryptoMethod = (NFLLWE*) cm;
  715. cryptoMethod = (LatticesBasedCryptosystem*) cm;
  716. lwe = (cryptoMethod->toString() == "LWE") ? true : false;
  717. }
  718. void PIRReplyGeneratorNFL_internal::freeInputData()
  719. {
  720. #ifdef DEBUG
  721. std:cout << "PIRReplyGeneratorNFL_internal: freeing input_data" << std::endl;
  722. #endif
  723. uint64_t theoretical_files_nbr = 1;
  724. for (unsigned int i = 0 ; i < pirParam.d ; i++) theoretical_files_nbr *= pirParam.n[i];
  725. if (input_data != NULL){
  726. for (unsigned int i = 0 ; i < theoretical_files_nbr ; i++){
  727. if (input_data[i].p != NULL){
  728. if (input_data[i].p[0] != NULL){
  729. free(input_data[i].p[0]);
  730. input_data[i].p[0] = NULL;
  731. }
  732. free(input_data[i].p);
  733. input_data[i].p = NULL;
  734. }
  735. }
  736. delete[] input_data;
  737. input_data = NULL;
  738. }
  739. #ifdef DEBUG
  740. printf( "PIRReplyGeneratorNFL_internal: input_data freed\n");
  741. #endif
  742. }
  743. void PIRReplyGeneratorNFL_internal::freeQueries()
  744. {
  745. for (unsigned int i = 0; i < pirParam.d; i++)
  746. {
  747. for (unsigned int j = 0 ; j < pirParam.n[i] ; j++)
  748. {
  749. if (queriesBuf != NULL && queriesBuf[i] != NULL && queriesBuf[i][0][j].a != NULL)
  750. {
  751. free(queriesBuf[i][0][j].a); //only free a because a and b and contingus, see pushQuery
  752. queriesBuf[i][0][j].a = NULL;
  753. }
  754. if (queriesBuf != NULL && queriesBuf[i] != NULL && queriesBuf[i][1][j].a != NULL)
  755. {
  756. free(queriesBuf[i][1][j].a); //only free a because a and b and contingus, see pushQuery
  757. queriesBuf[i][1][j].a = NULL;
  758. }
  759. }
  760. }
  761. current_query_index = 0;
  762. current_dim_index = 0;
  763. #ifdef DEBUG
  764. printf( "queriesBuf freed\n");
  765. #endif
  766. }
  767. void PIRReplyGeneratorNFL_internal::freeQueriesBuffer()
  768. {
  769. if (queriesBuf != NULL){
  770. for (unsigned int i = 0; i < pirParam.d; i++){
  771. if (queriesBuf[i] != NULL){
  772. if (queriesBuf[i][0] != NULL){
  773. delete[] queriesBuf[i][0]; //allocated in intQueriesBuf with new.
  774. queriesBuf[i][0] = NULL;
  775. }
  776. if (queriesBuf[i][1] != NULL){
  777. delete[] queriesBuf[i][1]; //allocated in intQueriesBuf with new.
  778. queriesBuf[i][1] = NULL;
  779. }
  780. delete[] queriesBuf[i]; //allocated in intQueriesBuf with new.
  781. queriesBuf[i] = NULL;
  782. }
  783. }
  784. delete[] queriesBuf; //allocated in intQueriesBuf with new.
  785. queriesBuf = NULL;
  786. }
  787. }
  788. void PIRReplyGeneratorNFL_internal::freeResult()
  789. {
  790. if(repliesArray!=NULL)
  791. {
  792. for(unsigned i=0 ; i < repliesAmount; i++)
  793. {
  794. if(repliesArray[i]!=NULL) free(repliesArray[i]);
  795. repliesArray[i] = NULL;
  796. }
  797. free(repliesArray);
  798. repliesArray=NULL;
  799. }
  800. }
  801. PIRReplyGeneratorNFL_internal::~PIRReplyGeneratorNFL_internal()
  802. {
  803. freeQueries();
  804. freeQueriesBuffer();
  805. freeResult();
  806. mutex.try_lock();
  807. mutex.unlock();
  808. }