poly.rs 22 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739
  1. #[cfg(target_feature = "avx2")]
  2. use std::arch::x86_64::*;
  3. use rand::distributions::Standard;
  4. use rand::Rng;
  5. use std::cell::RefCell;
  6. use std::ops::{Add, Mul, Neg};
  7. use crate::{aligned_memory::*, arith::*, discrete_gaussian::*, ntt::*, params::*, util::*};
  8. const SCRATCH_SPACE: usize = 8192;
  9. thread_local!(static SCRATCH: RefCell<AlignedMemory64> = RefCell::new(AlignedMemory64::new(SCRATCH_SPACE)));
  10. pub trait PolyMatrix<'a> {
  11. fn is_ntt(&self) -> bool;
  12. fn get_rows(&self) -> usize;
  13. fn get_cols(&self) -> usize;
  14. fn get_params(&self) -> &Params;
  15. fn num_words(&self) -> usize;
  16. fn zero(params: &'a Params, rows: usize, cols: usize) -> Self;
  17. fn random(params: &'a Params, rows: usize, cols: usize) -> Self;
  18. fn random_rng<T: Rng>(params: &'a Params, rows: usize, cols: usize, rng: &mut T) -> Self;
  19. fn as_slice(&self) -> &[u64];
  20. fn as_mut_slice(&mut self) -> &mut [u64];
  21. fn zero_out(&mut self) {
  22. for item in self.as_mut_slice() {
  23. *item = 0;
  24. }
  25. }
  26. fn get_poly(&self, row: usize, col: usize) -> &[u64] {
  27. let num_words = self.num_words();
  28. let start = (row * self.get_cols() + col) * num_words;
  29. &self.as_slice()[start..start + num_words]
  30. }
  31. fn get_poly_mut(&mut self, row: usize, col: usize) -> &mut [u64] {
  32. let num_words = self.num_words();
  33. let start = (row * self.get_cols() + col) * num_words;
  34. &mut self.as_mut_slice()[start..start + num_words]
  35. }
  36. fn copy_into(&mut self, p: &Self, target_row: usize, target_col: usize) {
  37. assert!(target_row < self.get_rows());
  38. assert!(target_col < self.get_cols());
  39. assert!(target_row + p.get_rows() <= self.get_rows());
  40. assert!(target_col + p.get_cols() <= self.get_cols());
  41. for r in 0..p.get_rows() {
  42. for c in 0..p.get_cols() {
  43. let pol_src = p.get_poly(r, c);
  44. let pol_dst = self.get_poly_mut(target_row + r, target_col + c);
  45. pol_dst.copy_from_slice(pol_src);
  46. }
  47. }
  48. }
  49. fn submatrix(&self, target_row: usize, target_col: usize, rows: usize, cols: usize) -> Self;
  50. fn pad_top(&self, pad_rows: usize) -> Self;
  51. }
  52. pub struct PolyMatrixRaw<'a> {
  53. pub params: &'a Params,
  54. pub rows: usize,
  55. pub cols: usize,
  56. pub data: AlignedMemory64,
  57. }
  58. pub struct PolyMatrixNTT<'a> {
  59. pub params: &'a Params,
  60. pub rows: usize,
  61. pub cols: usize,
  62. pub data: AlignedMemory64,
  63. }
  64. impl<'a> PolyMatrix<'a> for PolyMatrixRaw<'a> {
  65. fn is_ntt(&self) -> bool {
  66. false
  67. }
  68. fn get_rows(&self) -> usize {
  69. self.rows
  70. }
  71. fn get_cols(&self) -> usize {
  72. self.cols
  73. }
  74. fn get_params(&self) -> &Params {
  75. &self.params
  76. }
  77. fn as_slice(&self) -> &[u64] {
  78. self.data.as_slice()
  79. }
  80. fn as_mut_slice(&mut self) -> &mut [u64] {
  81. self.data.as_mut_slice()
  82. }
  83. fn num_words(&self) -> usize {
  84. self.params.poly_len
  85. }
  86. fn zero(params: &'a Params, rows: usize, cols: usize) -> PolyMatrixRaw<'a> {
  87. let num_coeffs = rows * cols * params.poly_len;
  88. let data = AlignedMemory64::new(num_coeffs);
  89. PolyMatrixRaw {
  90. params,
  91. rows,
  92. cols,
  93. data,
  94. }
  95. }
  96. fn random_rng<T: Rng>(params: &'a Params, rows: usize, cols: usize, rng: &mut T) -> Self {
  97. let mut iter = rng.sample_iter(&Standard);
  98. let mut out = PolyMatrixRaw::zero(params, rows, cols);
  99. for r in 0..rows {
  100. for c in 0..cols {
  101. for i in 0..params.poly_len {
  102. let val: u64 = iter.next().unwrap();
  103. out.get_poly_mut(r, c)[i] = val % params.modulus;
  104. }
  105. }
  106. }
  107. out
  108. }
  109. fn random(params: &'a Params, rows: usize, cols: usize) -> Self {
  110. let mut rng = rand::thread_rng();
  111. Self::random_rng(params, rows, cols, &mut rng)
  112. }
  113. fn pad_top(&self, pad_rows: usize) -> Self {
  114. let mut padded = Self::zero(self.params, self.rows + pad_rows, self.cols);
  115. padded.copy_into(&self, pad_rows, 0);
  116. padded
  117. }
  118. fn submatrix(&self, target_row: usize, target_col: usize, rows: usize, cols: usize) -> Self {
  119. let mut m = Self::zero(self.params, rows, cols);
  120. assert!(target_row < self.rows);
  121. assert!(target_col < self.cols);
  122. assert!(target_row + rows <= self.rows);
  123. assert!(target_col + cols <= self.cols);
  124. for r in 0..rows {
  125. for c in 0..cols {
  126. let pol_src = self.get_poly(target_row + r, target_col + c);
  127. let pol_dst = m.get_poly_mut(r, c);
  128. pol_dst.copy_from_slice(pol_src);
  129. }
  130. }
  131. m
  132. }
  133. }
  134. impl<'a> Clone for PolyMatrixRaw<'a> {
  135. fn clone(&self) -> Self {
  136. let mut data_clone = AlignedMemory64::new(self.data.len());
  137. data_clone.as_mut_slice().copy_from_slice(self.data.as_slice());
  138. PolyMatrixRaw {
  139. params: self.params,
  140. rows: self.rows,
  141. cols: self.cols,
  142. data: data_clone,
  143. }
  144. }
  145. }
  146. impl<'a> PolyMatrixRaw<'a> {
  147. pub fn identity(params: &'a Params, rows: usize, cols: usize) -> PolyMatrixRaw<'a> {
  148. let num_coeffs = rows * cols * params.poly_len;
  149. let mut data = AlignedMemory::new(num_coeffs);
  150. for r in 0..rows {
  151. let c = r;
  152. let idx = r * cols * params.poly_len + c * params.poly_len;
  153. data[idx] = 1;
  154. }
  155. PolyMatrixRaw {
  156. params,
  157. rows,
  158. cols,
  159. data,
  160. }
  161. }
  162. pub fn noise<T: Rng>(
  163. params: &'a Params,
  164. rows: usize,
  165. cols: usize,
  166. dg: &mut DiscreteGaussian<T>,
  167. ) -> Self {
  168. let mut out = PolyMatrixRaw::zero(params, rows, cols);
  169. dg.sample_matrix(&mut out);
  170. out
  171. }
  172. pub fn ntt(&self) -> PolyMatrixNTT<'a> {
  173. to_ntt_alloc(&self)
  174. }
  175. pub fn reduce_mod(&mut self, modulus: u64) {
  176. for r in 0..self.rows {
  177. for c in 0..self.cols {
  178. for z in 0..self.params.poly_len {
  179. self.get_poly_mut(r, c)[z] %= modulus;
  180. }
  181. }
  182. }
  183. }
  184. pub fn apply_func<F: Fn(u64) -> u64>(&mut self, func: F) {
  185. for r in 0..self.rows {
  186. for c in 0..self.cols {
  187. let pol_mut = self.get_poly_mut(r, c);
  188. for el in pol_mut {
  189. *el = func(*el);
  190. }
  191. }
  192. }
  193. }
  194. pub fn to_vec(&self, modulus_bits: usize, num_coeffs: usize) -> Vec<u8> {
  195. let sz_bits = self.rows * self.cols * num_coeffs * modulus_bits;
  196. let sz_bytes = f64::ceil((sz_bits as f64) / 8f64) as usize + 32;
  197. let sz_bytes_roundup_16 = ((sz_bytes + 15) / 16) * 16;
  198. let mut data = vec![0u8; sz_bytes_roundup_16];
  199. let mut bit_offs = 0;
  200. for r in 0..self.rows {
  201. for c in 0..self.cols {
  202. for z in 0..num_coeffs {
  203. write_arbitrary_bits(
  204. data.as_mut_slice(),
  205. self.get_poly(r, c)[z],
  206. bit_offs,
  207. modulus_bits,
  208. );
  209. bit_offs += modulus_bits;
  210. }
  211. // round bit_offs down to nearest byte boundary
  212. bit_offs = (bit_offs / 8) * 8
  213. }
  214. }
  215. data
  216. }
  217. pub fn single_value(params: &'a Params, value: u64) -> PolyMatrixRaw<'a> {
  218. let mut out = Self::zero(params, 1, 1);
  219. out.data[0] = value;
  220. out
  221. }
  222. }
  223. impl<'a> PolyMatrix<'a> for PolyMatrixNTT<'a> {
  224. fn is_ntt(&self) -> bool {
  225. true
  226. }
  227. fn get_rows(&self) -> usize {
  228. self.rows
  229. }
  230. fn get_cols(&self) -> usize {
  231. self.cols
  232. }
  233. fn get_params(&self) -> &Params {
  234. &self.params
  235. }
  236. fn as_slice(&self) -> &[u64] {
  237. self.data.as_slice()
  238. }
  239. fn as_mut_slice(&mut self) -> &mut [u64] {
  240. self.data.as_mut_slice()
  241. }
  242. fn num_words(&self) -> usize {
  243. self.params.poly_len * self.params.crt_count
  244. }
  245. fn zero(params: &'a Params, rows: usize, cols: usize) -> PolyMatrixNTT<'a> {
  246. let num_coeffs = rows * cols * params.poly_len * params.crt_count;
  247. let data = AlignedMemory::new(num_coeffs);
  248. PolyMatrixNTT {
  249. params,
  250. rows,
  251. cols,
  252. data,
  253. }
  254. }
  255. fn random_rng<T: Rng>(params: &'a Params, rows: usize, cols: usize, rng: &mut T) -> Self {
  256. let mut iter = rng.sample_iter(&Standard);
  257. let mut out = PolyMatrixNTT::zero(params, rows, cols);
  258. for r in 0..rows {
  259. for c in 0..cols {
  260. for i in 0..params.crt_count {
  261. for j in 0..params.poly_len {
  262. let idx = calc_index(&[i, j], &[params.crt_count, params.poly_len]);
  263. let val: u64 = iter.next().unwrap();
  264. out.get_poly_mut(r, c)[idx] = val % params.moduli[i];
  265. }
  266. }
  267. }
  268. }
  269. out
  270. }
  271. fn random(params: &'a Params, rows: usize, cols: usize) -> Self {
  272. let mut rng = rand::thread_rng();
  273. Self::random_rng(params, rows, cols, &mut rng)
  274. }
  275. fn pad_top(&self, pad_rows: usize) -> Self {
  276. let mut padded = Self::zero(self.params, self.rows + pad_rows, self.cols);
  277. padded.copy_into(&self, pad_rows, 0);
  278. padded
  279. }
  280. fn submatrix(&self, target_row: usize, target_col: usize, rows: usize, cols: usize) -> Self {
  281. let mut m = Self::zero(self.params, rows, cols);
  282. assert!(target_row < self.rows);
  283. assert!(target_col < self.cols);
  284. assert!(target_row + rows <= self.rows);
  285. assert!(target_col + cols <= self.cols);
  286. for r in 0..rows {
  287. for c in 0..cols {
  288. let pol_src = self.get_poly(target_row + r, target_col + c);
  289. let pol_dst = m.get_poly_mut(r, c);
  290. pol_dst.copy_from_slice(pol_src);
  291. }
  292. }
  293. m
  294. }
  295. }
  296. impl<'a> Clone for PolyMatrixNTT<'a> {
  297. fn clone(&self) -> Self {
  298. let mut data_clone = AlignedMemory64::new(self.data.len());
  299. data_clone.as_mut_slice().copy_from_slice(self.data.as_slice());
  300. PolyMatrixNTT {
  301. params: self.params,
  302. rows: self.rows,
  303. cols: self.cols,
  304. data: data_clone,
  305. }
  306. }
  307. }
  308. impl<'a> PolyMatrixNTT<'a> {
  309. pub fn raw(&self) -> PolyMatrixRaw<'a> {
  310. from_ntt_alloc(&self)
  311. }
  312. }
  313. pub fn multiply_poly(params: &Params, res: &mut [u64], a: &[u64], b: &[u64]) {
  314. for c in 0..params.crt_count {
  315. for i in 0..params.poly_len {
  316. let idx = c * params.poly_len + i;
  317. res[idx] = multiply_modular(params, a[idx], b[idx], c);
  318. }
  319. }
  320. }
  321. pub fn multiply_add_poly(params: &Params, res: &mut [u64], a: &[u64], b: &[u64]) {
  322. for c in 0..params.crt_count {
  323. for i in 0..params.poly_len {
  324. let idx = c * params.poly_len + i;
  325. res[idx] = multiply_add_modular(params, a[idx], b[idx], res[idx], c);
  326. }
  327. }
  328. }
  329. pub fn add_poly(params: &Params, res: &mut [u64], a: &[u64], b: &[u64]) {
  330. for c in 0..params.crt_count {
  331. for i in 0..params.poly_len {
  332. let idx = c * params.poly_len + i;
  333. res[idx] = add_modular(params, a[idx], b[idx], c);
  334. }
  335. }
  336. }
  337. pub fn add_poly_into(params: &Params, res: &mut [u64], a: &[u64]) {
  338. for c in 0..params.crt_count {
  339. for i in 0..params.poly_len {
  340. let idx = c * params.poly_len + i;
  341. res[idx] = add_modular(params, res[idx], a[idx], c);
  342. }
  343. }
  344. }
  345. pub fn invert_poly(params: &Params, res: &mut [u64], a: &[u64]) {
  346. for i in 0..params.poly_len {
  347. res[i] = params.modulus - a[i];
  348. }
  349. }
  350. pub fn automorph_poly(params: &Params, res: &mut [u64], a: &[u64], t: usize) {
  351. let poly_len = params.poly_len;
  352. for i in 0..poly_len {
  353. let num = (i * t) / poly_len;
  354. let rem = (i * t) % poly_len;
  355. if num % 2 == 0 {
  356. res[rem] = a[i];
  357. } else {
  358. res[rem] = params.modulus - a[i];
  359. }
  360. }
  361. }
  362. #[cfg(target_feature = "avx2")]
  363. pub fn multiply_add_poly_avx(params: &Params, res: &mut [u64], a: &[u64], b: &[u64]) {
  364. for c in 0..params.crt_count {
  365. for i in (0..params.poly_len).step_by(4) {
  366. unsafe {
  367. let p_x = &a[c * params.poly_len + i] as *const u64;
  368. let p_y = &b[c * params.poly_len + i] as *const u64;
  369. let p_z = &mut res[c * params.poly_len + i] as *mut u64;
  370. let x = _mm256_load_si256(p_x as *const __m256i);
  371. let y = _mm256_load_si256(p_y as *const __m256i);
  372. let z = _mm256_load_si256(p_z as *const __m256i);
  373. let product = _mm256_mul_epu32(x, y);
  374. let out = _mm256_add_epi64(z, product);
  375. _mm256_store_si256(p_z as *mut __m256i, out);
  376. }
  377. }
  378. }
  379. }
  380. pub fn modular_reduce(params: &Params, res: &mut [u64]) {
  381. for c in 0..params.crt_count {
  382. for i in 0..params.poly_len {
  383. let idx = c * params.poly_len + i;
  384. res[idx] = barrett_coeff_u64(params, res[idx], c);
  385. }
  386. }
  387. }
  388. #[cfg(not(target_feature = "avx2"))]
  389. pub fn multiply(res: &mut PolyMatrixNTT, a: &PolyMatrixNTT, b: &PolyMatrixNTT) {
  390. assert!(res.rows == a.rows);
  391. assert!(res.cols == b.cols);
  392. assert!(a.cols == b.rows);
  393. let params = res.params;
  394. for i in 0..a.rows {
  395. for j in 0..b.cols {
  396. for z in 0..params.poly_len * params.crt_count {
  397. res.get_poly_mut(i, j)[z] = 0;
  398. }
  399. for k in 0..a.cols {
  400. let params = res.params;
  401. let res_poly = res.get_poly_mut(i, j);
  402. let pol1 = a.get_poly(i, k);
  403. let pol2 = b.get_poly(k, j);
  404. multiply_add_poly(params, res_poly, pol1, pol2);
  405. }
  406. }
  407. }
  408. }
  409. #[cfg(target_feature = "avx2")]
  410. pub fn multiply(res: &mut PolyMatrixNTT, a: &PolyMatrixNTT, b: &PolyMatrixNTT) {
  411. assert_eq!(res.rows, a.rows);
  412. assert_eq!(res.cols, b.cols);
  413. assert_eq!(a.cols, b.rows);
  414. let params = res.params;
  415. for i in 0..a.rows {
  416. for j in 0..b.cols {
  417. for z in 0..params.poly_len * params.crt_count {
  418. res.get_poly_mut(i, j)[z] = 0;
  419. }
  420. let res_poly = res.get_poly_mut(i, j);
  421. for k in 0..a.cols {
  422. let pol1 = a.get_poly(i, k);
  423. let pol2 = b.get_poly(k, j);
  424. multiply_add_poly_avx(params, res_poly, pol1, pol2);
  425. }
  426. modular_reduce(params, res_poly);
  427. }
  428. }
  429. }
  430. pub fn add(res: &mut PolyMatrixNTT, a: &PolyMatrixNTT, b: &PolyMatrixNTT) {
  431. assert!(res.rows == a.rows);
  432. assert!(res.cols == a.cols);
  433. assert!(a.rows == b.rows);
  434. assert!(a.cols == b.cols);
  435. let params = res.params;
  436. for i in 0..a.rows {
  437. for j in 0..a.cols {
  438. let res_poly = res.get_poly_mut(i, j);
  439. let pol1 = a.get_poly(i, j);
  440. let pol2 = b.get_poly(i, j);
  441. add_poly(params, res_poly, pol1, pol2);
  442. }
  443. }
  444. }
  445. pub fn add_into(res: &mut PolyMatrixNTT, a: &PolyMatrixNTT) {
  446. assert!(res.rows == a.rows);
  447. assert!(res.cols == a.cols);
  448. let params = res.params;
  449. for i in 0..res.rows {
  450. for j in 0..res.cols {
  451. let res_poly = res.get_poly_mut(i, j);
  452. let pol2 = a.get_poly(i, j);
  453. add_poly_into(params, res_poly, pol2);
  454. }
  455. }
  456. }
  457. pub fn add_into_at(res: &mut PolyMatrixNTT, a: &PolyMatrixNTT, t_row: usize, t_col: usize) {
  458. let params = res.params;
  459. for i in 0..a.rows {
  460. for j in 0..a.cols {
  461. let res_poly = res.get_poly_mut(t_row + i, t_col + j);
  462. let pol2 = a.get_poly(i, j);
  463. add_poly_into(params, res_poly, pol2);
  464. }
  465. }
  466. }
  467. pub fn invert(res: &mut PolyMatrixRaw, a: &PolyMatrixRaw) {
  468. assert!(res.rows == a.rows);
  469. assert!(res.cols == a.cols);
  470. let params = res.params;
  471. for i in 0..a.rows {
  472. for j in 0..a.cols {
  473. let res_poly = res.get_poly_mut(i, j);
  474. let pol1 = a.get_poly(i, j);
  475. invert_poly(params, res_poly, pol1);
  476. }
  477. }
  478. }
  479. pub fn automorph<'a>(res: &mut PolyMatrixRaw<'a>, a: &PolyMatrixRaw<'a>, t: usize) {
  480. assert!(res.rows == a.rows);
  481. assert!(res.cols == a.cols);
  482. let params = res.params;
  483. for i in 0..a.rows {
  484. for j in 0..a.cols {
  485. let res_poly = res.get_poly_mut(i, j);
  486. let pol1 = a.get_poly(i, j);
  487. automorph_poly(params, res_poly, pol1, t);
  488. }
  489. }
  490. }
  491. pub fn automorph_alloc<'a>(a: &PolyMatrixRaw<'a>, t: usize) -> PolyMatrixRaw<'a> {
  492. let mut res = PolyMatrixRaw::zero(a.params, a.rows, a.cols);
  493. automorph(&mut res, a, t);
  494. res
  495. }
  496. pub fn stack<'a>(a: &PolyMatrixRaw<'a>, b: &PolyMatrixRaw<'a>) -> PolyMatrixRaw<'a> {
  497. assert_eq!(a.cols, b.cols);
  498. let mut c = PolyMatrixRaw::zero(a.params, a.rows + b.rows, a.cols);
  499. c.copy_into(a, 0, 0);
  500. c.copy_into(b, a.rows, 0);
  501. c
  502. }
  503. pub fn scalar_multiply(res: &mut PolyMatrixNTT, a: &PolyMatrixNTT, b: &PolyMatrixNTT) {
  504. assert_eq!(a.rows, 1);
  505. assert_eq!(a.cols, 1);
  506. let params = res.params;
  507. let pol2 = a.get_poly(0, 0);
  508. for i in 0..b.rows {
  509. for j in 0..b.cols {
  510. let res_poly = res.get_poly_mut(i, j);
  511. let pol1 = b.get_poly(i, j);
  512. multiply_poly(params, res_poly, pol1, pol2);
  513. }
  514. }
  515. }
  516. pub fn scalar_multiply_alloc<'a>(
  517. a: &PolyMatrixNTT<'a>,
  518. b: &PolyMatrixNTT<'a>,
  519. ) -> PolyMatrixNTT<'a> {
  520. let mut res = PolyMatrixNTT::zero(b.params, b.rows, b.cols);
  521. scalar_multiply(&mut res, a, b);
  522. res
  523. }
  524. pub fn single_poly<'a>(params: &'a Params, val: u64) -> PolyMatrixRaw<'a> {
  525. let mut res = PolyMatrixRaw::zero(params, 1, 1);
  526. res.get_poly_mut(0, 0)[0] = val;
  527. res
  528. }
  529. fn reduce_copy(params: &Params, out: &mut [u64], inp: &[u64]) {
  530. for n in 0..params.crt_count {
  531. for z in 0..params.poly_len {
  532. out[n * params.poly_len + z] = barrett_coeff_u64(params, inp[z], n);
  533. }
  534. }
  535. }
  536. pub fn to_ntt(a: &mut PolyMatrixNTT, b: &PolyMatrixRaw) {
  537. let params = a.params;
  538. for r in 0..a.rows {
  539. for c in 0..a.cols {
  540. let pol_src = b.get_poly(r, c);
  541. let pol_dst = a.get_poly_mut(r, c);
  542. reduce_copy(params, pol_dst, pol_src);
  543. ntt_forward(params, pol_dst);
  544. }
  545. }
  546. }
  547. pub fn to_ntt_no_reduce(a: &mut PolyMatrixNTT, b: &PolyMatrixRaw) {
  548. let params = a.params;
  549. for r in 0..a.rows {
  550. for c in 0..a.cols {
  551. let pol_src = b.get_poly(r, c);
  552. let pol_dst = a.get_poly_mut(r, c);
  553. for n in 0..params.crt_count {
  554. let idx = n * params.poly_len;
  555. pol_dst[idx..idx + params.poly_len].copy_from_slice(pol_src);
  556. }
  557. ntt_forward(params, pol_dst);
  558. }
  559. }
  560. }
  561. pub fn to_ntt_alloc<'a>(b: &PolyMatrixRaw<'a>) -> PolyMatrixNTT<'a> {
  562. let mut a = PolyMatrixNTT::zero(b.params, b.rows, b.cols);
  563. to_ntt(&mut a, b);
  564. a
  565. }
  566. pub fn from_ntt(a: &mut PolyMatrixRaw, b: &PolyMatrixNTT) {
  567. let params = a.params;
  568. SCRATCH.with(|scratch_cell| {
  569. let scratch_vec = &mut *scratch_cell.borrow_mut();
  570. let scratch = scratch_vec.as_mut_slice();
  571. for r in 0..a.rows {
  572. for c in 0..a.cols {
  573. let pol_src = b.get_poly(r, c);
  574. let pol_dst = a.get_poly_mut(r, c);
  575. scratch[0..pol_src.len()].copy_from_slice(pol_src);
  576. ntt_inverse(params, scratch);
  577. for z in 0..params.poly_len {
  578. pol_dst[z] = params.crt_compose(scratch, z);
  579. }
  580. }
  581. }
  582. });
  583. }
  584. pub fn from_ntt_alloc<'a>(b: &PolyMatrixNTT<'a>) -> PolyMatrixRaw<'a> {
  585. let mut a = PolyMatrixRaw::zero(b.params, b.rows, b.cols);
  586. from_ntt(&mut a, b);
  587. a
  588. }
  589. impl<'a, 'b> Neg for &'b PolyMatrixRaw<'a> {
  590. type Output = PolyMatrixRaw<'a>;
  591. fn neg(self) -> Self::Output {
  592. let mut out = PolyMatrixRaw::zero(self.params, self.rows, self.cols);
  593. invert(&mut out, self);
  594. out
  595. }
  596. }
  597. impl<'a, 'b> Mul for &'b PolyMatrixNTT<'a> {
  598. type Output = PolyMatrixNTT<'a>;
  599. fn mul(self, rhs: Self) -> Self::Output {
  600. let mut out = PolyMatrixNTT::zero(self.params, self.rows, rhs.cols);
  601. multiply(&mut out, self, rhs);
  602. out
  603. }
  604. }
  605. impl<'a, 'b> Add for &'b PolyMatrixNTT<'a> {
  606. type Output = PolyMatrixNTT<'a>;
  607. fn add(self, rhs: Self) -> Self::Output {
  608. let mut out = PolyMatrixNTT::zero(self.params, self.rows, self.cols);
  609. add(&mut out, self, rhs);
  610. out
  611. }
  612. }
  613. #[cfg(test)]
  614. mod test {
  615. use super::*;
  616. fn get_params() -> Params {
  617. get_test_params()
  618. }
  619. fn assert_all_zero(a: &[u64]) {
  620. for i in a {
  621. assert_eq!(*i, 0);
  622. }
  623. }
  624. #[test]
  625. fn sets_all_zeros() {
  626. let params = get_params();
  627. let m1 = PolyMatrixNTT::zero(&params, 2, 1);
  628. assert_all_zero(m1.as_slice());
  629. }
  630. #[test]
  631. fn multiply_correctness() {
  632. let params = get_params();
  633. let m1 = PolyMatrixNTT::zero(&params, 2, 1);
  634. let m2 = PolyMatrixNTT::zero(&params, 3, 2);
  635. let m3 = &m2 * &m1;
  636. assert_all_zero(m3.as_slice());
  637. }
  638. #[test]
  639. fn full_multiply_correctness() {
  640. let params = get_params();
  641. let mut m1 = PolyMatrixRaw::zero(&params, 1, 1);
  642. let mut m2 = PolyMatrixRaw::zero(&params, 1, 1);
  643. m1.get_poly_mut(0, 0)[1] = 100;
  644. m2.get_poly_mut(0, 0)[1] = 7;
  645. let m1_ntt = to_ntt_alloc(&m1);
  646. let m2_ntt = to_ntt_alloc(&m2);
  647. let m3_ntt = &m1_ntt * &m2_ntt;
  648. let m3 = from_ntt_alloc(&m3_ntt);
  649. assert_eq!(m3.get_poly(0, 0)[2], 700);
  650. }
  651. #[test]
  652. fn to_vec_correctness() {
  653. let params = get_params();
  654. let mut m1 = PolyMatrixRaw::zero(&params, 1, 1);
  655. for i in 0..params.poly_len {
  656. m1.data[i] = 1;
  657. }
  658. let modulus_bits = 9;
  659. let v = m1.to_vec(modulus_bits, params.poly_len);
  660. for i in 0..v.len() {
  661. println!("{:?}", v[i]);
  662. }
  663. let mut bit_offs = 0;
  664. for i in 0..params.poly_len {
  665. let val = read_arbitrary_bits(v.as_slice(), bit_offs, modulus_bits);
  666. assert_eq!(m1.data[i], val);
  667. bit_offs += modulus_bits;
  668. }
  669. }
  670. }