Compatability changes to work with arb nitro (#7)

* add getter for expanded roots of unity to debug midmatch

change to older version

commit generated lock

update tracing dependency

able to create already padded blobs

add function to commit to polynomial that already has data as evaluations

also add function to commit to kzg proof with polynomial that already has data as evaluation

update num_cpus dep

update tracing-core

update traciing

* add getter for expanded roots of unity to debug midmatch

* change to older version

* commit generated lock

* update tracing dependency

* able to create already padded blobs

* add function to commit to polynomial that already has data as evaluations

* also add function to commit to kzg proof with polynomial that already has data as evaluation

* update num_cpus dep

* update tracing-core

* update traciing

* Update toolchain to get build working

* remove duplicate functions

* update tracing dependency

* also add function to commit to kzg proof with polynomial that already has data as evaluation

* added funtionality needed for arbitrum

* add more tests and error type for polynomial fft

* update false to true in a test

* readd is_padded

* remove get_expanded_roots_of_unity

* readd Cargo.lock to .gitignore

* remove Cargo.lock

* remove now reduntant compute_kzg_proof_with_evaluation_polynomial

* cargo fmt

* update variable name in test

* update variable name elsewhere

* fix failing kzg test

* merge fft and ifft functions, readd test to new test files

* fix clippy

* update kzg test signatures

* run cargo fmt

* move to dev deps

* add enum to polynomial specifying what form it's in, use that to inform what SRS to use when commiting and computing proofs for a polynomial

* fix syntax errors in tests

* fix more test syntax errors

* fix logical error in transform_to_form

* update transform_error test strings

* replace from_padded_bytes with from_padded_bytes_unchecked

* update benchmarks

* add test coverage for blob tests

* readd polynomial tests

---------

Co-authored-by: Ethen Pociask <ethenpo@gmail.com>
Co-authored-by: Teddy Knox <teddyknox@gmail.com>

Cargo.toml

@@ -30,11 +30,13 @@ num-traits = "0.2"
 byteorder = "1.4"
 ark-poly = "0.4.2"
 crossbeam-channel = "0.5"
-num_cpus = "^1.16.0"
+num_cpus = "1.13.0"
 
 [dev-dependencies]
 criterion = "0.5"
 lazy_static = "1.4"
+tracing = { version = "^0.1.34", features = ["log"] }
+tracing-subscriber = "0.3.18"
 
 [[test]]
 name = "kzg"

benches/bench_kzg_commit.rs

@@ -1,6 +1,6 @@
 use criterion::{criterion_group, criterion_main, Criterion};
 use rand::Rng;
-use rust_kzg_bn254::{blob::Blob, kzg::Kzg};
+use rust_kzg_bn254::{blob::Blob, kzg::Kzg, polynomial::PolynomialFormat};
 use std::time::Duration;
 
 fn bench_kzg_commit(c: &mut Criterion) {
@@ -17,7 +17,7 @@ fn bench_kzg_commit(c: &mut Criterion) {
     c.bench_function("bench_kzg_commit_10000", |b| {
         let random_blob: Vec<u8> = (0..10000).map(|_| rng.gen_range(32..=126) as u8).collect();
         let input = Blob::from_bytes_and_pad(&random_blob);
-        let input_poly = input.to_polynomial().unwrap();
+        let input_poly = input.to_polynomial(PolynomialFormat::InCoefficientForm).unwrap();
         kzg.data_setup_custom(1, input.len().try_into().unwrap())
             .unwrap();
         b.iter(|| kzg.commit(&input_poly).unwrap());
@@ -26,7 +26,7 @@ fn bench_kzg_commit(c: &mut Criterion) {
     c.bench_function("bench_kzg_commit_30000", |b| {
         let random_blob: Vec<u8> = (0..30000).map(|_| rng.gen_range(32..=126) as u8).collect();
         let input = Blob::from_bytes_and_pad(&random_blob);
-        let input_poly = input.to_polynomial().unwrap();
+        let input_poly = input.to_polynomial(PolynomialFormat::InCoefficientForm).unwrap();
         kzg.data_setup_custom(1, input.len().try_into().unwrap())
             .unwrap();
         b.iter(|| kzg.commit(&input_poly).unwrap());
@@ -35,7 +35,7 @@ fn bench_kzg_commit(c: &mut Criterion) {
     c.bench_function("bench_kzg_commit_50000", |b| {
         let random_blob: Vec<u8> = (0..50000).map(|_| rng.gen_range(32..=126) as u8).collect();
         let input = Blob::from_bytes_and_pad(&random_blob);
-        let input_poly = input.to_polynomial().unwrap();
+        let input_poly = input.to_polynomial(PolynomialFormat::InCoefficientForm).unwrap();
         kzg.data_setup_custom(1, input.len().try_into().unwrap())
             .unwrap();
         b.iter(|| kzg.commit(&input_poly).unwrap());

benches/bench_kzg_proof.rs

@@ -1,6 +1,6 @@
 use criterion::{criterion_group, criterion_main, Criterion};
 use rand::Rng;
-use rust_kzg_bn254::{blob::Blob, kzg::Kzg};
+use rust_kzg_bn254::{blob::Blob, kzg::Kzg, polynomial::PolynomialFormat};
 use std::time::Duration;
 
 fn bench_kzg_proof(c: &mut Criterion) {
@@ -17,7 +17,7 @@ fn bench_kzg_proof(c: &mut Criterion) {
     c.bench_function("bench_kzg_proof_10000", |b| {
         let random_blob: Vec<u8> = (0..10000).map(|_| rng.gen_range(32..=126) as u8).collect();
         let input = Blob::from_bytes_and_pad(&random_blob);
-        let input_poly = input.to_polynomial().unwrap();
+        let input_poly = input.to_polynomial(PolynomialFormat::InCoefficientForm).unwrap();
         kzg.data_setup_custom(1, input.len().try_into().unwrap())
             .unwrap();
         let index =
@@ -31,7 +31,7 @@ fn bench_kzg_proof(c: &mut Criterion) {
     c.bench_function("bench_kzg_proof_30000", |b| {
         let random_blob: Vec<u8> = (0..30000).map(|_| rng.gen_range(32..=126) as u8).collect();
         let input = Blob::from_bytes_and_pad(&random_blob);
-        let input_poly = input.to_polynomial().unwrap();
+        let input_poly = input.to_polynomial(PolynomialFormat::InCoefficientForm).unwrap();
         kzg.data_setup_custom(1, input.len().try_into().unwrap())
             .unwrap();
         let index =
@@ -45,7 +45,7 @@ fn bench_kzg_proof(c: &mut Criterion) {
     c.bench_function("bench_kzg_proof_50000", |b| {
         let random_blob: Vec<u8> = (0..50000).map(|_| rng.gen_range(32..=126) as u8).collect();
         let input = Blob::from_bytes_and_pad(&random_blob);
-        let input_poly = input.to_polynomial().unwrap();
+        let input_poly = input.to_polynomial(PolynomialFormat::InCoefficientForm).unwrap();
         kzg.data_setup_custom(1, input.len().try_into().unwrap())
             .unwrap();
         let index =

benches/bench_kzg_verify.rs

@@ -1,6 +1,6 @@
 use criterion::{criterion_group, criterion_main, Criterion};
 use rand::Rng;
-use rust_kzg_bn254::{blob::Blob, kzg::Kzg};
+use rust_kzg_bn254::{blob::Blob, kzg::Kzg, polynomial::PolynomialFormat};
 use std::time::Duration;
 
 fn bench_kzg_verify(c: &mut Criterion) {
@@ -17,7 +17,7 @@ fn bench_kzg_verify(c: &mut Criterion) {
     c.bench_function("bench_kzg_verify_10000", |b| {
         let random_blob: Vec<u8> = (0..10000).map(|_| rng.gen_range(32..=126) as u8).collect();
         let input = Blob::from_bytes_and_pad(&random_blob);
-        let input_poly = input.to_polynomial().unwrap();
+        let input_poly = input.to_polynomial(PolynomialFormat::InCoefficientForm).unwrap();
         kzg.data_setup_custom(1, input.len().try_into().unwrap())
             .unwrap();
         let index =
@@ -34,7 +34,7 @@ fn bench_kzg_verify(c: &mut Criterion) {
     c.bench_function("bench_kzg_verify_30000", |b| {
         let random_blob: Vec<u8> = (0..30000).map(|_| rng.gen_range(32..=126) as u8).collect();
         let input = Blob::from_bytes_and_pad(&random_blob);
-        let input_poly = input.to_polynomial().unwrap();
+        let input_poly = input.to_polynomial(PolynomialFormat::InCoefficientForm).unwrap();
         kzg.data_setup_custom(1, input.len().try_into().unwrap())
             .unwrap();
         let index =
@@ -51,7 +51,7 @@ fn bench_kzg_verify(c: &mut Criterion) {
     c.bench_function("bench_kzg_verify_50000", |b| {
         let random_blob: Vec<u8> = (0..50000).map(|_| rng.gen_range(32..=126) as u8).collect();
         let input = Blob::from_bytes_and_pad(&random_blob);
-        let input_poly = input.to_polynomial().unwrap();
+        let input_poly = input.to_polynomial(PolynomialFormat::InCoefficientForm).unwrap();
         kzg.data_setup_custom(1, input.len().try_into().unwrap())
             .unwrap();
         let index =

rust-toolchain

@@ -1,5 +1,5 @@
 [toolchain]
-channel = 'nightly-2024-04-29'
+channel = '1.74'
 profile = 'minimal'
 components = ['clippy', 'rustfmt']
-targets = ["x86_64-unknown-linux-gnu", "x86_64-pc-windows-gnu", "wasm32-unknown-unknown"]
+targets = ["x86_64-unknown-linux-gnu", "x86_64-pc-windows-gnu", "wasm32-unknown-unknown", "aarch64-apple-darwin"]

src/blob.rs

@@ -1,4 +1,8 @@
-use crate::{errors::BlobError, helpers, polynomial::Polynomial};
+use crate::{
+    errors::BlobError,
+    helpers,
+    polynomial::{Polynomial, PolynomialFormat},
+};
 
 /// A blob which is Eigen DA spec aligned.
 #[derive(Clone, Debug, PartialEq, Eq)]
@@ -39,6 +43,22 @@ impl Blob {
         }
     }
 
+    /// Creates a new `Blob` from the provided byte slice and assumes it's
+    /// already padded according to DA specs.
+    /// WARNING: This function does not check if the bytes are modulo bn254
+    /// if the data has 32 byte segments exceeding the modulo of the field
+    /// then the bytes will be modded by the order of the field and the data
+    /// will be transformed incorrectly
+    pub fn from_padded_bytes_unchecked(input: &[u8]) -> Self {
+        let length_after_padding = input.len();
+
+        Blob {
+            blob_data: input.to_vec(),
+            is_padded: true,
+            length_after_padding,
+        }
+    }
+
     /// Returns the blob data
     pub fn get_blob_data(&self) -> Vec<u8> {
         self.blob_data.clone()
@@ -49,6 +69,11 @@ impl Blob {
         self.blob_data.len()
     }
 
+    /// Checks if the blob data is empty.
+    pub fn is_empty(&self) -> bool {
+        self.blob_data.is_empty()
+    }
+
     /// Pads the blob data in-place if it is not already padded.
     pub fn pad_data(&mut self) -> Result<(), BlobError> {
         if self.is_padded {
@@ -66,20 +91,21 @@ impl Blob {
         if !self.is_padded {
             Err(BlobError::NotPaddedError)
         } else {
-            self.blob_data = helpers::remove_empty_byte_from_padded_bytes(&self.blob_data);
+            self.blob_data =
+                helpers::remove_empty_byte_from_padded_bytes_unchecked(&self.blob_data);
             self.is_padded = false;
             self.length_after_padding = 0;
             Ok(())
         }
     }
 
     /// Converts the blob data to a `Polynomial` if the data is padded.
-    pub fn to_polynomial(&self) -> Result<Polynomial, BlobError> {
+    pub fn to_polynomial(&self, form: PolynomialFormat) -> Result<Polynomial, BlobError> {
         if !self.is_padded {
             Err(BlobError::NotPaddedError)
         } else {
             let fr_vec = helpers::to_fr_array(&self.blob_data);
-            let poly = Polynomial::new(&fr_vec, self.length_after_padding)
+            let poly = Polynomial::new(&fr_vec, self.length_after_padding, form)
                 .map_err(|err| BlobError::GenericError(err.to_string()))?;
             Ok(poly)
         }

src/errors.rs

@@ -24,6 +24,8 @@ pub enum PolynomialError {
     SerializationFromStringError,
     CommitError(String),
     GenericError(String),
+    FFTError(String),
+    IncorrectFormError(String),
 }
 
 impl fmt::Display for PolynomialError {
@@ -33,7 +35,11 @@ impl fmt::Display for PolynomialError {
                 write!(f, "couldn't load string to fr vector")
             },
             PolynomialError::CommitError(ref msg) => write!(f, "Commitment error: {}", msg),
+            PolynomialError::FFTError(ref msg) => write!(f, "FFT error: {}", msg),
             PolynomialError::GenericError(ref msg) => write!(f, "generic error: {}", msg),
+            PolynomialError::IncorrectFormError(ref msg) => {
+                write!(f, "Incorrect form error: {}", msg)
+            },
         }
     }
 }

src/helpers.rs

@@ -11,7 +11,7 @@ use crate::{
     traits::ReadPointFromBytes,
 };
 
-pub fn blob_to_polynomial(blob: &Vec<u8>) -> Vec<Fr> {
+pub fn blob_to_polynomial(blob: &[u8]) -> Vec<Fr> {
     to_fr_array(blob)
 }
 
@@ -55,7 +55,7 @@ pub fn convert_by_padding_empty_byte(data: &[u8]) -> Vec<u8> {
     valid_data
 }
 
-pub fn remove_empty_byte_from_padded_bytes(data: &[u8]) -> Vec<u8> {
+pub fn remove_empty_byte_from_padded_bytes_unchecked(data: &[u8]) -> Vec<u8> {
     let data_size = data.len();
     let parse_size = BYTES_PER_FIELD_ELEMENT;
     let data_len = (data_size + parse_size - 1) / parse_size;
@@ -94,15 +94,15 @@ pub fn to_fr_array(data: &[u8]) -> Vec<Fr> {
     let num_ele = get_num_element(data.len(), BYTES_PER_FIELD_ELEMENT);
     let mut eles = vec![Fr::zero(); num_ele]; // Initialize with zero elements
 
-    for i in 0..num_ele {
+    for (i, element) in eles.iter_mut().enumerate().take(num_ele) {
         let start = i * BYTES_PER_FIELD_ELEMENT;
         let end = (i + 1) * BYTES_PER_FIELD_ELEMENT;
         if end > data.len() {
             let mut padded = vec![0u8; BYTES_PER_FIELD_ELEMENT];
             padded[..data.len() - start].copy_from_slice(&data[start..]);
-            eles[i] = set_bytes_canonical(&padded);
+            *element = set_bytes_canonical(&padded);
         } else {
-            eles[i] = set_bytes_canonical(&data[start..end]);
+            *element = set_bytes_canonical(&data[start..end]);
         }
     }
     eles
@@ -113,8 +113,8 @@ pub fn to_byte_array(data_fr: &[Fr], max_data_size: usize) -> Vec<u8> {
     let data_size = cmp::min(n * BYTES_PER_FIELD_ELEMENT, max_data_size);
     let mut data = vec![0u8; data_size];
 
-    for i in 0..n {
-        let v: Vec<u8> = data_fr[i].into_bigint().to_bytes_be();
+    for (i, element) in data_fr.iter().enumerate().take(n) {
+        let v: Vec<u8> = element.into_bigint().to_bytes_be();
 
         let start = i * BYTES_PER_FIELD_ELEMENT;
         let end = (i + 1) * BYTES_PER_FIELD_ELEMENT;
@@ -136,8 +136,8 @@ pub fn is_zeroed(first_byte: u8, buf: Vec<u8>) -> bool {
         return false;
     }
 
-    for i in 0..buf.len() {
-        if buf[i] != 0 {
+    for byte in &buf {
+        if *byte != 0 {
             return false;
         }
     }
@@ -147,13 +147,13 @@ pub fn is_zeroed(first_byte: u8, buf: Vec<u8>) -> bool {
 pub fn str_vec_to_fr_vec(input: Vec<&str>) -> Result<Vec<Fr>, &str> {
     let mut output: Vec<Fr> = Vec::<Fr>::with_capacity(input.len());
 
-    for i in 0..input.len() {
-        if input[i] == "-1" {
+    for element in &input {
+        if *element == "-1" {
             let mut test = Fr::one();
             test.neg_in_place();
             output.push(test);
         } else {
-            let fr_data = Fr::from_str(input[i]).expect("could not load string to Fr");
+            let fr_data = Fr::from_str(element).expect("could not load string to Fr");
             output.push(fr_data);
         }
     }
@@ -232,13 +232,11 @@ pub fn read_g2_point_from_bytes_be(g2_bytes_be: &Vec<u8>) -> Result<G2Affine, &s
 
     let mut y_sqrt = added_result.sqrt().ok_or("no square root found").unwrap();
 
-    let mut lexicographical_check_result = false;
-
-    if y_sqrt.c1.0.is_zero() {
-        lexicographical_check_result = lexicographically_largest(&y_sqrt.c0);
+    let lexicographical_check_result = if y_sqrt.c1.0.is_zero() {
+        lexicographically_largest(&y_sqrt.c0)
     } else {
-        lexicographical_check_result = lexicographically_largest(&y_sqrt.c1);
-    }
+        lexicographically_largest(&y_sqrt.c1)
+    };
 
     if lexicographical_check_result {
         if m_data == m_compressed_smallest {
@@ -305,12 +303,11 @@ where
 {
     receiver
         .iter()
-        .filter_map(
-            |(chunk, position)| match T::read_point_from_bytes_be(&chunk) {
-                Ok(point) => Some((point, position)),
-                Err(err) => panic!("{}", err),
-            },
-        )
+        .map(|(chunk, position)| {
+            let point =
+                T::read_point_from_bytes_be(&chunk).expect("Failed to read point from bytes");
+            (point, position)
+        })
         .collect()
 }