Use ordered-float as the Real implementation in batch executors (tikv…

…#4729) Signed-off-by: Breezewish <breezewish@pingcap.com>
jswh · May 27, 2019 · c085a34 · c085a34
1 parent b1a182b
commit c085a34
Show file tree

Hide file tree

Showing 21 changed files with 517 additions and 278 deletions.
diff --git a/Cargo.lock b/Cargo.lock
diff --git a/Cargo.toml b/Cargo.toml
@@ -120,6 +120,7 @@ hyper = { version = "0.12", default-features = false, features = ["runtime"] }
 tokio-threadpool = "0.1.13"
 vlog = "0.1.4"
 twoway = "0.2.0"
+ordered-float = "1.0"
 profiler = { path = "components/profiler" }
 match_template = { path = "components/match_template" }
 cop_datatype = { path = "components/cop_datatype" }

diff --git a/src/coprocessor/codec/batch/lazy_column_vec.rs b/src/coprocessor/codec/batch/lazy_column_vec.rs
@@ -217,6 +217,7 @@ mod tests {
 
     use cop_datatype::EvalType;
 
+    use crate::coprocessor::codec::data_type::Real;
     use crate::coprocessor::codec::datum::{Datum, DatumEncoder};
 
     /// Pushes a raw row. There must be no empty datum.
@@ -329,7 +330,7 @@ mod tests {
                     .unwrap();
                 assert_eq!(col.len(), 2);
                 assert_eq!(col.eval_type(), EvalType::Real);
-                assert_eq!(col.as_real_slice(), &[Some(1.0), None]);
+                assert_eq!(col.as_real_slice(), &[Real::new(1.0).ok(), None]);
             }
             assert!(columns.is_column_decoded(1));
         }
@@ -378,7 +379,7 @@ mod tests {
             assert_eq!(column1.decoded().eval_type(), EvalType::Real);
             assert_eq!(
                 column1.decoded().as_real_slice(),
-                &[Some(1.3), None, Some(7.5)]
+                &[Real::new(1.3).ok(), None, Real::new(7.5).ok()]
             );
         }
 
@@ -409,13 +410,13 @@ mod tests {
             assert_eq!(
                 column1.decoded().as_real_slice(),
                 &[
-                    Some(1.3),
+                    Real::new(1.3).ok(),
                     None,
-                    Some(7.5),
-                    Some(101.51),
+                    Real::new(7.5).ok(),
+                    Real::new(101.51).ok(),
                     None,
-                    Some(1.9),
-                    Some(101.51)
+                    Real::new(1.9).ok(),
+                    Real::new(101.51).ok()
                 ]
             );
         }
@@ -444,7 +445,12 @@ mod tests {
             assert_eq!(column1.decoded().eval_type(), EvalType::Real);
             assert_eq!(
                 column1.decoded().as_real_slice(),
-                &[Some(1.3), Some(7.5), Some(1.9), Some(101.51)]
+                &[
+                    Real::new(1.3).ok(),
+                    Real::new(7.5).ok(),
+                    Real::new(1.9).ok(),
+                    Real::new(101.51).ok()
+                ]
             );
         }
 
@@ -471,7 +477,12 @@ mod tests {
             assert_eq!(column1.decoded().eval_type(), EvalType::Real);
             assert_eq!(
                 column1.decoded().as_real_slice(),
-                &[Some(1.3), Some(7.5), Some(1.9), Some(101.51)]
+                &[
+                    Real::new(1.3).ok(),
+                    Real::new(7.5).ok(),
+                    Real::new(1.9).ok(),
+                    Real::new(101.51).ok()
+                ]
             );
         }
 
@@ -518,7 +529,7 @@ mod tests {
             assert_eq!(column1.decoded().eval_type(), EvalType::Real);
             assert_eq!(
                 column1.decoded().as_real_slice(),
-                &[Some(7.77), None, Some(7.17)]
+                &[Real::new(7.77).ok(), None, Real::new(7.17).ok()]
             );
         }
 
@@ -546,7 +557,7 @@ mod tests {
             assert_eq!(column1.decoded().eval_type(), EvalType::Real);
             assert_eq!(
                 column1.decoded().as_real_slice(),
-                &[Some(7.77), None, Some(7.17)]
+                &[Real::new(7.77).ok(), None, Real::new(7.17).ok()]
             );
         }
 
@@ -568,7 +579,10 @@ mod tests {
             column1.decode(&Tz::utc(), &schema[1]).unwrap();
             assert_eq!(column1.decoded().len(), 2);
             assert_eq!(column1.decoded().eval_type(), EvalType::Real);
-            assert_eq!(column1.decoded().as_real_slice(), &[Some(7.77), Some(7.17)]);
+            assert_eq!(
+                column1.decoded().as_real_slice(),
+                &[Real::new(7.77).ok(), Real::new(7.17).ok()]
+            );
         }
 
         columns.retain_rows_by_index(|_| false);

diff --git a/src/coprocessor/codec/data_type/mod.rs b/src/coprocessor/codec/data_type/mod.rs
@@ -9,7 +9,7 @@ mod vector_like;
 
 // Concrete eval types without a nullable wrapper.
 pub type Int = i64;
-pub type Real = f64;
+pub type Real = ordered_float::NotNan<f64>;
 pub type Bytes = Vec<u8>;
 pub use crate::coprocessor::codec::mysql::{Decimal, Duration, Json, Time as DateTime};
 

diff --git a/src/coprocessor/codec/data_type/scalar.rs b/src/coprocessor/codec/data_type/scalar.rs
@@ -115,3 +115,17 @@ impl_from! { Bytes }
 impl_from! { DateTime }
 impl_from! { Duration }
 impl_from! { Json }
+
+impl From<Option<f64>> for ScalarValue {
+    #[inline]
+    fn from(s: Option<f64>) -> ScalarValue {
+        ScalarValue::Real(s.and_then(|f| Real::new(f).ok()))
+    }
+}
+
+impl From<f64> for ScalarValue {
+    #[inline]
+    fn from(s: f64) -> ScalarValue {
+        ScalarValue::Real(Real::new(s).ok())
+    }
+}
diff --git a/src/coprocessor/codec/data_type/vector.rs b/src/coprocessor/codec/data_type/vector.rs
@@ -306,7 +306,7 @@ impl VectorValue {
                     if tp == FieldTypeTp::Float {
                         v = (v as f32) as f64;
                     }
-                    vec.push(Some(v));
+                    vec.push(Real::new(v).ok()); // NaN to None
                 }
                 flag => {
                     return Err(Error::InvalidDataType(format!(
@@ -494,7 +494,7 @@ impl VectorValue {
                     }
                     Some(val) => {
                         output.push(datum::FLOAT_FLAG);
-                        output.encode_f64(val)?;
+                        output.encode_f64(val.into_inner())?;
                     }
                 }
                 Ok(())
@@ -719,43 +719,49 @@ mod tests {
         assert_eq!(column.clone().capacity(), column.capacity());
         assert_eq!(column.clone().as_real_slice(), column.as_real_slice());
 
-        column.push_real(Some(1.0));
+        column.push_real(Real::new(1.0).ok());
         assert_eq!(column.len(), 1);
         assert_eq!(column.capacity(), 3);
         assert!(!column.is_empty());
-        assert_eq!(column.as_real_slice(), &[Some(1.0)]);
+        assert_eq!(column.as_real_slice(), &[Real::new(1.0).ok()]);
         assert_eq!(column.clone().capacity(), column.capacity());
         assert_eq!(column.clone().as_real_slice(), column.as_real_slice());
 
         column.push_real(None);
         assert_eq!(column.len(), 2);
         assert_eq!(column.capacity(), 3);
         assert!(!column.is_empty());
-        assert_eq!(column.as_real_slice(), &[Some(1.0), None]);
+        assert_eq!(column.as_real_slice(), &[Real::new(1.0).ok(), None]);
         assert_eq!(column.clone().capacity(), column.capacity());
         assert_eq!(column.clone().as_real_slice(), column.as_real_slice());
 
-        column.push_real(Some(4.5));
+        column.push_real(Real::new(4.5).ok());
         assert_eq!(column.len(), 3);
         assert_eq!(column.capacity(), 3);
         assert!(!column.is_empty());
-        assert_eq!(column.as_real_slice(), &[Some(1.0), None, Some(4.5)]);
+        assert_eq!(
+            column.as_real_slice(),
+            &[Real::new(1.0).ok(), None, Real::new(4.5).ok()]
+        );
         assert_eq!(column.clone().capacity(), column.capacity());
         assert_eq!(column.clone().as_real_slice(), column.as_real_slice());
 
         column.push_real(None);
         assert_eq!(column.len(), 4);
         assert!(column.capacity() > 3);
         assert!(!column.is_empty());
-        assert_eq!(column.as_real_slice(), &[Some(1.0), None, Some(4.5), None]);
+        assert_eq!(
+            column.as_real_slice(),
+            &[Real::new(1.0).ok(), None, Real::new(4.5).ok(), None]
+        );
         assert_eq!(column.clone().capacity(), column.capacity());
         assert_eq!(column.clone().as_real_slice(), column.as_real_slice());
 
         column.truncate(2);
         assert_eq!(column.len(), 2);
         assert!(column.capacity() > 3);
         assert!(!column.is_empty());
-        assert_eq!(column.as_real_slice(), &[Some(1.0), None]);
+        assert_eq!(column.as_real_slice(), &[Real::new(1.0).ok(), None]);
         assert_eq!(column.clone().capacity(), column.capacity());
         assert_eq!(column.clone().as_real_slice(), column.as_real_slice());
 
@@ -785,47 +791,64 @@ mod tests {
         assert_eq!(column.capacity(), 3);
 
         column.push_real(None);
-        column.push_real(Some(2.0));
-        column.push_real(Some(1.0));
+        column.push_real(Real::new(2.0).ok());
+        column.push_real(Real::new(1.0).ok());
         column.push_real(None);
-        column.push_real(Some(5.0));
+        column.push_real(Real::new(5.0).ok());
         column.push_real(None);
 
         let retain_map = &[true, true, false, false, true, false];
         column.retain_by_index(|idx| retain_map[idx]);
 
         assert_eq!(column.len(), 3);
         assert!(column.capacity() > 3);
-        assert_eq!(column.as_real_slice(), &[None, Some(2.0), Some(5.0)]);
+        assert_eq!(
+            column.as_real_slice(),
+            &[None, Real::new(2.0).ok(), Real::new(5.0).ok()]
+        );
 
         column.push_real(None);
-        column.push_real(Some(1.5));
+        column.push_real(Real::new(1.5).ok());
         column.push_real(None);
-        column.push_real(Some(4.0));
+        column.push_real(Real::new(4.0).ok());
 
         assert_eq!(column.len(), 7);
         assert_eq!(
             column.as_real_slice(),
-            &[None, Some(2.0), Some(5.0), None, Some(1.5), None, Some(4.0)]
+            &[
+                None,
+                Real::new(2.0).ok(),
+                Real::new(5.0).ok(),
+                None,
+                Real::new(1.5).ok(),
+                None,
+                Real::new(4.0).ok()
+            ]
         );
 
         let retain_map = &[true, false, true, false, false, true, true];
         column.retain_by_index(|idx| retain_map[idx]);
 
         assert_eq!(column.len(), 4);
-        assert_eq!(column.as_real_slice(), &[None, Some(5.0), None, Some(4.0)]);
+        assert_eq!(
+            column.as_real_slice(),
+            &[None, Real::new(5.0).ok(), None, Real::new(4.0).ok()]
+        );
 
         column.retain_by_index(|_| true);
         assert_eq!(column.len(), 4);
-        assert_eq!(column.as_real_slice(), &[None, Some(5.0), None, Some(4.0)]);
+        assert_eq!(
+            column.as_real_slice(),
+            &[None, Real::new(5.0).ok(), None, Real::new(4.0).ok()]
+        );
 
         column.retain_by_index(|_| false);
         assert_eq!(column.len(), 0);
         assert_eq!(column.as_real_slice(), &[]);
 
         column.push_real(None);
-        column.push_real(Some(1.5));
-        assert_eq!(column.as_real_slice(), &[None, Some(1.5)]);
+        column.push_real(Real::new(1.5).ok());
+        assert_eq!(column.as_real_slice(), &[None, Real::new(1.5).ok()]);
     }
 
     #[test]
@@ -839,28 +862,28 @@ mod tests {
         assert_eq!(column2.len(), 0);
         assert_eq!(column2.capacity(), 3);
 
-        column2.push_real(Some(1.0));
+        column2.push_real(Real::new(1.0).ok());
         column2.append(&mut column1);
         assert_eq!(column1.len(), 0);
         assert_eq!(column1.capacity(), 0);
         assert_eq!(column1.as_real_slice(), &[]);
         assert_eq!(column2.len(), 1);
         assert_eq!(column2.capacity(), 3);
-        assert_eq!(column2.as_real_slice(), &[Some(1.0)]);
+        assert_eq!(column2.as_real_slice(), &[Real::new(1.0).ok()]);
 
         column1.push_real(None);
         column1.push_real(None);
         column1.append(&mut column2);
         assert_eq!(column1.len(), 3);
         assert!(column1.capacity() > 0);
-        assert_eq!(column1.as_real_slice(), &[None, None, Some(1.0)]);
+        assert_eq!(column1.as_real_slice(), &[None, None, Real::new(1.0).ok()]);
         assert_eq!(column2.len(), 0);
         assert_eq!(column2.capacity(), 3);
         assert_eq!(column2.as_real_slice(), &[]);
 
-        column1.push_real(Some(1.1));
-        column2.push_real(Some(3.5));
-        column2.push_real(Some(4.1));
+        column1.push_real(Real::new(1.1).ok());
+        column2.push_real(Real::new(3.5).ok());
+        column2.push_real(Real::new(4.1).ok());
         column2.truncate(1);
         column2.append(&mut column1);
         assert_eq!(column1.len(), 0);
@@ -870,13 +893,19 @@ mod tests {
         assert!(column2.capacity() > 3);
         assert_eq!(
             column2.as_real_slice(),
-            &[Some(3.5), None, None, Some(1.0), Some(1.1)]
+            &[
+                Real::new(3.5).ok(),
+                None,
+                None,
+                Real::new(1.0).ok(),
+                Real::new(1.1).ok()
+            ]
         );
     }
 
     #[test]
     fn test_from() {
-        let slice: &[_] = &[None, Some(1.0)];
+        let slice: &[_] = &[None, Real::new(1.0).ok()];
         let vec = slice.to_vec();
         let column = VectorValue::from(vec);
         assert_eq!(column.len(), 2);