Add cross build for native io (lakesoul-io#241)

* add cross build github action Signed-off-by: chenxu <chenxu@dmetasoul.com> * enable hdfs on all platform Signed-off-by: chenxu <chenxu@dmetasoul.com> * fix memory address passing on windows Signed-off-by: chenxu <chenxu@dmetasoul.com> * fix hadoop user default behavior Signed-off-by: chenxu <chenxu@dmetasoul.com> --------- Signed-off-by: chenxu <chenxu@dmetasoul.com> Co-authored-by: chenxu <chenxu@dmetasoul.com>
LaSola0 · May 29, 2023 · 2ef557c · 2ef557c
1 parent 27c6c59
commit 2ef557c
Show file tree

Hide file tree

Showing 11 changed files with 375 additions and 182 deletions.
diff --git a/.github/workflows/native-build.yml b/.github/workflows/native-build.yml
@@ -0,0 +1,120 @@
+name: Cross build
+
+on: push
+
+jobs:
+  build-linux-x86_64:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          lfs: true
+      - name: Set up JDK 8
+        uses: actions/setup-java@v3
+        with:
+          java-version: '8'
+          distribution: 'temurin'
+          cache: maven
+      - uses: actions-rs/toolchain@v1
+        with:
+          toolchain: nightly
+      - uses: actions-rs/cargo@v1
+        with:
+          use-cross: true
+          toolchain: nightly
+          command: build
+          args: '--manifest-path native-io/Cargo.toml --target x86_64-unknown-linux-gnu --release --all-features'
+      - uses: actions/upload-artifact@master
+        with:
+          name: lakesoul-nativeio-x86_64-unknown-linux-gnu
+          path: ./native-io/target/x86_64-unknown-linux-gnu/release/liblakesoul_io_c.so
+
+  build-windows-x86_64:
+    runs-on: windows-latest
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          lfs: true
+      - name: Set up JDK 8
+        uses: actions/setup-java@v3
+        with:
+          java-version: '8'
+          distribution: 'temurin'
+          cache: maven
+      - uses: actions-rs/toolchain@v1
+        with:
+          toolchain: nightly
+      - uses: actions-rs/cargo@v1
+        with:
+          toolchain: nightly
+          command: build
+          args: '--manifest-path native-io/Cargo.toml --release --all-features'
+      - uses: actions/upload-artifact@master
+        with:
+          name: lakesoul-nativeio-x86_64-pc-windows-msvc
+          path: ./native-io/target/release/lakesoul_io_c.dll
+
+  build-macos-x86_64:
+    runs-on: macos-latest
+    steps:
+      - name: Install automake
+        run: brew install automake
+      - uses: actions/checkout@v3
+        with:
+          lfs: true
+      - name: Set up JDK 8
+        uses: actions/setup-java@v3
+        with:
+          java-version: '8'
+          distribution: 'temurin'
+          cache: maven
+      - uses: actions-rs/toolchain@v1
+        with:
+          toolchain: nightly
+      - uses: actions-rs/cargo@v1
+        with:
+          toolchain: nightly
+          command: build
+          args: '--manifest-path native-io/Cargo.toml --release --all-features'
+      - uses: actions/upload-artifact@master
+        with:
+          name: lakesoul-nativeio-x86_64-apple-darwin
+          path: ./native-io/target/release/liblakesoul_io_c.dylib
+
+  build-maven-package:
+    runs-on: ubuntu-latest
+    needs: [build-linux-x86_64, build-windows-x86_64, build-macos-x86_64]
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          lfs: true
+      - uses: actions/download-artifact@v3
+        with:
+          name: lakesoul-nativeio-x86_64-unknown-linux-gnu
+          path: ./native-io/target/release/
+      - uses: actions/download-artifact@v3
+        with:
+          name: lakesoul-nativeio-x86_64-apple-darwin
+          path: ./native-io/target/release/
+      - uses: actions/download-artifact@v3
+        with:
+          name: lakesoul-nativeio-x86_64-pc-windows-msvc
+          path: ./native-io/target/release/
+      - name: Set up JDK 8
+        uses: actions/setup-java@v3
+        with:
+          java-version: '8'
+          distribution: 'temurin'
+          cache: maven
+      - name: Build with Maven
+        run: |
+          MAVEN_OPTS="-Xmx4000m" mvn -q -B package --file pom.xml -Pcross-build -DskipTests -Dmaven.test.skip=true
+      - name: Upload Package
+        uses: actions/upload-artifact@v3
+        with:
+          name: maven-package-upload
+          path: |
+            lakesoul-spark/target/lakesoul-spark-*.jar
+            lakesoul-flink/target/lakesoul-flink-*.jar
+          retention-days: 3
+          if-no-files-found: error
diff --git a/lakesoul-flink/pom.xml b/lakesoul-flink/pom.xml
@@ -419,6 +419,12 @@
                                 <exclude>META-INF/maven/com.google.guava/**</exclude>
                             </excludes>
                         </filter>
+                        <filter>
+                            <artifact>org.apache.arrow:arrow-c-data</artifact>
+                            <excludes>
+                                <exclude>org/apache/arrow/c/jni/JniLoader.class</exclude>
+                            </excludes>
+                        </filter>
                     </filters>
                     <relocations>
                         <relocation>

diff --git a/lakesoul-spark/pom.xml b/lakesoul-spark/pom.xml
@@ -358,6 +358,14 @@
                             <include>mysql:mysql-connector-java</include>
                         </includes>
                     </artifactSet>
+                    <filters>
+                        <filter>
+                            <artifact>org.apache.arrow:arrow-c-data</artifact>
+                            <excludes>
+                                <exclude>org/apache/arrow/c/jni/JniLoader.class</exclude>
+                            </excludes>
+                        </filter>
+                    </filters>
                     <relocations>
                         <relocation>
                             <pattern>com.zaxxer.hikari</pattern>

diff --git a/lakesoul-spark/src/main/scala/org/apache/spark/sql/vectorized/NativeIOUtils.scala b/lakesoul-spark/src/main/scala/org/apache/spark/sql/vectorized/NativeIOUtils.scala
@@ -21,7 +21,6 @@ import org.apache.arrow.vector.{ValueVector, VectorSchemaRoot}
 import org.apache.hadoop.fs.Path
 import org.apache.hadoop.fs.s3a.S3AFileSystem
 import org.apache.hadoop.mapreduce.TaskAttemptContext
-import org.apache.spark.util.Utils
 
 import scala.collection.JavaConverters._
 
@@ -58,7 +57,9 @@ object NativeIOUtils{
   }
 
   def getNativeIOOptions(taskAttemptContext: TaskAttemptContext, file: Path): NativeIOOptions = {
-    val user = Utils.getCurrentUserName
+    var user: String = null
+    val userConf = taskAttemptContext.getConfiguration.get("fs.hdfs.user")
+    if (userConf != null) user = userConf
     var defaultFS = taskAttemptContext.getConfiguration.get("fs.defaultFS")
     if (defaultFS == null) defaultFS = taskAttemptContext.getConfiguration.get("fs.default.name")
     val fileSystem = file.getFileSystem(taskAttemptContext.getConfiguration)