Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

DX-11176: Allow transferring between VarChar and VarBinary vectors #6

Open
wants to merge 13 commits into
base: dremio
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
ARROW-1864: [Java] Upgrade Netty to 4.1.17
Upgrade Netty to 4.1.17 since the Netty community will deprecate 4.0.x soon. This PR includes the following changes:
- Bump Netty version.
- Implement new ByteBuf APIs added in Netty 4.1.x: a bunch of get/setXXXLE methods. They are the opposite of get/setXXX method regarding byte order. E.g., as ArrowBuf is little endian, `setInt` will put an `int` to the buffer in little endian byte order, while `setIntLE` will put `int` in big byte endian order. The method naming seems confusing anyway, and I opened a Netty issue: netty/netty#7465. The user can call these new methods to get or set multi-byte integers in big endian byte order.
- Make ArrowByteBufAllocator overwrite AbstractByteBufAllocator.

Author: Shixiong Zhu <zsxwing@gmail.com>

Closes apache#1376 from zsxwing/ARROW-1864 and squashes the following commits:

96a93e1 [Shixiong Zhu] extend AbstractByteBufAllocator; add javadoc for new methods
bb97333 [Shixiong Zhu] Add comment for calculateNewCapacity
555f88a [Shixiong Zhu] Add methods back
5e09cca [Shixiong Zhu] Upgrade Netty to 4.1.x
  • Loading branch information
zsxwing authored and siddharthteotia committed Dec 29, 2017
commit 707af38787c24ef2403ae2c3f7f6aa5422bdf1b2
199 changes: 191 additions & 8 deletions java/memory/src/main/java/io/netty/buffer/ArrowBuf.java
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@
import java.io.OutputStream;
import java.nio.ByteBuffer;
import java.nio.ByteOrder;
import java.nio.channels.FileChannel;
import java.nio.channels.GatheringByteChannel;
import java.nio.channels.ScatteringByteChannel;
import java.nio.charset.Charset;
Expand Down Expand Up @@ -493,6 +494,16 @@ public ArrowBuf retain() {
return retain(1);
}

@Override
public ByteBuf touch() {
return this;
}

@Override
public ByteBuf touch(Object hint) {
return this;
}

@Override
public long getLong(int index) {
chk(index, 8);
Expand All @@ -505,6 +516,17 @@ public float getFloat(int index) {
return Float.intBitsToFloat(getInt(index));
}

/**
* Gets a 64-bit long integer at the specified absolute {@code index} in
* this buffer in Big Endian Byte Order.
*/
@Override
public long getLongLE(int index) {
chk(index, 8);
final long v = PlatformDependent.getLong(addr(index));
return Long.reverseBytes(v);
}

@Override
public double getDouble(int index) {
return Double.longBitsToDouble(getLong(index));
Expand All @@ -527,6 +549,17 @@ public int getInt(int index) {
return v;
}

/**
* Gets a 32-bit integer at the specified absolute {@code index} in
* this buffer in Big Endian Byte Order.
*/
@Override
public int getIntLE(int index) {
chk(index, 4);
final int v = PlatformDependent.getInt(addr(index));
return Integer.reverseBytes(v);
}

@Override
public int getUnsignedShort(int index) {
return getShort(index) & 0xFFFF;
Expand All @@ -535,31 +568,125 @@ public int getUnsignedShort(int index) {
@Override
public short getShort(int index) {
chk(index, 2);
short v = PlatformDependent.getShort(addr(index));
final short v = PlatformDependent.getShort(addr(index));
return v;
}

/**
* Gets a 16-bit short integer at the specified absolute {@code index} in
* this buffer in Big Endian Byte Order.
*/
@Override
public short getShortLE(int index) {
final short v = PlatformDependent.getShort(addr(index));
return Short.reverseBytes(v);
}

/**
* Gets an unsigned 24-bit medium integer at the specified absolute
* {@code index} in this buffer.
*/
@Override
public int getUnsignedMedium(int index) {
chk(index, 3);
final long addr = addr(index);
return (PlatformDependent.getByte(addr) & 0xff) << 16 |
(PlatformDependent.getShort(addr + 1) & 0xffff);
}

/**
* Gets an unsigned 24-bit medium integer at the specified absolute {@code index} in
* this buffer in Big Endian Byte Order.
*/
@Override
public int getUnsignedMediumLE(int index) {
chk(index, 3);
final long addr = addr(index);
return (PlatformDependent.getByte(addr) & 0xff) |
(Short.reverseBytes(PlatformDependent.getShort(addr + 1)) & 0xffff) << 8;
}

@Override
public ArrowBuf setShort(int index, int value) {
chk(index, 2);
PlatformDependent.putShort(addr(index), (short) value);
return this;
}

/**
* Sets the specified 16-bit short integer at the specified absolute {@code index}
* in this buffer with Big Endian byte order.
*/
@Override
public ByteBuf setShortLE(int index, int value) {
chk(index, 2);
PlatformDependent.putShort(addr(index), Short.reverseBytes((short) value));
return this;
}

/**
* Sets the specified 24-bit medium integer at the specified absolute
* {@code index} in this buffer.
*/
@Override
public ByteBuf setMedium(int index, int value) {
chk(index, 3);
final long addr = addr(index);
PlatformDependent.putByte(addr, (byte) (value >>> 16));
PlatformDependent.putShort(addr + 1, (short) value);
return this;
}


/**
* Sets the specified 24-bit medium integer at the specified absolute {@code index}
* in this buffer with Big Endian byte order.
*/
@Override
public ByteBuf setMediumLE(int index, int value) {
chk(index, 3);
final long addr = addr(index);
PlatformDependent.putByte(addr, (byte) value);
PlatformDependent.putShort(addr + 1, Short.reverseBytes((short) (value >>> 8)));
return this;
}

@Override
public ArrowBuf setInt(int index, int value) {
chk(index, 4);
PlatformDependent.putInt(addr(index), value);
return this;
}

/**
* Sets the specified 32-bit integer at the specified absolute {@code index}
* in this buffer with Big Endian byte order.
*/
@Override
public ByteBuf setIntLE(int index, int value) {
chk(index, 4);
PlatformDependent.putInt(addr(index), Integer.reverseBytes(value));
return this;
}

@Override
public ArrowBuf setLong(int index, long value) {
chk(index, 8);
PlatformDependent.putLong(addr(index), value);
return this;
}

/**
* Sets the specified 64-bit long integer at the specified absolute {@code index}
* in this buffer with Big Endian byte order.
*/
@Override
public ByteBuf setLongLE(int index, long value) {
chk(index, 8);
PlatformDependent.putLong(addr(index), Long.reverseBytes(value));
return this;
}

@Override
public ArrowBuf setChar(int index, int value) {
chk(index, 2);
Expand Down Expand Up @@ -668,16 +795,46 @@ protected short _getShort(int index) {
return getShort(index);
}

/** @see {@link #getShortLE(int)} */
@Override
protected short _getShortLE(int index) {
return getShortLE(index);
}

@Override
protected int _getInt(int index) {
return getInt(index);
}

/** @see {@link #getIntLE(int)} */
@Override
protected int _getIntLE(int index) {
return getIntLE(index);
}

/** @see {@link #getUnsignedMedium(int)} */
@Override
protected int _getUnsignedMedium(int index) {
return getUnsignedMedium(index);
}

/** @see {@link #getUnsignedMediumLE(int)} */
@Override
protected int _getUnsignedMediumLE(int index) {
return getUnsignedMediumLE(index);
}

@Override
protected long _getLong(int index) {
return getLong(index);
}

/** @see {@link #getLongLE(int)} */
@Override
protected long _getLongLE(int index) {
return getLongLE(index);
}

@Override
protected void _setByte(int index, int value) {
setByte(index, value);
Expand All @@ -688,21 +845,45 @@ protected void _setShort(int index, int value) {
setShort(index, value);
}

/** @see {@link #setShortLE(int, int)} */
@Override
protected void _setShortLE(int index, int value) {
setShortLE(index, value);
}

@Override
protected void _setMedium(int index, int value) {
setMedium(index, value);
}

/** @see {@link #setMediumLE(int, int)} */
@Override
protected void _setMediumLE(int index, int value) {
setMediumLE(index, value);
}

@Override
protected void _setInt(int index, int value) {
setInt(index, value);
}

/** @see {@link #setIntLE(int, int)} */
@Override
protected void _setIntLE(int index, int value) {
setIntLE(index, value);
}

@Override
protected void _setLong(int index, long value) {
setLong(index, value);
}

/** @see {@link #setLongLE(int, long)} */
@Override
public void _setLongLE(int index, long value) {
setLongLE(index, value);
}

@Override
public ArrowBuf getBytes(int index, ByteBuf dst, int dstIndex, int length) {
udle.getBytes(index + offset, dst, dstIndex, length);
Expand All @@ -716,16 +897,13 @@ public ArrowBuf getBytes(int index, OutputStream out, int length) throws IOExcep
}

@Override
protected int _getUnsignedMedium(int index) {
final long addr = addr(index);
return (PlatformDependent.getByte(addr) & 0xff) << 16 |
(PlatformDependent.getByte(addr + 1) & 0xff) << 8 |
PlatformDependent.getByte(addr + 2) & 0xff;
public int getBytes(int index, GatheringByteChannel out, int length) throws IOException {
return udle.getBytes(index + offset, out, length);
}

@Override
public int getBytes(int index, GatheringByteChannel out, int length) throws IOException {
return udle.getBytes(index + offset, out, length);
public int getBytes(int index, FileChannel out, long position, int length) throws IOException {
return udle.getBytes(index + offset, out, position, length);
}

@Override
Expand Down Expand Up @@ -776,6 +954,11 @@ public int setBytes(int index, ScatteringByteChannel in, int length) throws IOEx
return udle.setBytes(index + offset, in, length);
}

@Override
public int setBytes(int index, FileChannel in, long position, int length) throws IOException {
return udle.setBytes(index + offset, in, position, length);
}

@Override
public byte getByte(int index) {
chk(index, 1);
Expand Down
Loading